Remove version 3.0.0 from discussion template

salt/common/tools/sbin/so-common

@@ -550,22 +550,6 @@ retry() {
         return $exitcode
 }
 
-rollover_index() {
-  idx=$1
-  exists=$(so-elasticsearch-query $idx -o /dev/null -w "%{http_code}")
-  if [[ $exists -eq 200 ]]; then
-    rollover=$(so-elasticsearch-query $idx/_rollover -o /dev/null -w "%{http_code}" -XPOST)
-
-    if [[ $rollover -eq 200 ]]; then
-      echo "Successfully triggered rollover for $idx..."
-    else
-      echo "Could not trigger rollover for $idx..."
-    fi
-  else
-    echo "Could not find index $idx..."
-  fi
-}
-
 run_check_net_err() {
 	local cmd=$1
 	local err_msg=${2:-"Unknown error occured, please check /root/$WHATWOULDYOUSAYYAHDOHERE.log for details."} # Really need to rename that variable

salt/elasticsearch/defaults.yaml

@@ -117,7 +117,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - so-case*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -129,6 +129,8 @@ elasticsearch:
                 match_mapping_type: string
           settings:
             index:
+              lifecycle:
+                name: so-case-logs
               mapping:
                 total_fields:
                   limit: 1500
@@ -139,7 +141,14 @@ elasticsearch:
               sort:
                 field: '@timestamp'
                 order: desc
+      policy:
+        phases:
+          hot:
+            actions: {}
+            min_age: 0ms
     so-common:
+      close: 30
+      delete: 365
       index_sorting: false
       index_template:
         composed_of:
@@ -203,9 +212,7 @@ elasticsearch:
         - common-settings
         - common-dynamic-mappings
         - winlog-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-*-so*
@@ -265,7 +272,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - so-detection*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -277,6 +284,8 @@ elasticsearch:
                 match_mapping_type: string
           settings:
             index:
+              lifecycle:
+                name: so-detection-logs
               mapping:
                 total_fields:
                   limit: 1500
@@ -287,6 +296,11 @@ elasticsearch:
               sort:
                 field: '@timestamp'
                 order: desc
+      policy:
+        phases:
+          hot:
+            actions: {}
+            min_age: 0ms
     sos-backup:
       index_sorting: false
       index_template:
@@ -446,7 +460,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - endgame*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -494,6 +508,8 @@ elasticsearch:
                 priority: 50
             min_age: 30d
     so-idh:
+      close: 30
+      delete: 365
       index_sorting: false
       index_template:
         composed_of:
@@ -550,8 +566,8 @@ elasticsearch:
         - common-dynamic-mappings
         ignore_missing_component_templates: []
         index_patterns:
-        - logs-idh-so*
-        priority: 501
+        - so-idh-*
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -661,13 +677,11 @@ elasticsearch:
         - common-dynamic-mappings
         - winlog-mappings
         - hash-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-import-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -722,7 +736,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - so-ip*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -737,12 +751,19 @@ elasticsearch:
               mapping:
                 total_fields:
                   limit: 1500
+              lifecycle:
+                name: so-ip-mappings-logs
               number_of_replicas: 0
               number_of_shards: 1
               refresh_interval: 30s
               sort:
                 field: '@timestamp'
                 order: desc
+      policy:
+        phases:
+          hot:
+            actions: {}
+            min_age: 0ms
     so-items:
       index_sorting: false
       index_template:
@@ -751,7 +772,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - .items-default-**
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -830,6 +851,8 @@ elasticsearch:
                 priority: 50
             min_age: 30d
     so-kratos:
+      close: 30
+      delete: 365
       index_sorting: false
       index_template:
         composed_of:
@@ -850,7 +873,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - logs-kratos-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -898,6 +921,8 @@ elasticsearch:
                 priority: 50
             min_age: 30d
     so-hydra:
+      close: 30
+      delete: 365
       index_sorting: false
       index_template:
         composed_of:
@@ -958,7 +983,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - logs-hydra-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -1013,7 +1038,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - .lists-default-**
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -1499,9 +1524,6 @@ elasticsearch:
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
         - so-fleet_agent_id_verification-1
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
         ignore_missing_component_templates:
         - logs-elastic_agent.cloudbeat@custom
         index_patterns:
@@ -1737,9 +1759,6 @@ elasticsearch:
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
         - so-fleet_agent_id_verification-1
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
         ignore_missing_component_templates:
         - logs-elastic_agent.heartbeat@custom
         index_patterns:
@@ -2999,6 +3018,8 @@ elasticsearch:
                 priority: 50
             min_age: 30d
     so-logs-soc:
+      close: 30
+      delete: 365
       index_sorting: false
       index_template:
         composed_of:
@@ -3053,13 +3074,11 @@ elasticsearch:
         - dtc-user_agent-mappings
         - common-settings
         - common-dynamic-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-soc-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -3649,13 +3668,10 @@ elasticsearch:
         - vulnerability-mappings
         - common-settings
         - common-dynamic-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
         ignore_missing_component_templates: []
         index_patterns:
         - logs-logstash-default*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -3955,8 +3971,8 @@ elasticsearch:
         - common-dynamic-mappings
         ignore_missing_component_templates: []
         index_patterns:
-        - logs-redis.log*
-        priority: 501
+        - logs-redis-default*
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -4067,13 +4083,11 @@ elasticsearch:
         - common-settings
         - common-dynamic-mappings
         - hash-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-strelka-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -4183,13 +4197,11 @@ elasticsearch:
         - common-settings
         - common-dynamic-mappings
         - hash-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-suricata-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -4299,13 +4311,11 @@ elasticsearch:
         - common-settings
         - common-dynamic-mappings
         - hash-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-suricata.alerts-*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -4415,13 +4425,11 @@ elasticsearch:
         - vulnerability-mappings
         - common-settings
         - common-dynamic-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-syslog-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -4533,13 +4541,11 @@ elasticsearch:
         - common-settings
         - common-dynamic-mappings
         - hash-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-zeek-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false

salt/manager/tools/sbin/so-minion

@@ -462,14 +462,19 @@ function add_sensor_to_minion() {
         echo "      lb_procs: '$CORECOUNT'"
         echo "suricata:"
         echo "  enabled: True "
-        echo "  pcap:"
-        echo "    enabled: True"
         if [[ $is_pcaplimit ]]; then
+            echo "  pcap:"
             echo "    maxsize: $MAX_PCAP_SPACE"
         fi
         echo "  config:"
         echo "    af-packet:"
         echo "      threads: '$CORECOUNT'"
+        echo "pcap:"
+        echo "  enabled: True"
+        if [[ $is_pcaplimit ]]; then
+            echo "  config:"
+            echo "    diskfreepercentage: $DFREEPERCENT"
+        fi
         echo "  "
     } >> $PILLARFILE
     if [ $? -ne 0 ]; then

salt/manager/tools/sbin/so-yaml.py

@@ -256,7 +256,7 @@ def replacelistobject(args):
 def addKey(content, key, value):
     pieces = key.split(".", 1)
     if len(pieces) > 1:
-        if pieces[0] not in content or content[pieces[0]] is None:
+        if not pieces[0] in content:
             content[pieces[0]] = {}
         addKey(content[pieces[0]], pieces[1], value)
     elif key in content:
@@ -346,12 +346,7 @@ def get(args):
         print(f"Key '{key}' not found by so-yaml.py", file=sys.stderr)
         return 2
 
-    if isinstance(output, bool):
-        print(str(output).lower())
-    elif isinstance(output, (dict, list)):
-        print(yaml.safe_dump(output).strip())
-    else:
-        print(output)
+    print(yaml.safe_dump(output))
     return 0
 
 

salt/manager/tools/sbin/so-yaml_test.py

@@ -393,7 +393,7 @@ class TestRemove(unittest.TestCase):
 
             result = soyaml.get([filename, "key1.child2.deep1"])
             self.assertEqual(result, 0)
-            self.assertEqual("45\n", mock_stdout.getvalue())
+            self.assertIn("45\n...", mock_stdout.getvalue())
 
     def test_get_str(self):
         with patch('sys.stdout', new=StringIO()) as mock_stdout:
@@ -404,18 +404,7 @@ class TestRemove(unittest.TestCase):
 
             result = soyaml.get([filename, "key1.child2.deep1"])
             self.assertEqual(result, 0)
-            self.assertEqual("hello\n", mock_stdout.getvalue())
-
-    def test_get_bool(self):
-        with patch('sys.stdout', new=StringIO()) as mock_stdout:
-            filename = "/tmp/so-yaml_test-get.yaml"
-            file = open(filename, "w")
-            file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
-            file.close()
-
-            result = soyaml.get([filename, "key2"])
-            self.assertEqual(result, 0)
-            self.assertEqual("false\n", mock_stdout.getvalue())
+            self.assertIn("hello\n...", mock_stdout.getvalue())
 
     def test_get_list(self):
         with patch('sys.stdout', new=StringIO()) as mock_stdout:

salt/manager/tools/sbin/soup

@@ -88,7 +88,7 @@ check_err() {
         echo 'No route to host'
       ;;
       160)
-        echo 'Incompatible Elasticsearch upgrade'
+        echo 'Incompatiable Elasticsearch upgrade'
       ;;
       161)
         echo 'Required intermediate Elasticsearch upgrade not complete'
@@ -362,7 +362,7 @@ preupgrade_changes() {
     # This function is to add any new pillar items if needed.
     echo "Checking to see if changes are needed."
 
-    [[ "$INSTALLEDVERSION" =~ ^2\.4\.21[0-9]+$ ]] && up_to_3.0.0   
+    [[ "$INSTALLEDVERSION" == 2.4.210 ]] && up_to_3.0.0   
     true
 }
 
@@ -370,12 +370,12 @@ postupgrade_changes() {
     # This function is to add any new pillar items if needed.
     echo "Running post upgrade processes."
 
-    [[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0
+    [[ "$POSTVERSION" == 2.4.210 ]] && post_to_3.0.0
     true
 }
 
 check_minimum_version() {
-  if [[ ! "$INSTALLEDVERSION" =~ ^(2\.4\.21[0-9]+|3\.) ]]; then
+  if [[ "$INSTALLEDVERSION" != "2.4.210" ]] && [[ ! "$INSTALLEDVERSION" =~ ^3\. ]]; then
     echo "You must be on at least Security Onion 2.4.210 to upgrade. Currently installed version: $INSTALLEDVERSION"
     exit 1
   fi
@@ -385,33 +385,12 @@ check_minimum_version() {
 
 up_to_3.0.0() {
   determine_elastic_agent_upgrade
-  migrate_pcap_to_suricata
 
   INSTALLEDVERSION=3.0.0
 }
 
-migrate_pcap_to_suricata() {
-  local MINIONDIR=/opt/so/saltstack/local/pillar/minions
-  local PCAPFILE=/opt/so/saltstack/local/pillar/pcap/soc_pcap.sls
-
-  for pillar_file in "$PCAPFILE" "$MINIONDIR"/*.sls; do
-    [[ -f "$pillar_file" ]] || continue
-    pcap_enabled=$(so-yaml.py get "$pillar_file" pcap.enabled 2>/dev/null) || continue
-    so-yaml.py add "$pillar_file" suricata.pcap.enabled "$pcap_enabled"
-    so-yaml.py remove "$pillar_file" pcap
-  done
-}
-
 post_to_3.0.0() {
-  for idx in "logs-idh-so" "logs-redis.log-default"; do
-    rollover_index "$idx"
-  done
-
-  # Remove ILM for so-case and so-detection indices
-  for idx in "so-case" "so-casehistory" "so-detection" "so-detectionhistory"; do
-    so-elasticsearch-query $idx/_ilm/remove -XPOST
-  done
-
+  echo "Nothing to apply"
   POSTVERSION=3.0.0
 }
 

salt/nginx/etc/nginx.conf

@@ -387,13 +387,15 @@ http {
 		error_page 429 = @error429;
 
 		location @error401 {
-			if ($request_uri ~* (^/api/.*|^/connect/.*|^/oauth2/.*)) {
+			if ($request_uri ~* (^/connect/.*|^/oauth2/.*)) {
 				return    401;
 			}
 
-			add_header    Strict-Transport-Security   "max-age=31536000; includeSubDomains";
+			if ($request_uri ~* ^/(?!(^/api/.*))) {
+				add_header    Strict-Transport-Security   "max-age=31536000; includeSubDomains";
+			}
 
-			if ($request_uri ~* ^/(?!(login|auth|oauth2|$))) {
+			if ($request_uri ~* ^/(?!(api/|login|auth|oauth2|$))) {
 				add_header    Set-Cookie "AUTH_REDIRECT=$request_uri;Path=/;Max-Age=14400";
 			}
 			return        302 /auth/self-service/login/browser;

salt/suricata/defaults.yaml

@@ -1,7 +1,6 @@
 suricata:
   enabled: False
   pcap:
-    enabled: "no"
     filesize: 1000mb
     maxsize: 25
     compression: "none"
@@ -142,6 +141,8 @@ suricata:
         enabled: "no"
       tls-store:
         enabled: "no"
+      pcap-log:
+        enabled: "no"
       alert-debug:
         enabled: "no"
       alert-prelude:

salt/suricata/map.jinja

@@ -11,18 +11,13 @@
 {# before we change outputs back to list, enable pcap-log if suricata is the pcapengine #}
 {% if GLOBALS.pcap_engine in ["SURICATA"] %}
 
-{# initialize pcap-log in config.outputs since we dont put it in defaults #}
-{%   if 'pcap-log' not in SURICATAMERGED.config.outputs %}
-{%     do SURICATAMERGED.config.outputs.update({'pcap-log': {}}) %}
-{%   endif %}
-
 {%   from 'bpf/pcap.map.jinja' import PCAPBPF, PCAP_BPF_STATUS %}
 {%   if PCAPBPF and PCAP_BPF_STATUS %}
 {%     do SURICATAMERGED.config.outputs['pcap-log'].update({'bpf-filter': PCAPBPF|join(" ")}) %}
 {%   endif %}
 
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'enabled': 'yes'}) %}
 {#   move the items in suricata.pcap into suricata.config.outputs.pcap-log. these items were placed under suricata.config for ease of access in SOC #}
-{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'enabled': SURICATAMERGED.pcap.enabled}) %}
 {%   do SURICATAMERGED.config.outputs['pcap-log'].update({'compression': SURICATAMERGED.pcap.compression}) %}
 {%   do SURICATAMERGED.config.outputs['pcap-log'].update({'lz4-checksum': SURICATAMERGED.pcap['lz4-checksum']}) %}
 {%   do SURICATAMERGED.config.outputs['pcap-log'].update({'lz4-level': SURICATAMERGED.pcap['lz4-level']}) %}

salt/suricata/soc_suricata.yaml

@@ -22,10 +22,6 @@ suricata:
       title: Classifications
       helpLink: suricata.html
   pcap:
-    enabled:
-      description: Enables or disables the Suricata packet recording process.
-      forcedType: bool
-      helpLink: suricata.html
     filesize: 
       description: Maximum file size for individual PCAP files written by Suricata. Increasing this number could improve write performance at the expense of pcap retrieval time.
       advanced: True
@@ -213,6 +209,12 @@ suricata:
               header:
                 description: Header name where the actual IP address will be reported.
                 helpLink: suricata.html
+      pcap-log:
+        enabled:
+          description: This value is ignored by SO. pcapengine in globals takes precedence.
+          readonly: True
+          helpLink: suricata.html
+          advanced: True        
     asn1-max-frames:
       description: Maximum nuber of asn1 frames to decode.
       helpLink: suricata.html