Exclude README from zkg sync

Create /opt/so/conf/zeek/zkg directory and sync custom packages
from the manager via file.recurse. Bind mount the directory into
the so-zeek container so the entrypoint can install packages on
startup.

.github/workflows/pythontest.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.13"]
+        python-version: ["3.14"]
         python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin"]
 
     steps:

salt/common/tools/sbin/so-common

@@ -550,22 +550,6 @@ retry() {
         return $exitcode
 }
 
-rollover_index() {
-  idx=$1
-  exists=$(so-elasticsearch-query $idx -o /dev/null -w "%{http_code}")
-  if [[ $exists -eq 200 ]]; then
-    rollover=$(so-elasticsearch-query $idx/_rollover -o /dev/null -w "%{http_code}" -XPOST)
-
-    if [[ $rollover -eq 200 ]]; then
-      echo "Successfully triggered rollover for $idx..."
-    else
-      echo "Could not trigger rollover for $idx..."
-    fi
-  else
-    echo "Could not find index $idx..."
-  fi
-}
-
 run_check_net_err() {
 	local cmd=$1
 	local err_msg=${2:-"Unknown error occured, please check /root/$WHATWOULDYOUSAYYAHDOHERE.log for details."} # Really need to rename that variable

salt/elasticsearch/defaults.yaml

@@ -117,7 +117,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - so-case*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -129,6 +129,8 @@ elasticsearch:
                 match_mapping_type: string
           settings:
             index:
+              lifecycle:
+                name: so-case-logs
               mapping:
                 total_fields:
                   limit: 1500
@@ -139,7 +141,14 @@ elasticsearch:
               sort:
                 field: '@timestamp'
                 order: desc
+      policy:
+        phases:
+          hot:
+            actions: {}
+            min_age: 0ms
     so-common:
+      close: 30
+      delete: 365
       index_sorting: false
       index_template:
         composed_of:
@@ -203,9 +212,7 @@ elasticsearch:
         - common-settings
         - common-dynamic-mappings
         - winlog-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-*-so*
@@ -265,7 +272,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - so-detection*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -277,6 +284,8 @@ elasticsearch:
                 match_mapping_type: string
           settings:
             index:
+              lifecycle:
+                name: so-detection-logs
               mapping:
                 total_fields:
                   limit: 1500
@@ -287,6 +296,11 @@ elasticsearch:
               sort:
                 field: '@timestamp'
                 order: desc
+      policy:
+        phases:
+          hot:
+            actions: {}
+            min_age: 0ms
     sos-backup:
       index_sorting: false
       index_template:
@@ -446,7 +460,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - endgame*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -494,6 +508,8 @@ elasticsearch:
                 priority: 50
             min_age: 30d
     so-idh:
+      close: 30
+      delete: 365
       index_sorting: false
       index_template:
         composed_of:
@@ -550,8 +566,8 @@ elasticsearch:
         - common-dynamic-mappings
         ignore_missing_component_templates: []
         index_patterns:
-        - logs-idh-so*
-        priority: 501
+        - so-idh-*
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -661,13 +677,11 @@ elasticsearch:
         - common-dynamic-mappings
         - winlog-mappings
         - hash-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-import-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -722,7 +736,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - so-ip*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -737,12 +751,19 @@ elasticsearch:
               mapping:
                 total_fields:
                   limit: 1500
+              lifecycle:
+                name: so-ip-mappings-logs
               number_of_replicas: 0
               number_of_shards: 1
               refresh_interval: 30s
               sort:
                 field: '@timestamp'
                 order: desc
+      policy:
+        phases:
+          hot:
+            actions: {}
+            min_age: 0ms
     so-items:
       index_sorting: false
       index_template:
@@ -751,7 +772,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - .items-default-**
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -830,6 +851,8 @@ elasticsearch:
                 priority: 50
             min_age: 30d
     so-kratos:
+      close: 30
+      delete: 365
       index_sorting: false
       index_template:
         composed_of:
@@ -850,7 +873,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - logs-kratos-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -898,6 +921,8 @@ elasticsearch:
                 priority: 50
             min_age: 30d
     so-hydra:
+      close: 30
+      delete: 365
       index_sorting: false
       index_template:
         composed_of:
@@ -958,7 +983,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - logs-hydra-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -1013,7 +1038,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - .lists-default-**
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -1499,9 +1524,6 @@ elasticsearch:
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
         - so-fleet_agent_id_verification-1
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
         ignore_missing_component_templates:
         - logs-elastic_agent.cloudbeat@custom
         index_patterns:
@@ -1737,9 +1759,6 @@ elasticsearch:
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
         - so-fleet_agent_id_verification-1
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
         ignore_missing_component_templates:
         - logs-elastic_agent.heartbeat@custom
         index_patterns:
@@ -2999,6 +3018,8 @@ elasticsearch:
                 priority: 50
             min_age: 30d
     so-logs-soc:
+      close: 30
+      delete: 365
       index_sorting: false
       index_template:
         composed_of:
@@ -3053,13 +3074,11 @@ elasticsearch:
         - dtc-user_agent-mappings
         - common-settings
         - common-dynamic-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-soc-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -3649,13 +3668,10 @@ elasticsearch:
         - vulnerability-mappings
         - common-settings
         - common-dynamic-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
         ignore_missing_component_templates: []
         index_patterns:
         - logs-logstash-default*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -3955,8 +3971,8 @@ elasticsearch:
         - common-dynamic-mappings
         ignore_missing_component_templates: []
         index_patterns:
-        - logs-redis.log*
-        priority: 501
+        - logs-redis-default*
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -4067,13 +4083,11 @@ elasticsearch:
         - common-settings
         - common-dynamic-mappings
         - hash-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-strelka-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -4183,13 +4197,11 @@ elasticsearch:
         - common-settings
         - common-dynamic-mappings
         - hash-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-suricata-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -4299,13 +4311,11 @@ elasticsearch:
         - common-settings
         - common-dynamic-mappings
         - hash-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-suricata.alerts-*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -4415,13 +4425,11 @@ elasticsearch:
         - vulnerability-mappings
         - common-settings
         - common-dynamic-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-syslog-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -4533,13 +4541,11 @@ elasticsearch:
         - common-settings
         - common-dynamic-mappings
         - hash-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-zeek-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false

salt/hydra/enabled.sls

@@ -67,7 +67,7 @@ delete_so-hydra_so-status.disabled:
 
 wait_for_hydra:
   http.wait_for_successful_query:
-    - name: 'http://{{ GLOBALS.manager }}:4444/'
+    - name: 'http://{{ GLOBALS.manager }}:4444/health/alive'
     - ssl: True
     - verify_ssl: False
     - status:

salt/manager/tools/sbin/so-client

@@ -134,8 +134,8 @@ function require() {
 function verifyEnvironment() {
   require "jq"
   require "curl"
-  response=$(curl -Ss -L ${hydraUrl}/)
-  [[ "$response" != *"Error 404"* ]] && fail "Unable to communicate with Hydra; specify URL via HYDRA_URL environment variable"
+  response=$(curl -Ss -L ${hydraUrl}/health/alive)
+  [[ "$response" != '{"status":"ok"}' ]] && fail "Unable to communicate with Hydra; specify URL via HYDRA_URL environment variable"
 }
 
 function createFile() {

salt/manager/tools/sbin/so-yaml.py

@@ -22,7 +22,7 @@ def showUsage(args):
     print('    removelistitem   - Remove a list item from a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
     print('    replacelistobject  - Replace a list object based on a condition. Requires KEY, CONDITION_FIELD, CONDITION_VALUE, and JSON_OBJECT args.', file=sys.stderr)
     print('    add              - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr)
-    print('    get              - Displays (to stdout) the value stored in the given key. Requires KEY arg.', file=sys.stderr)
+    print('    get [-r]         - Displays (to stdout) the value stored in the given key. Requires KEY arg. Use -r for raw output without YAML formatting.', file=sys.stderr)
     print('    remove           - Removes a yaml key, if it exists. Requires KEY arg.', file=sys.stderr)
     print('    replace          - Replaces (or adds) a new key and set its value. Requires KEY and VALUE args.', file=sys.stderr)
     print('    help             - Prints this usage information.', file=sys.stderr)
@@ -332,6 +332,11 @@ def getKeyValue(content, key):
 
 
 def get(args):
+    raw = False
+    if len(args) > 0 and args[0] == '-r':
+        raw = True
+        args = args[1:]
+
     if len(args) != 2:
         print('Missing filename or key arg', file=sys.stderr)
         showUsage(None)
@@ -346,12 +351,15 @@ def get(args):
         print(f"Key '{key}' not found by so-yaml.py", file=sys.stderr)
         return 2
 
-    if isinstance(output, bool):
-        print(str(output).lower())
-    elif isinstance(output, (dict, list)):
-        print(yaml.safe_dump(output).strip())
+    if raw:
+        if isinstance(output, bool):
+            print(str(output).lower())
+        elif isinstance(output, (dict, list)):
+            print(yaml.safe_dump(output).strip())
+        else:
+            print(output)
     else:
-        print(output)
+        print(yaml.safe_dump(output))
     return 0
 
 

salt/manager/tools/sbin/so-yaml_test.py

@@ -393,6 +393,17 @@ class TestRemove(unittest.TestCase):
 
             result = soyaml.get([filename, "key1.child2.deep1"])
             self.assertEqual(result, 0)
+            self.assertIn("45\n...", mock_stdout.getvalue())
+
+    def test_get_int_raw(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get(["-r", filename, "key1.child2.deep1"])
+            self.assertEqual(result, 0)
             self.assertEqual("45\n", mock_stdout.getvalue())
 
     def test_get_str(self):
@@ -404,6 +415,17 @@ class TestRemove(unittest.TestCase):
 
             result = soyaml.get([filename, "key1.child2.deep1"])
             self.assertEqual(result, 0)
+            self.assertIn("hello\n...", mock_stdout.getvalue())
+
+    def test_get_str_raw(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: \"hello\" } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get(["-r", filename, "key1.child2.deep1"])
+            self.assertEqual(result, 0)
             self.assertEqual("hello\n", mock_stdout.getvalue())
 
     def test_get_bool(self):
@@ -415,8 +437,31 @@ class TestRemove(unittest.TestCase):
 
             result = soyaml.get([filename, "key2"])
             self.assertEqual(result, 0)
+            self.assertIn("false\n...", mock_stdout.getvalue())
+
+    def test_get_bool_raw(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get(["-r", filename, "key2"])
+            self.assertEqual(result, 0)
             self.assertEqual("false\n", mock_stdout.getvalue())
 
+    def test_get_dict_raw(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get(["-r", filename, "key1"])
+            self.assertEqual(result, 0)
+            self.assertIn("child1: 123", mock_stdout.getvalue())
+            self.assertNotIn("...", mock_stdout.getvalue())
+
     def test_get_list(self):
         with patch('sys.stdout', new=StringIO()) as mock_stdout:
             filename = "/tmp/so-yaml_test-get.yaml"

salt/manager/tools/sbin/soup

@@ -396,22 +396,14 @@ migrate_pcap_to_suricata() {
 
   for pillar_file in "$PCAPFILE" "$MINIONDIR"/*.sls; do
     [[ -f "$pillar_file" ]] || continue
-    pcap_enabled=$(so-yaml.py get "$pillar_file" pcap.enabled 2>/dev/null) || continue
+    pcap_enabled=$(so-yaml.py get -r "$pillar_file" pcap.enabled 2>/dev/null) || continue
     so-yaml.py add "$pillar_file" suricata.pcap.enabled "$pcap_enabled"
     so-yaml.py remove "$pillar_file" pcap
   done
 }
 
 post_to_3.0.0() {
-  for idx in "logs-idh-so" "logs-redis.log-default"; do
-    rollover_index "$idx"
-  done
-
-  # Remove ILM for so-case and so-detection indices
-  for idx in "so-case" "so-casehistory" "so-detection" "so-detectionhistory"; do
-    so-elasticsearch-query $idx/_ilm/remove -XPOST
-  done
-
+  echo "Nothing to apply"
   POSTVERSION=3.0.0
 }
 

salt/zeek/config.sls

@@ -32,6 +32,15 @@ zeekpolicydir:
     - group: 939
     - makedirs: True
 
+zeekzkgsync:
+  file.recurse:
+    - name: /opt/so/conf/zeek/zkg
+    - source: salt://zeek/zkg
+    - user: 937
+    - group: 939
+    - makedirs: True
+    - exclude_pat: README
+
 # Zeek Log Directory
 zeeklogdir:
   file.directory:

salt/zeek/enabled.sls

@@ -35,6 +35,7 @@ so-zeek:
       - /opt/so/conf/zeek/policy/intel:/opt/zeek/share/zeek/policy/intel:rw
       - /opt/so/conf/zeek/bpf:/opt/zeek/etc/bpf:ro
       - /opt/so/conf/zeek/config.zeek:/opt/zeek/share/zeek/site/packages/ja4/config.zeek:ro
+      - /opt/so/conf/zeek/zkg:/opt/so/conf/zeek/zkg:ro
       {% if DOCKER.containers['so-zeek'].custom_bind_mounts %}
         {% for BIND in DOCKER.containers['so-zeek'].custom_bind_mounts %}
       - {{ BIND }}

salt/zeek/zkg/README

@@ -0,0 +1 @@
+# Place custom Zeek packages in /opt/so/saltstack/local/salt/zeek/zkg/