Keep JA4S_raw and JA4H_raw hardcoded to disabled

JA4 (BSD licensed) remains always enabled, but JA4+ variants (JA4S,
JA4D, JA4H, JA4L, JA4SSH, JA4T, JA4TS, JA4X) require a FoxIO license
and are now toggleable via the SOC UI. The toggle includes a license
agreement warning and defaults to disabled.

.github/workflows/pythontest.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.13"]
+        python-version: ["3.14"]
         python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin"]
 
     steps:

salt/common/tools/sbin/so-common

@@ -550,22 +550,6 @@ retry() {
         return $exitcode
 }
 
-rollover_index() {
-  idx=$1
-  exists=$(so-elasticsearch-query $idx -o /dev/null -w "%{http_code}")
-  if [[ $exists -eq 200 ]]; then
-    rollover=$(so-elasticsearch-query $idx/_rollover -o /dev/null -w "%{http_code}" -XPOST)
-
-    if [[ $rollover -eq 200 ]]; then
-      echo "Successfully triggered rollover for $idx..."
-    else
-      echo "Could not trigger rollover for $idx..."
-    fi
-  else
-    echo "Could not find index $idx..."
-  fi
-}
-
 run_check_net_err() {
 	local cmd=$1
 	local err_msg=${2:-"Unknown error occured, please check /root/$WHATWOULDYOUSAYYAHDOHERE.log for details."} # Really need to rename that variable

salt/elasticsearch/defaults.yaml

@@ -117,7 +117,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - so-case*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -129,6 +129,8 @@ elasticsearch:
                 match_mapping_type: string
           settings:
             index:
+              lifecycle:
+                name: so-case-logs
               mapping:
                 total_fields:
                   limit: 1500
@@ -139,7 +141,14 @@ elasticsearch:
               sort:
                 field: '@timestamp'
                 order: desc
+      policy:
+        phases:
+          hot:
+            actions: {}
+            min_age: 0ms
     so-common:
+      close: 30
+      delete: 365
       index_sorting: false
       index_template:
         composed_of:
@@ -203,9 +212,7 @@ elasticsearch:
         - common-settings
         - common-dynamic-mappings
         - winlog-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-*-so*
@@ -265,7 +272,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - so-detection*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -277,6 +284,8 @@ elasticsearch:
                 match_mapping_type: string
           settings:
             index:
+              lifecycle:
+                name: so-detection-logs
               mapping:
                 total_fields:
                   limit: 1500
@@ -287,6 +296,11 @@ elasticsearch:
               sort:
                 field: '@timestamp'
                 order: desc
+      policy:
+        phases:
+          hot:
+            actions: {}
+            min_age: 0ms
     sos-backup:
       index_sorting: false
       index_template:
@@ -446,7 +460,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - endgame*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -494,6 +508,8 @@ elasticsearch:
                 priority: 50
             min_age: 30d
     so-idh:
+      close: 30
+      delete: 365
       index_sorting: false
       index_template:
         composed_of:
@@ -550,8 +566,8 @@ elasticsearch:
         - common-dynamic-mappings
         ignore_missing_component_templates: []
         index_patterns:
-        - logs-idh-so*
-        priority: 501
+        - so-idh-*
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -661,13 +677,11 @@ elasticsearch:
         - common-dynamic-mappings
         - winlog-mappings
         - hash-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-import-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -722,7 +736,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - so-ip*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -737,12 +751,19 @@ elasticsearch:
               mapping:
                 total_fields:
                   limit: 1500
+              lifecycle:
+                name: so-ip-mappings-logs
               number_of_replicas: 0
               number_of_shards: 1
               refresh_interval: 30s
               sort:
                 field: '@timestamp'
                 order: desc
+      policy:
+        phases:
+          hot:
+            actions: {}
+            min_age: 0ms
     so-items:
       index_sorting: false
       index_template:
@@ -751,7 +772,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - .items-default-**
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -830,6 +851,8 @@ elasticsearch:
                 priority: 50
             min_age: 30d
     so-kratos:
+      close: 30
+      delete: 365
       index_sorting: false
       index_template:
         composed_of:
@@ -850,7 +873,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - logs-kratos-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -898,6 +921,8 @@ elasticsearch:
                 priority: 50
             min_age: 30d
     so-hydra:
+      close: 30
+      delete: 365
       index_sorting: false
       index_template:
         composed_of:
@@ -958,7 +983,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - logs-hydra-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -1013,7 +1038,7 @@ elasticsearch:
         ignore_missing_component_templates: []
         index_patterns:
         - .lists-default-**
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -1499,9 +1524,6 @@ elasticsearch:
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
         - so-fleet_agent_id_verification-1
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
         ignore_missing_component_templates:
         - logs-elastic_agent.cloudbeat@custom
         index_patterns:
@@ -1737,9 +1759,6 @@ elasticsearch:
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
         - so-fleet_agent_id_verification-1
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
         ignore_missing_component_templates:
         - logs-elastic_agent.heartbeat@custom
         index_patterns:
@@ -2999,6 +3018,8 @@ elasticsearch:
                 priority: 50
             min_age: 30d
     so-logs-soc:
+      close: 30
+      delete: 365
       index_sorting: false
       index_template:
         composed_of:
@@ -3053,13 +3074,11 @@ elasticsearch:
         - dtc-user_agent-mappings
         - common-settings
         - common-dynamic-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-soc-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -3649,13 +3668,10 @@ elasticsearch:
         - vulnerability-mappings
         - common-settings
         - common-dynamic-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
         ignore_missing_component_templates: []
         index_patterns:
         - logs-logstash-default*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -3955,8 +3971,8 @@ elasticsearch:
         - common-dynamic-mappings
         ignore_missing_component_templates: []
         index_patterns:
-        - logs-redis.log*
-        priority: 501
+        - logs-redis-default*
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -4067,13 +4083,11 @@ elasticsearch:
         - common-settings
         - common-dynamic-mappings
         - hash-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-strelka-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -4183,13 +4197,11 @@ elasticsearch:
         - common-settings
         - common-dynamic-mappings
         - hash-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-suricata-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -4299,13 +4311,11 @@ elasticsearch:
         - common-settings
         - common-dynamic-mappings
         - hash-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-suricata.alerts-*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -4415,13 +4425,11 @@ elasticsearch:
         - vulnerability-mappings
         - common-settings
         - common-dynamic-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-syslog-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false
@@ -4533,13 +4541,11 @@ elasticsearch:
         - common-settings
         - common-dynamic-mappings
         - hash-mappings
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
+        data_stream: {}
         ignore_missing_component_templates: []
         index_patterns:
         - logs-zeek-so*
-        priority: 501
+        priority: 500
         template:
           mappings:
             date_detection: false

salt/hydra/enabled.sls

@@ -67,7 +67,7 @@ delete_so-hydra_so-status.disabled:
 
 wait_for_hydra:
   http.wait_for_successful_query:
-    - name: 'http://{{ GLOBALS.manager }}:4444/'
+    - name: 'http://{{ GLOBALS.manager }}:4444/health/alive'
     - ssl: True
     - verify_ssl: False
     - status:

salt/manager/tools/sbin/so-client

@@ -134,8 +134,8 @@ function require() {
 function verifyEnvironment() {
   require "jq"
   require "curl"
-  response=$(curl -Ss -L ${hydraUrl}/)
-  [[ "$response" != *"Error 404"* ]] && fail "Unable to communicate with Hydra; specify URL via HYDRA_URL environment variable"
+  response=$(curl -Ss -L ${hydraUrl}/health/alive)
+  [[ "$response" != '{"status":"ok"}' ]] && fail "Unable to communicate with Hydra; specify URL via HYDRA_URL environment variable"
 }
 
 function createFile() {

salt/manager/tools/sbin/so-yaml.py

@@ -22,7 +22,7 @@ def showUsage(args):
     print('    removelistitem   - Remove a list item from a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
     print('    replacelistobject  - Replace a list object based on a condition. Requires KEY, CONDITION_FIELD, CONDITION_VALUE, and JSON_OBJECT args.', file=sys.stderr)
     print('    add              - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr)
-    print('    get              - Displays (to stdout) the value stored in the given key. Requires KEY arg.', file=sys.stderr)
+    print('    get [-r]         - Displays (to stdout) the value stored in the given key. Requires KEY arg. Use -r for raw output without YAML formatting.', file=sys.stderr)
     print('    remove           - Removes a yaml key, if it exists. Requires KEY arg.', file=sys.stderr)
     print('    replace          - Replaces (or adds) a new key and set its value. Requires KEY and VALUE args.', file=sys.stderr)
     print('    help             - Prints this usage information.', file=sys.stderr)
@@ -332,6 +332,11 @@ def getKeyValue(content, key):
 
 
 def get(args):
+    raw = False
+    if len(args) > 0 and args[0] == '-r':
+        raw = True
+        args = args[1:]
+
     if len(args) != 2:
         print('Missing filename or key arg', file=sys.stderr)
         showUsage(None)
@@ -346,12 +351,15 @@ def get(args):
         print(f"Key '{key}' not found by so-yaml.py", file=sys.stderr)
         return 2
 
-    if isinstance(output, bool):
-        print(str(output).lower())
-    elif isinstance(output, (dict, list)):
-        print(yaml.safe_dump(output).strip())
+    if raw:
+        if isinstance(output, bool):
+            print(str(output).lower())
+        elif isinstance(output, (dict, list)):
+            print(yaml.safe_dump(output).strip())
+        else:
+            print(output)
     else:
-        print(output)
+        print(yaml.safe_dump(output))
     return 0
 
 

salt/manager/tools/sbin/so-yaml_test.py

@@ -393,6 +393,17 @@ class TestRemove(unittest.TestCase):
 
             result = soyaml.get([filename, "key1.child2.deep1"])
             self.assertEqual(result, 0)
+            self.assertIn("45\n...", mock_stdout.getvalue())
+
+    def test_get_int_raw(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get(["-r", filename, "key1.child2.deep1"])
+            self.assertEqual(result, 0)
             self.assertEqual("45\n", mock_stdout.getvalue())
 
     def test_get_str(self):
@@ -404,6 +415,17 @@ class TestRemove(unittest.TestCase):
 
             result = soyaml.get([filename, "key1.child2.deep1"])
             self.assertEqual(result, 0)
+            self.assertIn("hello\n...", mock_stdout.getvalue())
+
+    def test_get_str_raw(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: \"hello\" } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get(["-r", filename, "key1.child2.deep1"])
+            self.assertEqual(result, 0)
             self.assertEqual("hello\n", mock_stdout.getvalue())
 
     def test_get_bool(self):
@@ -415,8 +437,31 @@ class TestRemove(unittest.TestCase):
 
             result = soyaml.get([filename, "key2"])
             self.assertEqual(result, 0)
+            self.assertIn("false\n...", mock_stdout.getvalue())
+
+    def test_get_bool_raw(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get(["-r", filename, "key2"])
+            self.assertEqual(result, 0)
             self.assertEqual("false\n", mock_stdout.getvalue())
 
+    def test_get_dict_raw(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            filename = "/tmp/so-yaml_test-get.yaml"
+            file = open(filename, "w")
+            file.write("{key1: { child1: 123, child2: { deep1: 45 } }, key2: false, key3: [e,f,g]}")
+            file.close()
+
+            result = soyaml.get(["-r", filename, "key1"])
+            self.assertEqual(result, 0)
+            self.assertIn("child1: 123", mock_stdout.getvalue())
+            self.assertNotIn("...", mock_stdout.getvalue())
+
     def test_get_list(self):
         with patch('sys.stdout', new=StringIO()) as mock_stdout:
             filename = "/tmp/so-yaml_test-get.yaml"

salt/manager/tools/sbin/soup

@@ -396,22 +396,14 @@ migrate_pcap_to_suricata() {
 
   for pillar_file in "$PCAPFILE" "$MINIONDIR"/*.sls; do
     [[ -f "$pillar_file" ]] || continue
-    pcap_enabled=$(so-yaml.py get "$pillar_file" pcap.enabled 2>/dev/null) || continue
+    pcap_enabled=$(so-yaml.py get -r "$pillar_file" pcap.enabled 2>/dev/null) || continue
     so-yaml.py add "$pillar_file" suricata.pcap.enabled "$pcap_enabled"
     so-yaml.py remove "$pillar_file" pcap
   done
 }
 
 post_to_3.0.0() {
-  for idx in "logs-idh-so" "logs-redis.log-default"; do
-    rollover_index "$idx"
-  done
-
-  # Remove ILM for so-case and so-detection indices
-  for idx in "so-case" "so-casehistory" "so-detection" "so-detectionhistory"; do
-    so-elasticsearch-query $idx/_ilm/remove -XPOST
-  done
-
+  echo "Nothing to apply"
   POSTVERSION=3.0.0
 }
 

salt/zeek/config.sls

@@ -156,6 +156,9 @@ zeekja4cfg:
     - source: salt://zeek/files/config.zeek.ja4
     - user: 937
     - group: 939
+    - template: jinja
+    - defaults:
+        JA4PLUS_ENABLED: {{ ZEEKMERGED.ja4plus_enabled }}
 
 # BPF compilation failed
 {% if ZEEKBPF and not ZEEK_BPF_STATUS %}

salt/zeek/defaults.yaml

@@ -1,5 +1,6 @@
 zeek:
   enabled: False
+  ja4plus_enabled: False
   config:
     node:
       lb_procs: 0

salt/zeek/files/config.zeek.ja4

@@ -8,20 +8,20 @@ export {
   option JA4_raw:        bool = F;
 
   # FoxIO license required for JA4+
-  option JA4S_enabled:   bool = F;
+  option JA4S_enabled:   bool = {{ 'T' if JA4PLUS_ENABLED else 'F' }};
   option JA4S_raw:   bool = F;
 
-  option JA4D_enabled:  bool = F;
+  option JA4D_enabled:  bool = {{ 'T' if JA4PLUS_ENABLED else 'F' }};
 
-  option JA4H_enabled:   bool = F;
+  option JA4H_enabled:   bool = {{ 'T' if JA4PLUS_ENABLED else 'F' }};
   option JA4H_raw:   bool = F;
 
-  option JA4L_enabled:   bool = F;
-  
-  option JA4SSH_enabled: bool = F;
+  option JA4L_enabled:   bool = {{ 'T' if JA4PLUS_ENABLED else 'F' }};
 
-  option JA4T_enabled: bool = F;
-  option JA4TS_enabled: bool = F;
+  option JA4SSH_enabled: bool = {{ 'T' if JA4PLUS_ENABLED else 'F' }};
 
-  option JA4X_enabled:   bool = F;
+  option JA4T_enabled: bool = {{ 'T' if JA4PLUS_ENABLED else 'F' }};
+  option JA4TS_enabled: bool = {{ 'T' if JA4PLUS_ENABLED else 'F' }};
+
+  option JA4X_enabled:   bool = {{ 'T' if JA4PLUS_ENABLED else 'F' }};
 }

salt/zeek/soc_zeek.yaml

@@ -2,6 +2,10 @@ zeek:
   enabled:
     description: Controls whether the Zeek (network packet inspection) process runs. Disabling this process could result in loss of network protocol metadata. If Suricata was selected as the protocol metadata engine during setup then this will already be disabled.
     helpLink: zeek.html
+  ja4plus_enabled:
+    description: "Enables JA4+ fingerprinting (JA4S, JA4D, JA4H, JA4L, JA4SSH, JA4T, JA4TS, JA4X). By enabling this, you agree to the terms of the JA4+ license (https://github.com/FoxIO-LLC/ja4/blob/main/LICENSE-JA4)."
+    forcedType: bool
+    helpLink: zeek.html
   config:
     local:
       load: