Merge pull request #16278 from Security-Onion-Solutions/evtx-import-datastreams

EVTX Import cleanup

salt/common/tools/sbin/so-log-check

@@ -177,6 +177,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unexpected authorization header" # expected WARN log lines indicating invalid auth header
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Missing ory_kratos_session cookie" # expected WARN log lines indicating invalid auth header
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Static assets preprocessor only supports GET and HEAD requests" # expected WARN log lines indicating invalid auth header
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|respondError"                 # respondError is a function name, output via http middleware as standard request logging
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
@@ -240,7 +241,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal"           # docker container getting recycled
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown"             # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown"             # so-dockerregistry logs a tag lookup miss during image copy; not tied to one docker version
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Could not index event to Elasticsearch.*\"version\" => \"9.0.8\"" # Expected during Elastic upgrade temporarily, as policies referencing older pipelines are updated
 fi
 

salt/docker/init.sls

@@ -18,10 +18,10 @@ dockergroup:
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 2.2.1-1.el9
+      - containerd.io: 2.3.6-1.el9
-      - docker-ce: 3:29.2.1-1.el9
+      - docker-ce: 3:29.8.1-1.el9
-      - docker-ce-cli: 1:29.2.1-1.el9
+      - docker-ce-cli: 1:29.8.1-1.el9
-      - docker-ce-rootless-extras: 29.2.1-1.el9
+      - docker-ce-rootless-extras: 29.8.1-1.el9
     - hold: True
     - update_holds: True
 

salt/elastalert/config.sls

@@ -21,6 +21,7 @@ elastalert:
     - gid: 933
     - home: /opt/so/conf/elastalert
     - createhome: False
+    - shell: /sbin/nologin
 
 elastalogdir:
   file.directory:

salt/elastic-fleet-package-registry/config.sls

@@ -19,6 +19,7 @@ elastic-agent-pr:
     - gid: 948
     - home: /opt/so/conf/elastic-fleet-pr
     - createhome: False
+    - shell: /sbin/nologin
 
 {% else %}
 

salt/elasticagent/config.sls

@@ -20,6 +20,7 @@ elastic-agent:
     - gid: 949
     - home: /opt/so/conf/elastic-agent
     - createhome: False
+    - shell: /sbin/nologin
 
 elasticagentconfdir:
   file.directory:

salt/elasticfleet/config.sls

@@ -26,6 +26,7 @@ elastic-fleet:
     - gid: 947
     - home: /opt/so/conf/elastic-fleet
     - createhome: False
+    - shell: /sbin/nologin
 
 elasticfleet_sbin:
   file.recurse:

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -29,7 +29,7 @@
               "\\.gz$"
             ],
             "include_files": [],
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.22.3\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.9.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.22.3\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.22.3\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.9.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- add_fields:\n    target: event\n    fields:\n      dataset: windows.forwarded\n      module: windows\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: import.evtx\n- if:\n    equals:\n      winlog.channel: 'Security'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: system.security\n          module: system\n- if:\n    equals:\n      winlog.channel: 'Windows PowerShell'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell\n          module: windows\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n          module: system\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n          module: system\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n- add_fields:\n    target: data_stream\n    fields:\n        type: logs\n        dataset: import",
             "tags": [
               "import"
             ],

salt/elasticfleet/integration-defaults.map.jinja

@@ -30,17 +30,14 @@
     'azure_metrics.monitor': 'azure.monitor',
     'azure_metrics.storage_account': 'azure.storage_account',
     'azure_openai.metrics': 'azure.open_ai',
-    'beat.state': 'beats.stack_monitoring.state',
-    'beat.stats': 'beats.stack_monitoring.stats',
-    'enterprisesearch.health': 'enterprisesearch.stack_monitoring.health',
-    'enterprisesearch.stats': 'enterprisesearch.stack_monitoring.stats',
     'kibana.cluster_actions': 'kibana.stack_monitoring.cluster_actions',
     'kibana.cluster_rules': 'kibana.stack_monitoring.cluster_rules',
     'kibana.node_actions': 'kibana.stack_monitoring.node_actions',
     'kibana.node_rules': 'kibana.stack_monitoring.node_rules',
     'kibana.stats': 'kibana.stack_monitoring.stats',
     'kibana.status': 'kibana.stack_monitoring.status',
-    'logstash.node_cel': 'logstash.stack_monitoring.node',
+    'logstash.node': 'logstash.stack_monitoring.node',
+    'logstash.node_cel': 'logstash.node',
     'logstash.node_stats': 'logstash.stack_monitoring.node_stats',
     'synthetics.browser': 'synthetics-browser',
     'synthetics.browser_network': 'synthetics-browser.network',

salt/elasticsearch/config.sls

@@ -32,6 +32,7 @@ elasticsearch:
     - gid: 930
     - home: /opt/so/conf/elasticsearch
     - createhome: False
+    - shell: /sbin/nologin
 
 elasticsearch_sbin:
   file.recurse:

salt/elasticsearch/defaults.yaml

@@ -3309,6 +3309,7 @@ elasticsearch:
         composed_of:
         - event-mappings
         - logs-system.security@package
+        - so-fleet_system.security_caseless-1
         - logs-system.security@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -4175,6 +4176,7 @@ elasticsearch:
       index_template:
         composed_of:
         - logs-windows.forwarded@package
+        - so-fleet_process_caseless-1
         - logs-windows.forwarded@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -4224,6 +4226,7 @@ elasticsearch:
       index_template:
         composed_of:
         - logs-windows.powershell@package
+        - so-fleet_process_caseless-1
         - logs-windows.powershell@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -4273,6 +4276,7 @@ elasticsearch:
       index_template:
         composed_of:
         - logs-windows.powershell_operational@package
+        - so-fleet_process_caseless-1
         - logs-windows.powershell_operational@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -4322,6 +4326,7 @@ elasticsearch:
       index_template:
         composed_of:
         - logs-windows.sysmon_operational@package
+        - so-fleet_process_caseless-1
         - logs-windows.sysmon_operational@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1

salt/elasticsearch/files/ingest/global@custom

@@ -99,7 +99,7 @@
         },
         {
             "set": {
-                "if": "ctx.tags != null && ctx.tags.contains('import')",
+                "if": "ctx.tags != null && ctx.tags.contains('import') && ctx._index != null && ctx._index.startsWith('logs-import-')",
                 "override": true,
                 "field": "data_stream.dataset",
                 "value": "import"
@@ -107,7 +107,7 @@
         },
         {
             "set": {
-                "if": "ctx.tags != null && ctx.tags.contains('import')",
+                "if": "ctx.tags != null && ctx.tags.contains('import') && ctx._index != null && ctx._index.startsWith('logs-import-')",
                 "override": true,
                 "field": "data_stream.namespace",
                 "value": "so"

salt/elasticsearch/files/ingest/import.evtx

@@ -0,0 +1,31 @@
+{
+  "description" : "import.evtx: normalize imported EVTX and reroute to logs-<dataset>-import",
+  "processors" : [
+    { "script":  {
+        "description": "Host from the event, not the importing node",
+        "lang": "painless",
+        "source": "Map host = ['os': ['type': 'windows', 'family': 'windows', 'platform': 'windows']]; def cn = ctx.winlog?.computer_name; if (cn != null && cn.toString().length() > 0) { String name = cn.toString(); int dot = name.indexOf('.'); if (dot > 0) { name = name.substring(0, dot); } host.put('hostname', name); host.put('name', name.toLowerCase()); } ctx.host = host;"
+    } },
+    { "script":  {
+        "description": "String event IDs, as Winlogbeat sends",
+        "lang": "painless",
+        "source": "if (ctx.winlog?.event_id != null) { ctx.winlog.event_id = ctx.winlog.event_id.toString(); } if (ctx.event?.code != null) { ctx.event.code = ctx.event.code.toString(); }"
+    } },
+    { "script":  {
+        "description": "Unnamed <Data> to param1..N, as Winlogbeat",
+        "lang": "painless",
+        "if": "ctx.winlog?.event_data?.Data instanceof Map && ctx.winlog.event_data.Data['#text'] != null",
+        "source": "def t = ctx.winlog.event_data.Data['#text']; List vals = t instanceof List ? t : [t]; for (int i = 0; i < vals.size(); i++) { ctx.winlog.event_data['param' + (i + 1)] = vals.get(i); } ctx.winlog.event_data.remove('Data');"
+    } },
+    { "script":  {
+        "description": "String values and LF line endings, as Winlogbeat",
+        "lang": "painless",
+        "if": "ctx.winlog?.event_data instanceof Map || ctx.winlog?.user_data instanceof Map",
+        "source": "String lf = String.valueOf((char) 10); String crlf = String.valueOf((char) 13) + lf; for (def key : ['event_data', 'user_data']) { def m = ctx.winlog[key]; if (!(m instanceof Map)) { continue; } for (def e : m.entrySet()) { def v = e.getValue(); if (v instanceof String) { e.setValue(v.replace(crlf, lf)); } else if (v instanceof Number || v instanceof Boolean) { e.setValue(v.toString()); } } }"
+    } },
+    { "set":     { "description": "event.kind, as Winlogbeat", "field": "event.kind", "value": "event", "override": false } },
+    { "set":     { "field": "data_stream.dataset", "copy_from": "event.dataset", "override": true, "ignore_empty_value": true } },
+    { "set":     { "field": "data_stream.namespace", "value": "import", "override": true } },
+    { "reroute": { "dataset": "{{data_stream.dataset}}", "namespace": "{{data_stream.namespace}}" } }
+  ]
+}

salt/elasticsearch/templates/component/elastic-agent/so-fleet_process_caseless-1.json

@@ -0,0 +1,123 @@
+{
+  "_meta": {
+    "managed_by": "security_onion",
+    "managed": true,
+    "description": "Adds .caseless for Lucene queries. Restates each field's package type and .text."
+  },
+  "template": {
+    "mappings": {
+      "properties": {
+        "process": {
+          "properties": {
+            "executable": {
+              "type": "keyword",
+              "ignore_above": 1024,
+              "fields": {
+                "caseless": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "normalizer": "lowercase"
+                },
+                "text": {
+                  "type": "match_only_text"
+                }
+              }
+            },
+            "name": {
+              "type": "keyword",
+              "ignore_above": 1024,
+              "fields": {
+                "caseless": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "normalizer": "lowercase"
+                },
+                "text": {
+                  "type": "match_only_text"
+                }
+              }
+            },
+            "command_line": {
+              "type": "wildcard",
+              "ignore_above": 1024,
+              "fields": {
+                "caseless": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "normalizer": "lowercase"
+                },
+                "text": {
+                  "type": "match_only_text"
+                }
+              }
+            },
+            "parent": {
+              "properties": {
+                "executable": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "fields": {
+                    "caseless": {
+                      "type": "keyword",
+                      "ignore_above": 1024,
+                      "normalizer": "lowercase"
+                    },
+                    "text": {
+                      "type": "match_only_text"
+                    }
+                  }
+                },
+                "name": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "fields": {
+                    "caseless": {
+                      "type": "keyword",
+                      "ignore_above": 1024,
+                      "normalizer": "lowercase"
+                    },
+                    "text": {
+                      "type": "match_only_text"
+                    }
+                  }
+                },
+                "command_line": {
+                  "type": "wildcard",
+                  "ignore_above": 1024,
+                  "fields": {
+                    "caseless": {
+                      "type": "keyword",
+                      "ignore_above": 1024,
+                      "normalizer": "lowercase"
+                    },
+                    "text": {
+                      "type": "match_only_text"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
+        "file": {
+          "properties": {
+            "path": {
+              "type": "keyword",
+              "ignore_above": 1024,
+              "fields": {
+                "caseless": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "normalizer": "lowercase"
+                },
+                "text": {
+                  "type": "match_only_text"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/elastic-agent/so-fleet_system.security_caseless-1.json

@@ -0,0 +1,80 @@
+{
+  "_meta": {
+    "managed_by": "security_onion",
+    "managed": true,
+    "description": "Adds .caseless for Lucene queries. Keeps each field's existing keyword type."
+  },
+  "template": {
+    "mappings": {
+      "properties": {
+        "process": {
+          "properties": {
+            "command_line": {
+              "type": "keyword",
+              "ignore_above": 1024,
+              "fields": {
+                "caseless": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "normalizer": "lowercase"
+                }
+              }
+            },
+            "parent": {
+              "properties": {
+                "executable": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "fields": {
+                    "caseless": {
+                      "type": "keyword",
+                      "ignore_above": 1024,
+                      "normalizer": "lowercase"
+                    }
+                  }
+                },
+                "name": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "fields": {
+                    "caseless": {
+                      "type": "keyword",
+                      "ignore_above": 1024,
+                      "normalizer": "lowercase"
+                    }
+                  }
+                },
+                "command_line": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "fields": {
+                    "caseless": {
+                      "type": "keyword",
+                      "ignore_above": 1024,
+                      "normalizer": "lowercase"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
+        "file": {
+          "properties": {
+            "path": {
+              "type": "keyword",
+              "ignore_above": 1024,
+              "fields": {
+                "caseless": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "normalizer": "lowercase"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/kafka/config.sls

@@ -21,6 +21,7 @@ kafka_user:
     - gid: 960
     - home: /opt/so/conf/kafka
     - createhome: False
+    - shell: /sbin/nologin
 
 kafka_home_dir:
   file.absent:

salt/kibana/config.sls

@@ -22,6 +22,7 @@ kibana:
     - gid: 932
     - home: /opt/so/conf/kibana
     - createhome: False
+    - shell: /sbin/nologin
 
 # Drop the correct nginx config based on role
 

salt/kratos/config.sls

@@ -27,6 +27,7 @@ kratos:
     - uid: 928
     - gid: 928
     - home: /opt/so/conf/kratos
+    - shell: /sbin/nologin
 
 kratosdir:
   file.directory:

salt/logstash/config.sls

@@ -35,6 +35,7 @@ logstash:
     - uid: 931
     - gid: 931
     - home: /opt/so/conf/logstash
+    - shell: /sbin/nologin
 
 logstash_sbin:
   file.recurse:

salt/manager/tools/sbin/so-yaml.py

@@ -42,7 +42,8 @@ def loadYaml(filename):
     try:
         with open(filename, "r") as file:
             content = file.read()
-            return yaml.safe_load(content)
+            loaded = yaml.safe_load(content)
+            return loaded if loaded is not None else {}
     except FileNotFoundError:
         print(f"File not found: {filename}", file=sys.stderr)
         sys.exit(1)

salt/manager/tools/sbin/so-yaml_test.py

@@ -95,6 +95,20 @@ class TestRemove(unittest.TestCase):
         expected = "key1:\n  child1: 123\n  child2:\n    deep2: ab\nkey2: false\n"
         self.assertEqual(actual, expected)
 
+    def test_remove_empty_file(self):
+        filename = "/tmp/so-yaml_test-remove-empty.yaml"
+        file = open(filename, "w")
+        file.close()
+
+        code = soyaml.remove([filename, "key1"])
+        self.assertEqual(code, 0)
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        self.assertEqual(actual, "{}\n")
+
     def test_remove_missing_args(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
             with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -294,6 +308,36 @@ class TestRemove(unittest.TestCase):
         expected = "key1:\n  child1: 123\n  child2:\n    deep1: 45\n    deep2: d\nkey2: false\nkey3:\n- e\n- f\n- g\n"
         self.assertEqual(actual, expected)
 
+    def test_add_empty_file(self):
+        filename = "/tmp/so-yaml_test-add-empty.yaml"
+        file = open(filename, "w")
+        file.close()
+
+        code = soyaml.add([filename, "telegraf.output", "BOTH"])
+        self.assertEqual(code, 0)
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "telegraf:\n  output: BOTH\n"
+        self.assertEqual(actual, expected)
+
+    def test_add_empty_file_simple(self):
+        filename = "/tmp/so-yaml_test-add-empty-simple.yaml"
+        file = open(filename, "w")
+        file.close()
+
+        code = soyaml.add([filename, "telegraf", "BOTH"])
+        self.assertEqual(code, 0)
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "telegraf: BOTH\n"
+        self.assertEqual(actual, expected)
+
     def test_replace_missing_arg(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
             with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -346,6 +390,21 @@ class TestRemove(unittest.TestCase):
         expected = "key1:\n  child1: 123\n  child2:\n    deep1: 46\nkey2: false\nkey3:\n- e\n- f\n- g\n"
         self.assertEqual(actual, expected)
 
+    def test_replace_empty_file(self):
+        filename = "/tmp/so-yaml_test-replace-empty.yaml"
+        file = open(filename, "w")
+        file.close()
+
+        code = soyaml.replace([filename, "telegraf.output", "BOTH"])
+        self.assertEqual(code, 0)
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "telegraf:\n  output: BOTH\n"
+        self.assertEqual(actual, expected)
+
     def test_convert(self):
         self.assertEqual(soyaml.convertType("foo"), "foo")
         self.assertEqual(soyaml.convertType("foo.bar"), "foo.bar")
@@ -506,6 +565,18 @@ class TestRemove(unittest.TestCase):
             self.assertEqual(result, 2)
             self.assertEqual("", mock_stdout.getvalue())
 
+    def test_get_empty_file(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                filename = "/tmp/so-yaml_test-get-empty.yaml"
+                file = open(filename, "w")
+                file.close()
+
+                result = soyaml.get([filename, "telegraf.output"])
+                self.assertEqual(result, 2)
+                self.assertEqual("", mock_stdout.getvalue())
+                self.assertIn("Key 'telegraf.output' not found by so-yaml.py", mock_stderr.getvalue())
+
     def test_get_usage(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
             with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -991,3 +1062,29 @@ class TestLoadYaml(unittest.TestCase):
                     soyaml.loadYaml("/tmp/so-yaml_test-unreadable.yaml")
                     sysmock.assert_called_with(1)
                     self.assertIn("Error reading file", mock_stderr.getvalue())
+
+    def test_load_yaml_empty_file(self):
+        filename = "/tmp/so-yaml_test-load-empty.yaml"
+        file = open(filename, "w")
+        file.close()
+
+        result = soyaml.loadYaml(filename)
+        self.assertEqual(result, {})
+
+    def test_load_yaml_whitespace_only(self):
+        filename = "/tmp/so-yaml_test-load-whitespace.yaml"
+        file = open(filename, "w")
+        file.write("   \n\n  \n")
+        file.close()
+
+        result = soyaml.loadYaml(filename)
+        self.assertEqual(result, {})
+
+    def test_load_yaml_comments_only(self):
+        filename = "/tmp/so-yaml_test-load-comments.yaml"
+        file = open(filename, "w")
+        file.write("# Just a comment\n# Another comment\n")
+        file.close()
+
+        result = soyaml.loadYaml(filename)
+        self.assertEqual(result, {})

salt/manager/tools/sbin/soup

@@ -1183,6 +1183,13 @@ up_to_3.4.0() {
   echo "Removing so-kratos, so-hydra and so-soc so they are recreated on the soauth network."
   docker rm -f so-kratos so-hydra so-soc >> $SOUP_LOG 2>&1
 
+  for template in so-metrics-logstash.node so-metrics-logstash.stack_monitoring.node; do
+    if ! remove_elasticsearch_index_template "$template" "logstash node and node_cel index patterns reversed"; then
+      FINAL_MESSAGE_QUEUE+=("WARNING: Unable to automatically remove the $template index template. Addon integration templates may fail to load until it is removed:")
+      FINAL_MESSAGE_QUEUE+=("    - sudo so-elasticsearch-query _index_template/$template -XDELETE && so-checkin")
+    fi
+  done
+
   INSTALLEDVERSION=3.4.0
 }
 
@@ -1248,6 +1255,10 @@ valid_soauth_range() {
 }
 
 post_to_3.4.0() {
+  for idx in "metrics-logstash.node-default" "metrics-logstash.stack_monitoring.node-default"; do
+    rollover_index "$idx"
+  done
+
   set_postversion 3.4.0
 }
 ### 3.4.0 End ###

salt/repo/client/map.jinja

@@ -9,7 +9,7 @@
        'epel-testing.repo',
        'saltstack.repo',
        'salt-latest.repo',
-       'wazuh.repo'
+       'wazuh.repo',
        'Rocky-Base.repo',
        'Rocky-CR.repo',
        'Rocky-Debuginfo.repo',

salt/soc/defaults.yaml

@@ -1496,7 +1496,7 @@ soc:
           verifyCert: false
         notification:
           dismissedPruneDays: 30
-          enabled: false
+          enabled: true
         playbook:
           autoUpdateEnabled: true
           playbookImportFrequencySeconds: 86400
@@ -1561,6 +1561,14 @@ soc:
           reconcilePersona: ""
           toolUseTurnAttempts: 12
           toolUseTurnDelayMs: 175
+          agentSessionMaxTurns: 20
+          agentStreamFlushIntervalMs: 1000
+          agentStreamIdleTimeoutSeconds: 300
+          automationSettings:
+            tickIntervalSeconds: 60
+            maxConcurrentItems: 4
+            maxQueuedItems: 0
+            alertTriageEpoch: "2026-09-24T00:00:00Z"
           tools:
             filterEventFields:
               - "@timestamp"
@@ -2799,7 +2807,7 @@ soc:
             - id: sonnet
               displayName: Claude Sonnet
               origin: USA
-              contextLimitSmall: 200000
+              contextLimitSmall: 1000000
               contextLimitLarge: 1000000
               lowBalanceColorAlert: 500000
               enabled: true

salt/soc/files/soc/sigma_playbook_pipeline.yaml

@@ -2,13 +2,13 @@ name: Security Onion - Playbook Pipeline
 priority: 97
 transformations:
     # Route to lowercase-normalized .caseless subfields for case-insensitive matching.
-    # file.path.caseless exists on Defend only (Sysmon file events lack it);
     # registry.path / dll.path / file.name have no .caseless on any source.
     - id: case_insensitive_string_fields
       type: field_name_mapping
       mapping:
         process.executable: process.executable.caseless
         process.parent.executable: process.parent.executable.caseless
+        process.parent.name: process.parent.name.caseless
         process.command_line: process.command_line.caseless
         process.parent.command_line: process.parent.command_line.caseless
         file.path: file.path.caseless

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -26,6 +26,16 @@ transformations:
       type: set_state
       key: keep
       val: "_id, _index, _source"
+    # Not every source maps .caseless; EQL/ES|QL already match case-insensitively.
+    - id: caseless_to_parent_fields
+      type: field_name_mapping
+      mapping:
+        process.executable.caseless: process.executable
+        process.name.caseless: process.name
+        process.parent.executable.caseless: process.parent.executable
+        process.parent.name.caseless: process.parent.name
+        target.process.executable.caseless: target.process.executable
+        target.process.name.caseless: target.process.name
     - id: baseline_field_name_mapping
       type: field_name_mapping
       mapping:

salt/soc/soc_soc.yaml

@@ -160,6 +160,7 @@ soc:
         description: Schedules that are shared across the Security Onion product. Modify via one of the SOC Schedules view.
         readonlyUi: True
         global: True
+        advanced: True
         forcedType: string
         syntax: json
         storage: db
@@ -495,6 +496,7 @@ soc:
             description: JSON list of notifications. Modify via the SOC Notifications view.
             readonlyUi: True
             global: True
+            advanced: True
             forcedType: string
             syntax: json
             storage: db
@@ -503,6 +505,10 @@ soc:
             description: The number of days to retain dismissed notifications. When a notification is dismissed, it will be pruned after this many days. Only one user need dismiss a notification for it to be pruned.
             forcedType: int
             global: True
+          maxListLimit:
+            description: Maximum number of notifications to display.
+            forcedType: int
+            global: True
           enabled:
             description: Enables or disables the SOC notification module.
             forcedType: bool
@@ -533,6 +539,48 @@ soc:
             global: True
             sensitive: True
             advanced: True
+        postgresmetrics:
+          host:
+            description: Hostname or IP address of the PostgreSQL server used by Telegraf. Defaults to the manager hostname.
+            global: True
+            advanced: True
+          port:
+            description: Port of the PostgreSQL server used by Telegraf.
+            global: True
+            advanced: True
+          sslMode:
+            description: "Use encrypted connections to the PostgreSQL server used by Telegraf. Must be one of the following values: disable, allow, prefer, require, verify-ca, verify-full."
+            global: True
+            advanced: True
+          database:
+            description: Database to authenticate to on the PostgreSQL server.
+            global: True
+            advanced: True
+          user:
+            description: Username to authenticate to the PostgreSQL server used by Telegraf.
+            global: True
+            advanced: True
+          password:
+            description: Password used to authenticate to the PostgreSQL server used by Telegraf.
+            global: True
+            sensitive: True
+            advanced: True
+          cacheExpirationMs:
+            description: The interval (in milliseconds) to wait before querying the DB for updated metrics.
+            global: True
+            advanced: True
+          maxMetricAgeSeconds:
+            description: The maximum age (in seconds) of metrics to display in the SOC Grid Metrics view. Metrics older than this value will not be displayed.
+            global: True
+            advanced: True
+          alarms:
+            description: JSON list of metric alarms. Modify via the SOC Grid Alarms view.
+            readonlyUi: True
+            advanced: True
+            global: True
+            forcedType: string
+            syntax: json
+            storage: db
         salt:
           longRelayTimeoutMs:
             description: Duration (in milliseconds) to wait for a response from the Salt API when executing tasks known for being long running before giving up and showing an error on the SOC UI.
@@ -791,6 +839,7 @@ soc:
                 - gemini
                 - openai_responses
                 - openai_chat
+                - openai_embeddings
             - field: apiUrl
               label: API URL
               required: False
@@ -812,6 +861,15 @@ soc:
             description: Indicates if the Assistant Module should operate in agentic mode or not. If true, agents can work together to solve tasks.
             global: True
             forcedType: bool
+          automations:
+            description: Scheduled automations for the Onion AI assistant, managed from the Agent Studio.
+            global: True
+            advanced: True
+            readonlyUi: True
+            storage: db
+            forcedType: string
+            syntax: json
+            helpLink: onion-ai
           agents:
             description: Agent definitions for the Onion AI assistant, managed from the Agent Studio. An entry naming a system agent overrides only the fields an admin may change; everything else comes from the built-in definition.
             global: True
@@ -844,6 +902,9 @@ soc:
             - field: persona
               label: Persona
               multiline: True
+            - field: maxConcurrentInstances
+              label: Max Concurrent Instances
+              forcedType: int
           skills:
             description: Skill definitions for the Onion AI assistant, managed from the Agent Studio. An entry naming a system skill overrides only its enabled state and persona addendum; its tool set comes from the built-in definition.
             global: True
@@ -947,6 +1008,43 @@ soc:
             description: The number of times to retry extracting memories from a session if errors occur.
             global: True
             advanced: True
+          agentSessionMaxTurns:
+            description: Maximum number of model turns a headless agent session, such as one started by an automation, may take before it is stopped. Turns taken by delegated sub-agents count toward this limit. A session that reaches the limit is recorded as failed.
+            global: True
+            advanced: True
+            forcedType: int
+          agentStreamFlushIntervalMs:
+            description: Milliseconds between writes of a streaming headless agent turn to the database. Lower values show progress sooner in the Agent Studio at the cost of more frequent Elasticsearch updates.
+            global: True
+            advanced: True
+            forcedType: int
+          agentStreamIdleTimeoutSeconds:
+            description: Seconds a streaming headless agent turn may go without receiving any output before it is abandoned and the session is recorded as failed. Set to 0 to disable the timeout.
+            global: True
+            advanced: True
+            forcedType: int
+          automationSettings:
+            tickIntervalSeconds:
+              description: How often, in seconds, the automation scheduler checks for automations that are due to run. Must be greater than 0.
+              global: True
+              advanced: True
+              forcedType: int
+            maxConcurrentItems:
+              description: Maximum number of automation work items that may run at the same time. Additional work items wait in the queue until a running item finishes. User chat sessions count toward this limit but are never held back by it. Set to 0 to disable the limit.
+              global: True
+              advanced: True
+              forcedType: int
+            maxQueuedItems:
+              description: Maximum number of automation work items that may wait to start. Once the queue is full, no new work items are created until the backlog drains. Set to 0 to disable the limit.
+              global: True
+              advanced: True
+              forcedType: int
+            alertTriageEpoch:
+              description: The earliest alert time the Alert Triage automation will consider. Alerts before this time are never triaged, which keeps a first run on an existing deployment from working through old history. Must be in UTC format (2026-09-24T00:00:00Z).
+              regex: '^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z)?$'
+              regexFailureMessage: Expecting date in RFC3339 format (2026-09-24T00:00:00Z)
+              global: True
+              advanced: True
           tools:
             filterEventFields:
               description: A whitelist of fields to return when OnionAI uses the query_events tool. All other fields are removed. One field per line.

salt/suricata/config.sls

@@ -64,6 +64,7 @@ suricata:
     - gid: 940
     - home: /nsm/suricata
     - createhome: False
+    - shell: /sbin/nologin
 
 socoregroupwithsuricata:
   group.present:

salt/zeek/config.sls

@@ -23,6 +23,7 @@ zeek:
     - gid: 937
     - home: /opt/so/conf/zeek
     - createhome: False
+    - shell: /sbin/nologin
 
 # Create some directories
 zeekpolicydir: