Merge pull request #16357 from Security-Onion-Solutions/mreeves/host-ca-trust

Use the host CA trust store, with the grid CA, for all containers

.github/DISCUSSION_TEMPLATE/3-0.yml

@@ -13,6 +13,7 @@ body:
         - 3.1.0
         - 3.2.0
         - 3.3.0
+        - 3.4.0
         - Other (please provide detail below)
     validations:
       required: true

.github/workflows/pythontest.yml

@@ -6,6 +6,10 @@ on:
       - "salt/sensoroni/files/analyzers/**"
       - "salt/manager/tools/sbin/**"
       - "salt/_beacons/**"
+      - "salt/elastalert/files/modules/**"
+      - "salt/telegraf/tools/sbin_jinja/**"
+      - "salt/telegraf/defaults.yaml"
+      - "salt/telegraf/soc_telegraf.yaml"
 
 jobs:
   build:
@@ -15,7 +19,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.14"]
-        python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin", "salt/_beacons"]
+        python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin", "salt/_beacons", "salt/elastalert/files/modules/so"]
 
     steps:
     - uses: actions/checkout@v3
@@ -34,3 +38,25 @@ jobs:
     - name: Test with pytest
       run: |
         PYTHONPATH=${{ matrix.python-code-path }} pytest ${{ matrix.python-code-path }} --cov=${{ matrix.python-code-path }} --doctest-modules --cov-report=term --cov-fail-under=100 --cov-config=pytest.ini
+
+  telegraf-collector:
+    # so-container-stats is a jinja template rather than an importable module, so it gets its
+    # own job: the test renders it the way salt does, then drives it with a faked docker engine
+    # and cgroup tree. No container runtime is needed.
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.14"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest jinja2 pyyaml
+    - name: Lint with flake8
+      run: |
+        flake8 salt/telegraf/tools/sbin_jinja/so-container-stats_test.py --config=pytest.ini
+    - name: Test with pytest
+      run: |
+        pytest salt/telegraf/tools/sbin_jinja/so-container-stats_test.py -v

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 3.2.0-20260729 ISO image released on 2026/07/29
+### 3.3.0-20260911 ISO image released on 2026/09/11
 
 
 ### Download and Verify
 
-3.2.0-20260729 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-3.2.0-20260729.iso
+3.3.0-20260911 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-3.3.0-20260911.iso
  
-MD5: B1E10F46DF872B655C29325DF965A4DB  
-SHA1: 0F3C7ED80F6D326B7A993C2F899B986320C01BF9  
-SHA256: 7465163C1D1ADFCDC3935530EAFB312E987C016941ADC11841B214553314D1FF  
+MD5: 12B18433D3A2198A185892FF79CF638F  
+SHA1: 2B3C2E1FA7A78ED1F956E7EDCC12E32593C14EEE  
+SHA256: 0938C73B76CE30EC9E4394D312C79EA7CAC721B6818541697279A6221F7D870D  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.2.0-20260729.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.3.0-20260911.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.2.0-20260729.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.3.0-20260911.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-3.2.0-20260729.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-3.3.0-20260911.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-3.2.0-20260729.iso.sig securityonion-3.2.0-20260729.iso
+gpg --verify securityonion-3.3.0-20260911.iso.sig securityonion-3.3.0-20260911.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Tue 28 Jul 2026 06:17:34 PM EDT using RSA key ID FE507013
+gpg: Signature made Fri 11 Sep 2026 11:23:56 AM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

VERSION

@@ -1 +1 @@
-3.3.0
+3.4.0

salt/_beacons/local_files_beacon.py

@@ -3,13 +3,14 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-# Custom salt beacon that watches the suricata/strelka rule directories for changes
-# and emits a beacon event per changed directory. This replaces the stock salt
-# `inotify` beacon, which leaks a kernel inotify instance every time the minion
-# rebuilds the beacon loader's __context__ (orphaning the old pyinotify.Notifier
-# without closing it) until fs.inotify.max_user_instances is exhausted and the
-# beacon dies with EMFILE. Polling holds zero inotify instances, so the leak is
-# impossible, and it keeps firing during state runs (no blackout).
+# Custom salt beacon that watches hand-edited directories under
+# /opt/so/saltstack/local/salt/ for changes and emits a beacon event per changed
+# directory. This replaces the stock salt `inotify` beacon, which leaks a kernel
+# inotify instance every time the minion rebuilds the beacon loader's __context__
+# (orphaning the old pyinotify.Notifier without closing it) until
+# fs.inotify.max_user_instances is exhausted and the beacon dies with EMFILE.
+# Polling holds zero inotify instances, so the leak is impossible, and it keeps
+# firing during state runs (no blackout).
 #
 # Detection is poll-based with a per-directory fingerprint persisted to
 # WATERMARK_DIR: each pass walks the directory and hashes every file's
@@ -19,9 +20,10 @@
 # up on the next one).
 #
 # Each emitted event carries the watched directory path under the configured tag
-# (e.g. salt/beacon/<minion>/rules_beacon/suricata); the push_suricata / push_strelka
-# reactors write a push intent, after which the existing so-push-drainer /
-# orch.push_batch pipeline takes over unchanged.
+# (e.g. salt/beacon/<minion>/local_files_beacon/zeek); the push_files reactor
+# looks the tag up in salt/reactor/pillar_push_map.yaml and writes a push intent,
+# after which the existing so-push-drainer / orch.push_batch pipeline takes over
+# unchanged.
 
 import hashlib
 import logging
@@ -77,7 +79,9 @@ def _fingerprint(directory):
     h = hashlib.sha1()
     if os.path.isdir(directory):
         entries = []
-        for root, _dirs, files in os.walk(directory):
+        for root, dirs, files in os.walk(directory):
+            # zkg packages are git clones; .git churn would fire a state apply on its own.
+            dirs[:] = [d for d in dirs if d != '.git']
             for name in files:
                 full = os.path.join(root, name)
                 if _excluded(full):
@@ -94,20 +98,22 @@ def _fingerprint(directory):
     return h.hexdigest()
 
 
-def _watermark_file(tag):
-    return os.path.join(WATERMARK_DIR, 'rules_beacon_%s.hash' % tag)
+def _watermark_file(tag, directory):
+    # Keyed by directory: zeek/policy and zeek/zkg share the tag `zeek`.
+    scope = hashlib.sha1(directory.encode('utf-8', 'surrogateescape')).hexdigest()[:12]
+    return os.path.join(WATERMARK_DIR, 'local_files_beacon_%s_%s.hash' % (tag, scope))
 
 
-def _read_watermark(tag):
+def _read_watermark(tag, directory):
     try:
-        with open(_watermark_file(tag), 'r') as f:
+        with open(_watermark_file(tag, directory), 'r') as f:
             return (f.read() or '').strip() or None
     except IOError:
         return None
 
 
-def _write_watermark(tag, digest):
-    path = _watermark_file(tag)
+def _write_watermark(tag, directory, digest):
+    path = _watermark_file(tag, directory)
     try:
         os.makedirs(WATERMARK_DIR, exist_ok=True)
         tmp = path + '.tmp'
@@ -115,7 +121,7 @@ def _write_watermark(tag, digest):
             f.write(digest)
         os.rename(tmp, path)
     except OSError:
-        log.exception('rules_beacon: failed to persist watermark to %s', path)
+        log.exception('local_files_beacon: failed to persist watermark to %s', path)
 
 
 def beacon(config):
@@ -123,17 +129,17 @@ def beacon(config):
 
     for directory, tag in _paths_from_config(config).items():
         digest = _fingerprint(directory)
-        previous = _read_watermark(tag)
+        previous = _read_watermark(tag, directory)
 
         # First run / missing watermark: seed the digest and emit nothing so a
         # fresh host does not fire a spurious fleetwide push.
         if previous is None:
-            _write_watermark(tag, digest)
+            _write_watermark(tag, directory, digest)
             continue
 
         if digest != previous:
-            _write_watermark(tag, digest)
+            _write_watermark(tag, directory, digest)
             retval.append({'tag': tag, 'path': directory})
-            log.info('rules_beacon: change detected in %s, emitting %s', directory, tag)
+            log.info('local_files_beacon: change detected in %s, emitting %s', directory, tag)
 
     return retval

salt/_beacons/local_files_beacon_test.py

@@ -0,0 +1,221 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+import hashlib
+import os
+import shutil
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import local_files_beacon
+
+
+class TestRulesBeacon(unittest.TestCase):
+
+    def setUp(self):
+        # Isolate all on-disk state (watermarks and the dirs we fingerprint) in a
+        # throwaway tree, and point WATERMARK_DIR at it so the real read/write
+        # helpers run against actual files.
+        self.tmpdir = tempfile.mkdtemp()
+        self.state = os.path.join(self.tmpdir, 'state')
+        patcher = patch.object(local_files_beacon, 'WATERMARK_DIR', self.state)
+        patcher.start()
+        self.addCleanup(patcher.stop)
+
+    def tearDown(self):
+        shutil.rmtree(self.tmpdir, ignore_errors=True)
+
+    def _make_dir(self, name, files=None):
+        path = os.path.join(self.tmpdir, name)
+        os.makedirs(path, exist_ok=True)
+        for fname, content in (files or {}).items():
+            with open(os.path.join(path, fname), 'w') as f:
+                f.write(content)
+        return path
+
+    # -- trivial contract -------------------------------------------------
+
+    def test_virtual_returns_true(self):
+        self.assertTrue(local_files_beacon.__virtual__())
+
+    def test_validate_returns_valid(self):
+        self.assertEqual(local_files_beacon.validate({}), (True, 'valid'))
+
+    # -- _paths_from_config -----------------------------------------------
+
+    def test_paths_from_config_list_of_dicts(self):
+        config = [{'interval': 10}, {'paths': {'/a': 'suricata', '/b': 'strelka'}}]
+        self.assertEqual(
+            local_files_beacon._paths_from_config(config),
+            {'/a': 'suricata', '/b': 'strelka'},
+        )
+
+    def test_paths_from_config_plain_dict(self):
+        self.assertEqual(
+            local_files_beacon._paths_from_config({'paths': {'/a': 'suricata'}}),
+            {'/a': 'suricata'},
+        )
+
+    def test_paths_from_config_skips_non_dict_items(self):
+        self.assertEqual(local_files_beacon._paths_from_config(['bogus', 42]), {})
+
+    def test_paths_from_config_paths_not_a_dict(self):
+        self.assertEqual(local_files_beacon._paths_from_config({'paths': 'nope'}), {})
+
+    def test_paths_from_config_unexpected_type(self):
+        self.assertEqual(local_files_beacon._paths_from_config('nonsense'), {})
+
+    # -- _excluded --------------------------------------------------------
+
+    def test_excluded_matches_temp_and_editor_files(self):
+        for pathname in ('/rules/foo.swp', '/rules/foo~', '/rules/4913', '/rules/.#foo'):
+            self.assertTrue(local_files_beacon._excluded(pathname), pathname)
+
+    def test_excluded_allows_real_rule_files(self):
+        self.assertFalse(local_files_beacon._excluded('/rules/suricata.rules'))
+
+    # -- _fingerprint -----------------------------------------------------
+
+    def test_fingerprint_missing_dir_is_empty_tree_digest(self):
+        missing = os.path.join(self.tmpdir, 'does-not-exist')
+        self.assertEqual(local_files_beacon._fingerprint(missing), hashlib.sha1().hexdigest())
+
+    def test_fingerprint_changes_when_content_changes(self):
+        d = self._make_dir('rules', {'a.rules': 'alert'})
+        before = local_files_beacon._fingerprint(d)
+        with open(os.path.join(d, 'a.rules'), 'w') as f:
+            f.write('alert tcp any any -> any any')  # different size
+        self.assertNotEqual(local_files_beacon._fingerprint(d), before)
+
+    def test_fingerprint_ignores_excluded_files(self):
+        d = self._make_dir('rules', {'a.rules': 'alert'})
+        before = local_files_beacon._fingerprint(d)
+        with open(os.path.join(d, 'a.rules.swp'), 'w') as f:
+            f.write('editor swap')
+        self.assertEqual(local_files_beacon._fingerprint(d), before)
+
+    def test_fingerprint_skips_unstatable_entries(self):
+        # A dangling symlink appears in os.walk's file list but os.stat raises
+        # OSError, exercising the except-continue path.
+        d = self._make_dir('rules', {'a.rules': 'alert'})
+        good = local_files_beacon._fingerprint(d)
+        os.symlink(os.path.join(d, 'missing-target'), os.path.join(d, 'broken.link'))
+        self.assertEqual(local_files_beacon._fingerprint(d), good)
+
+    def test_fingerprint_prunes_git_metadata(self):
+        # zkg packages are git clones, so the watched tree carries .git.
+        d = self._make_dir('zkg', {'pkg.zeek': 'print 1;'})
+        before = local_files_beacon._fingerprint(d)
+        git_dir = os.path.join(d, 'pkg', '.git', 'refs', 'heads')
+        os.makedirs(git_dir)
+        with open(os.path.join(git_dir, 'main'), 'w') as f:
+            f.write('0' * 40)
+        self.assertEqual(local_files_beacon._fingerprint(d), before)
+
+    def test_fingerprint_still_sees_worktree_next_to_git(self):
+        d = self._make_dir('zkg', {'pkg.zeek': 'print 1;'})
+        os.makedirs(os.path.join(d, 'pkg', '.git'))
+        before = local_files_beacon._fingerprint(d)
+        with open(os.path.join(d, 'pkg', 'scripts.zeek'), 'w') as f:
+            f.write('print 2;')
+        self.assertNotEqual(local_files_beacon._fingerprint(d), before)
+
+    # -- _read_watermark / _write_watermark -------------------------------
+
+    def test_watermark_round_trip(self):
+        local_files_beacon._write_watermark('suricata', '/rules/suricata', 'deadbeef')
+        self.assertEqual(
+            local_files_beacon._read_watermark('suricata', '/rules/suricata'), 'deadbeef')
+
+    def test_read_watermark_missing_returns_none(self):
+        self.assertIsNone(local_files_beacon._read_watermark('suricata', '/rules/suricata'))
+
+    def test_read_watermark_empty_file_returns_none(self):
+        os.makedirs(self.state, exist_ok=True)
+        with open(local_files_beacon._watermark_file('suricata', '/rules/suricata'), 'w') as f:
+            f.write('')
+        self.assertIsNone(local_files_beacon._read_watermark('suricata', '/rules/suricata'))
+
+    def test_write_watermark_swallows_oserror(self):
+        with patch.object(local_files_beacon.os, 'makedirs', side_effect=OSError):
+            local_files_beacon._write_watermark('suricata', '/rules/suricata', 'deadbeef')
+        self.assertIsNone(local_files_beacon._read_watermark('suricata', '/rules/suricata'))
+
+    def test_watermark_file_differs_per_directory_within_one_tag(self):
+        # zeek/policy and zeek/zkg share the tag 'zeek'.
+        self.assertNotEqual(
+            local_files_beacon._watermark_file('zeek', '/local/zeek/policy'),
+            local_files_beacon._watermark_file('zeek', '/local/zeek/zkg'),
+        )
+
+    def test_watermarks_are_independent_within_one_tag(self):
+        local_files_beacon._write_watermark('zeek', '/local/zeek/policy', 'policyhash')
+        local_files_beacon._write_watermark('zeek', '/local/zeek/zkg', 'zkghash')
+        self.assertEqual(
+            local_files_beacon._read_watermark('zeek', '/local/zeek/policy'), 'policyhash')
+        self.assertEqual(
+            local_files_beacon._read_watermark('zeek', '/local/zeek/zkg'), 'zkghash')
+
+    # -- beacon -----------------------------------------------------------
+
+    def _config(self, mapping):
+        return [{'paths': mapping}]
+
+    def test_beacon_seeds_first_run_and_emits_nothing(self):
+        with patch.object(local_files_beacon, '_fingerprint', return_value='hash1'), \
+             patch.object(local_files_beacon, '_read_watermark', return_value=None), \
+             patch.object(local_files_beacon, '_write_watermark') as mock_write:
+            result = local_files_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
+        self.assertEqual(result, [])
+        mock_write.assert_called_once_with('suricata', '/rules/suricata', 'hash1')
+
+    def test_beacon_emits_on_change(self):
+        with patch.object(local_files_beacon, '_fingerprint', return_value='newhash'), \
+             patch.object(local_files_beacon, '_read_watermark', return_value='oldhash'), \
+             patch.object(local_files_beacon, '_write_watermark') as mock_write:
+            result = local_files_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
+        self.assertEqual(result, [{'tag': 'suricata', 'path': '/rules/suricata'}])
+        mock_write.assert_called_once_with('suricata', '/rules/suricata', 'newhash')
+
+    def test_beacon_no_change_emits_nothing(self):
+        with patch.object(local_files_beacon, '_fingerprint', return_value='samehash'), \
+             patch.object(local_files_beacon, '_read_watermark', return_value='samehash'), \
+             patch.object(local_files_beacon, '_write_watermark') as mock_write:
+            result = local_files_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
+        self.assertEqual(result, [])
+        mock_write.assert_not_called()
+
+    def test_beacon_end_to_end_with_real_files(self):
+        # Exercise the full stack (real fingerprint + real watermark files) across
+        # two poll passes: first seeds silently, second fires after a write.
+        d = self._make_dir('rules', {'a.rules': 'alert'})
+        config = self._config({d: 'suricata'})
+
+        self.assertEqual(local_files_beacon.beacon(config), [])  # seed pass
+        self.assertEqual(local_files_beacon.beacon(config), [])  # unchanged pass
+
+        with open(os.path.join(d, 'b.rules'), 'w') as f:
+            f.write('alert tcp any any -> any any')
+        self.assertEqual(local_files_beacon.beacon(config), [{'tag': 'suricata', 'path': d}])
+
+    def test_beacon_two_dirs_one_tag_do_not_flap(self):
+        # Tag-keyed watermarks would clobber each other and emit on every pass.
+        policy = self._make_dir('zeek/policy', {'intel.dat': '#fields\tindicator'})
+        zkg = self._make_dir('zeek/zkg', {'README': 'place packages here'})
+        config = self._config({policy: 'zeek', zkg: 'zeek'})
+
+        self.assertEqual(local_files_beacon.beacon(config), [])  # seed pass
+        self.assertEqual(local_files_beacon.beacon(config), [])  # idle
+        self.assertEqual(local_files_beacon.beacon(config), [])  # still idle
+
+        with open(os.path.join(policy, 'intel.dat'), 'a') as f:
+            f.write('\nevil.com\tIntel::DOMAIN\tsource\n')
+        self.assertEqual(local_files_beacon.beacon(config), [{'tag': 'zeek', 'path': policy}])
+        self.assertEqual(local_files_beacon.beacon(config), [])  # quiet again
+
+
+if __name__ == '__main__':
+    unittest.main()

salt/_beacons/rules_beacon_test.py

@@ -1,172 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-import hashlib
-import os
-import shutil
-import tempfile
-import unittest
-from unittest.mock import patch
-
-import rules_beacon
-
-
-class TestRulesBeacon(unittest.TestCase):
-
-    def setUp(self):
-        # Isolate all on-disk state (watermarks and the dirs we fingerprint) in a
-        # throwaway tree, and point WATERMARK_DIR at it so the real read/write
-        # helpers run against actual files.
-        self.tmpdir = tempfile.mkdtemp()
-        self.state = os.path.join(self.tmpdir, 'state')
-        patcher = patch.object(rules_beacon, 'WATERMARK_DIR', self.state)
-        patcher.start()
-        self.addCleanup(patcher.stop)
-
-    def tearDown(self):
-        shutil.rmtree(self.tmpdir, ignore_errors=True)
-
-    def _make_dir(self, name, files=None):
-        path = os.path.join(self.tmpdir, name)
-        os.makedirs(path, exist_ok=True)
-        for fname, content in (files or {}).items():
-            with open(os.path.join(path, fname), 'w') as f:
-                f.write(content)
-        return path
-
-    # -- trivial contract -------------------------------------------------
-
-    def test_virtual_returns_true(self):
-        self.assertTrue(rules_beacon.__virtual__())
-
-    def test_validate_returns_valid(self):
-        self.assertEqual(rules_beacon.validate({}), (True, 'valid'))
-
-    # -- _paths_from_config -----------------------------------------------
-
-    def test_paths_from_config_list_of_dicts(self):
-        config = [{'interval': 10}, {'paths': {'/a': 'suricata', '/b': 'strelka'}}]
-        self.assertEqual(
-            rules_beacon._paths_from_config(config),
-            {'/a': 'suricata', '/b': 'strelka'},
-        )
-
-    def test_paths_from_config_plain_dict(self):
-        self.assertEqual(
-            rules_beacon._paths_from_config({'paths': {'/a': 'suricata'}}),
-            {'/a': 'suricata'},
-        )
-
-    def test_paths_from_config_skips_non_dict_items(self):
-        self.assertEqual(rules_beacon._paths_from_config(['bogus', 42]), {})
-
-    def test_paths_from_config_paths_not_a_dict(self):
-        self.assertEqual(rules_beacon._paths_from_config({'paths': 'nope'}), {})
-
-    def test_paths_from_config_unexpected_type(self):
-        self.assertEqual(rules_beacon._paths_from_config('nonsense'), {})
-
-    # -- _excluded --------------------------------------------------------
-
-    def test_excluded_matches_temp_and_editor_files(self):
-        for pathname in ('/rules/foo.swp', '/rules/foo~', '/rules/4913', '/rules/.#foo'):
-            self.assertTrue(rules_beacon._excluded(pathname), pathname)
-
-    def test_excluded_allows_real_rule_files(self):
-        self.assertFalse(rules_beacon._excluded('/rules/suricata.rules'))
-
-    # -- _fingerprint -----------------------------------------------------
-
-    def test_fingerprint_missing_dir_is_empty_tree_digest(self):
-        missing = os.path.join(self.tmpdir, 'does-not-exist')
-        self.assertEqual(rules_beacon._fingerprint(missing), hashlib.sha1().hexdigest())
-
-    def test_fingerprint_changes_when_content_changes(self):
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        before = rules_beacon._fingerprint(d)
-        with open(os.path.join(d, 'a.rules'), 'w') as f:
-            f.write('alert tcp any any -> any any')  # different size
-        self.assertNotEqual(rules_beacon._fingerprint(d), before)
-
-    def test_fingerprint_ignores_excluded_files(self):
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        before = rules_beacon._fingerprint(d)
-        with open(os.path.join(d, 'a.rules.swp'), 'w') as f:
-            f.write('editor swap')
-        self.assertEqual(rules_beacon._fingerprint(d), before)
-
-    def test_fingerprint_skips_unstatable_entries(self):
-        # A dangling symlink appears in os.walk's file list but os.stat raises
-        # OSError, exercising the except-continue path.
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        good = rules_beacon._fingerprint(d)
-        os.symlink(os.path.join(d, 'missing-target'), os.path.join(d, 'broken.link'))
-        self.assertEqual(rules_beacon._fingerprint(d), good)
-
-    # -- _read_watermark / _write_watermark -------------------------------
-
-    def test_watermark_round_trip(self):
-        rules_beacon._write_watermark('suricata', 'deadbeef')
-        self.assertEqual(rules_beacon._read_watermark('suricata'), 'deadbeef')
-
-    def test_read_watermark_missing_returns_none(self):
-        self.assertIsNone(rules_beacon._read_watermark('suricata'))
-
-    def test_read_watermark_empty_file_returns_none(self):
-        os.makedirs(self.state, exist_ok=True)
-        with open(rules_beacon._watermark_file('suricata'), 'w') as f:
-            f.write('')
-        self.assertIsNone(rules_beacon._read_watermark('suricata'))
-
-    def test_write_watermark_swallows_oserror(self):
-        with patch.object(rules_beacon.os, 'makedirs', side_effect=OSError):
-            rules_beacon._write_watermark('suricata', 'deadbeef')
-        self.assertIsNone(rules_beacon._read_watermark('suricata'))
-
-    # -- beacon -----------------------------------------------------------
-
-    def _config(self, mapping):
-        return [{'paths': mapping}]
-
-    def test_beacon_seeds_first_run_and_emits_nothing(self):
-        with patch.object(rules_beacon, '_fingerprint', return_value='hash1'), \
-             patch.object(rules_beacon, '_read_watermark', return_value=None), \
-             patch.object(rules_beacon, '_write_watermark') as mock_write:
-            result = rules_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
-        self.assertEqual(result, [])
-        mock_write.assert_called_once_with('suricata', 'hash1')
-
-    def test_beacon_emits_on_change(self):
-        with patch.object(rules_beacon, '_fingerprint', return_value='newhash'), \
-             patch.object(rules_beacon, '_read_watermark', return_value='oldhash'), \
-             patch.object(rules_beacon, '_write_watermark') as mock_write:
-            result = rules_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
-        self.assertEqual(result, [{'tag': 'suricata', 'path': '/rules/suricata'}])
-        mock_write.assert_called_once_with('suricata', 'newhash')
-
-    def test_beacon_no_change_emits_nothing(self):
-        with patch.object(rules_beacon, '_fingerprint', return_value='samehash'), \
-             patch.object(rules_beacon, '_read_watermark', return_value='samehash'), \
-             patch.object(rules_beacon, '_write_watermark') as mock_write:
-            result = rules_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
-        self.assertEqual(result, [])
-        mock_write.assert_not_called()
-
-    def test_beacon_end_to_end_with_real_files(self):
-        # Exercise the full stack (real fingerprint + real watermark files) across
-        # two poll passes: first seeds silently, second fires after a write.
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        config = self._config({d: 'suricata'})
-
-        self.assertEqual(rules_beacon.beacon(config), [])  # seed pass
-        self.assertEqual(rules_beacon.beacon(config), [])  # unchanged pass
-
-        with open(os.path.join(d, 'b.rules'), 'w') as f:
-            f.write('alert tcp any any -> any any')
-        self.assertEqual(rules_beacon.beacon(config), [{'tag': 'suricata', 'path': d}])
-
-
-if __name__ == '__main__':
-    unittest.main()

salt/ca/files/blocklist/emsign-ecc-root-ca-c3.pem

@@ -0,0 +1,14 @@
+-----BEGIN CERTIFICATE-----
+MIICKzCCAbGgAwIBAgIKe3G2gla4EnycqDAKBggqhkjOPQQDAzBaMQswCQYDVQQG
+EwJVUzETMBEGA1UECxMKZW1TaWduIFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJbmMx
+IDAeBgNVBAMTF2VtU2lnbiBFQ0MgUm9vdCBDQSAtIEMzMB4XDTE4MDIxODE4MzAw
+MFoXDTQzMDIxODE4MzAwMFowWjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln
+biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMSAwHgYDVQQDExdlbVNpZ24gRUND
+IFJvb3QgQ0EgLSBDMzB2MBAGByqGSM49AgEGBSuBBAAiA2IABP2lYa57JhAd6bci
+MK4G9IGzsUJxlTm801Ljr6/58pc1kjZGDoeVjbk5Wum739D+yAdBPLtVb4Ojavti
+sIGJAnB9SMVK4+kiVCJNk7tCDK93nCOmfddhEc5lx/h//vXyqaNCMEAwHQYDVR0O
+BBYEFPtaSNCAIEDyqOkAB2kZd6fmw/TPMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB
+Af8EBTADAQH/MAoGCCqGSM49BAMDA2gAMGUCMQC02C8Cif22TGK6Q04ThHK1rt0c
+3ta13FaPWEBaLd4gTCKDypOofu4SQMfWh0/434UCMBwUZOR8loMRnLDRWmFLpg9J
+0wD8ofzkpf9/rdcw0Md3f76BB1UwUCAU9Vc4CqgxUQ==
+-----END CERTIFICATE-----

salt/ca/files/blocklist/emsign-ecc-root-ca-g3.pem

@@ -0,0 +1,15 @@
+-----BEGIN CERTIFICATE-----
+MIICTjCCAdOgAwIBAgIKPPYHqWhwDtqLhDAKBggqhkjOPQQDAzBrMQswCQYDVQQG
+EwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNo
+bm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0g
+RzMwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBrMQswCQYDVQQGEwJJ
+TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s
+b2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0gRzMw
+djAQBgcqhkjOPQIBBgUrgQQAIgNiAAQjpQy4LRL1KPOxst3iAhKAnjlfSU2fySU0
+WXTsuwYc58Byr+iuL+FBVIcUqEqy6HyC5ltqtdyzdc6LBtCGI79G1Y4PPwT01xyS
+fvalY8L1X44uT6EYGQIrMgqCZH0Wk9GjQjBAMB0GA1UdDgQWBBR8XQKEE9TMipuB
+zhccLikenEhjQjAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggq
+hkjOPQQDAwNpADBmAjEAvvNhzwIQHWSVB7gYboiFBS+DCBeQyh+KTOgNG3qxrdWB
+CUfvO6wIBHxcmbHtRwfSAjEAnbpV/KlK6O3t5nYBQnvI+GDZjVGLVTv7jHvrZQnD
++JbNR6iC8hZVdyR+EhCVBCyj
+-----END CERTIFICATE-----

salt/ca/files/blocklist/emsign-root-ca-c1.pem

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDczCCAlugAwIBAgILAK7PALrEzzL4Q7IwDQYJKoZIhvcNAQELBQAwVjELMAkG
+A1UEBhMCVVMxEzARBgNVBAsTCmVtU2lnbiBQS0kxFDASBgNVBAoTC2VNdWRocmEg
+SW5jMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEMxMB4XDTE4MDIxODE4MzAw
+MFoXDTQzMDIxODE4MzAwMFowVjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln
+biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMRwwGgYDVQQDExNlbVNpZ24gUm9v
+dCBDQSAtIEMxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAz+upufGZ
+BczYKCFK83M0UYRWEPWgTywS4/oTmifQz/l5GnRfHXk5/Fv4cI7gklL35CX5VIPZ
+HdPIWoU/Xse2B+4+wM6ar6xWQio5JXDWv7V7Nq2s9nPczdcdioOl+yuQFTdrHCZH
+3DspVpNqs8FqOp099cGXOFgFixwR4+S0uF2FHYP+eF8LRWgYSKVGczQ7/g/IdrvH
+GPMF0Ybzhe3nudkyrVWIzqa2kbBPrH4VI5b2P/AgNBbeCsbEBEV5f6f9vtKppa+c
+xSMq9zwhbL2vj07FOrLzNBL834AaSaTUqZX3noleoomslMuoaJuvimUnzYnu3Yy1
+aylwQ6BpC+S5DwIDAQABo0IwQDAdBgNVHQ4EFgQU/qHgcB4qAzlSWkK+XJGFehiq
+TbUwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL
+BQADggEBAMJKVvoVIXsoounlHfv4LcQ5lkFMOycsxGwYFYDGrK9HWS8mC+M2sO87
+/kOXSTKZEhVb3xEp/6tT+LvBeA+snFOvV71ojD1pM/CjoCNjO2RnIkSt1XHLVip4
+kqNPEjE2NuLe/gDEo2APJ62gsIq1NnpSob0n9CAnYuhNlCQT5AoE6TyrLshDCUrG
+YQTlSTR+08TI9Q/Aqum6VF7zYytPT1DU/rl7mYw9wC68AivTxEDkigcxHpvOJpkT
++xHqmiIMERnHXhuBUDDIlhJu58tBf5E7oke3VIAb3ADMmpDqw8NQBmIMMMAVSKeo
+WXzhriKi4gp6D/piq1JM4fHfyr6DDUI=
+-----END CERTIFICATE-----

salt/ca/files/blocklist/emsign-root-ca-c2.pem

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFcjCCA1qgAwIBAgIKLwq3aw3LSq8nWDANBgkqhkiG9w0BAQwFADBWMQswCQYD
+VQQGEwJVUzETMBEGA1UECxMKZW1TaWduIFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJ
+bmMxHDAaBgNVBAMTE2VtU2lnbiBSb290IENBIC0gQzIwHhcNMTgwMjE4MTgzMDAw
+WhcNNDMwMjE4MTgzMDAwWjBWMQswCQYDVQQGEwJVUzETMBEGA1UECxMKZW1TaWdu
+IFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJbmMxHDAaBgNVBAMTE2VtU2lnbiBSb290
+IENBIC0gQzIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCMfX1lA+Tb
+mh9YInmRgOW97IVx4LUJf2DRZfs837Jrml+py64aVnYgWO4t6C78fgjfS7jX+c4T
+inIzEquWcI+zi0fd4Sc8NDf7JONp27VWX0qwUYqzLDRCt+s7zpLcfx1ky0zVIJj6
+L06uPyK3kIr9+YAsrVj+39utm6e2MBQsRNstSI3fCQYAGvoQTQ8fULauTqNWaYAk
+NYFe6HUHHQPp2u1Ua00odMXiD5oRFxLcDnGAcE1I/9E9mLCdkggXijYUmico7+Xw
+ZeFoPhva6eIJ5p03Lt3Du5W3EcHR0cJmmY1pyeA36JaXKWRNM9IRjYMVNCcp4jhB
+2tIYiZ+LVk8bwQ9/1c23txmv3u97taZlV22NF4ttS1qq3J+MOp0oGULBzpKfRx0q
+GVqbPukQNGAjOLIN8KDNQNzbR1iAl2d8H+MSoicBo4Aid8TjLWcNv48oCWL53ZrF
+BMTDjaIA6frG1t4IpbnHadA7qCJJe2qpJN6n2eQKAUn6UiQDHPsSqNBlcUhQ4Y/0
+Y0mU5rghm2OB9rXQS1Fb1JRCfJMNnJIm5AUB2+2RWzq5Tgz7SbSho8NsZk0UbQnF
+xciqQ9uoVTAsK14Sk9oG8Q3zfsM08cdPoRb0WlIZklR6mKD7L8nH/zfGu8PIJv94
+GGB9RZ9U4A69r3ePmy8MvrzfNxHKtH6svwIDAQABo0IwQDAdBgNVHQ4EFgQUs/eK
+pNYPiABZ6FEXT9V+7IYigZ0wDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMB
+Af8wDQYJKoZIhvcNAQEMBQADggIBADQlpiWM0cv2nZ0H5jVsBq0x2q62Q0LwqATs
+CFvyub7gxNCytRuoA8stmPOEu/lg8Igxj4FIjoyhIrWUVxyiLU7No4P+WjEUOwUT
+xIpkEOtvGUQ9fiOlcGHtIZDNBlZq7WpktXAxeV55RPPsor26p2FNAMRFfZQh0sLX
+hKgk8iulSSggqx8ezgPye63FaiYEi4c/dzRj3HOCnsZiwZZU02df5YpNFjxSwZvE
+41cjGpsrpWMfQFI2s53RbeXp47lSAxYE4NzjBFMe+EwFuEveBCJBEAH5rvYu3pi2
+orsJ424TqWEQV1tCsCkQz+Yq/Okal7yHAkKDeOXcP7oN4A+TdXc2pdqxuVCnBO0R
+mWz2JpGSSeJjiTk/OPwRsPNWtwG/KXL04o2ta3jiPpJuICVtWDAc9R3auBEgJl5r
+ShRmBdszG0LmzsHuZPCFSYC15RBDCOBsa8bDRJ8pBFU2Wi/CVXCACEuavgoveA4F
+a5bt38o0PWxsBP+MpocCdVtDMqzQhxy9IohKuXWAGresoIvKDg3xFk6rBOrjfVwJ
+elwi/xAisojHPJVQv9W1zVIoHp+EQg/4MQC21NbIX2RoioB+V3hK439b/w7deU8x
+2M8cl1OG0nPfbnARl5GPM7vJgi470jto4SeMg6HMAW3Egb56tQcNLwI9U8mZnNvR
+gUMrkAgL
+-----END CERTIFICATE-----

salt/ca/files/blocklist/emsign-root-ca-g1.pem

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDlDCCAnygAwIBAgIKMfXkYgxsWO3W2DANBgkqhkiG9w0BAQsFADBnMQswCQYD
+VQQGEwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBU
+ZWNobm9sb2dpZXMgTGltaXRlZDEcMBoGA1UEAxMTZW1TaWduIFJvb3QgQ0EgLSBH
+MTAeFw0xODAyMTgxODMwMDBaFw00MzAyMTgxODMwMDBaMGcxCzAJBgNVBAYTAklO
+MRMwEQYDVQQLEwplbVNpZ24gUEtJMSUwIwYDVQQKExxlTXVkaHJhIFRlY2hub2xv
+Z2llcyBMaW1pdGVkMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEcxMIIBIjAN
+BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAk0u76WaK7p1b1TST0Bsew+eeuGQz
+f2N4aLTNLnF115sgxk0pvLZoYIr3IZpWNVrzdr3YzZr/k1ZLpVkGoZM0Kd0WNHVO
+8oG0x5ZOrRkVUkr+PHB1cM2vK6sVmjM8qrOLqs1D/fXqcP/tzxE7lM5OMhbTI0Aq
+d7OvPAEsbO2ZLIvZTmmYsvePQbAyeGHWDV/D+qJAkh1cF+ZwPjXnorfCYuKrpDhM
+tTk1b+oDafo6VGiFbdbyL0NVHpENDtjVaqSW0RM8LHhQ6DqS0hdW5TUaQBw+jSzt
+Od9C4INBdN+jzcKGYEho42kLVACL5HZpIQ15TjQIXhTCzLG3rdd8cIrHhQIDAQAB
+o0IwQDAdBgNVHQ4EFgQU++8Nhp6w492pufEhF38+/PB3KxowDgYDVR0PAQH/BAQD
+AgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAFn/8oz1h31x
+PaOfG1vR2vjTnGs2vZupYeveFix0PZ7mddrXuqe8QhfnPZHr5X3dPpzxz5KsbEjM
+wiI/aTvFthUvozXGaCocV685743QNcMYDHsAVhzNixl03r4PEuDQqqE/AjSxcM6d
+GNYIAwlG7mDgfrbESQRRfXBgvKqy/3lyeqYdPV8q+Mri/Tm3R7nrft8EI6/6nAYH
+6ftjk4BAtcZsCjEozgyfz7MjNYBBjWzEN3uBL4ChQEKF6dk4jeihU80Bv2noWgby
+RQuQ+q7hv53yrlc8pa6yVvSLZUDp/TGBLPQ5Cdjua6e0ph0VpZj3AYHYhX3zUVxx
+iN66zB+Afko=
+-----END CERTIFICATE-----

salt/ca/files/blocklist/emsign-root-ca-g2.pem

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFlTCCA32gAwIBAgILAIZNvw/jXtd9jtgwDQYJKoZIhvcNAQEMBQAwZzELMAkG
+A1UEBhMCSU4xEzARBgNVBAsTCmVtU2lnbiBQS0kxJTAjBgNVBAoTHGVNdWRocmEg
+VGVjaG5vbG9naWVzIExpbWl0ZWQxHDAaBgNVBAMTE2VtU2lnbiBSb290IENBIC0g
+RzIwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBnMQswCQYDVQQGEwJJ
+TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s
+b2dpZXMgTGltaXRlZDEcMBoGA1UEAxMTZW1TaWduIFJvb3QgQ0EgLSBHMjCCAiIw
+DQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMNwGIWW2kHfHK+sXTNwxF07K+IV
+ySTuyFM2r1v002wUfcdT+zs5OM5QbMYFFnedXQI6gCFLsjKrcaej48Zt37OyEb3i
+aPs7CsP4kAyTwzKH9aZe6gXYHrJq40/ZVMNcQVI2PcIp40B/SAN2gUZ+ZaUtIOvV
+jEx26/ebNaXRIsthlkOG/caB+QRwDw1tl7338Zlv0M2oTBUy4B3e7dGP5pgXH71M
+jqHPCoNo+xv9f0NTBT+hUDa8h8wUtcGQq9CDeJTpjWcD2bP2AMdVG6oVpMAUeUzo
+cCyglvtFdUMjggxBbw4qhau1HXPG8Ot9hwL7ZMi8tkTzrvUIxxb8G9LF/7kKeCE7
+tGZaVzDTnXuifl3msR4ErHsQ4P7lVu2AIjIAhrAXoedDidb7pMcf7TABdrYUT1Jo
+G/AiK+J9jO6GTjeADD4LMDSBZhHMuBK/PJ/g0kGBt+/C1L+/HURzQhJkMlRnM6Rv
+XoCtfKopSlns5trZmTi971Wjbn88QXP61lGpBCUPwCjs7rpOYvSUJtI+lcbF+37q
+kIqOXYkVT3cupDSpw+H89kFtj5GKY+Xny4LxY+3IvDIRiyd6ky1DPj713DI0yqve
+EpsIr3A0PdwuyUI7CS1jg0NnGFT6Xxyr0xB+VDt83FJYW8v16k2pbaQ4kVxA3aXd
+X9dZYyVR1S59KM75AgMBAAGjQjBAMB0GA1UdDgQWBBTt7E1FYRgo57MjKBEcTaUn
+DV7s9DAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B
+AQwFAAOCAgEACFC/ilQg8KTCVBxFJW/sazomkS0kNYbEIZg4B3obqwsJ7SX98z8Z
+gfzBpz0nYClwwJjWbFN1R2zY8pCEot6/dgmA8Vbq0GxhwPM5YN/SZquNyRIxO3cU
+dlAcwf+vSezdVCf9wOzvSAF3q0a5ljvbdbNJNpfScQVp7UUd5sBsZk8jXO1KQ/go
+/Vf/GDPnrIFmxpAIGE3sgnO8lAv9FzUaAeuv7HWe47xN9J7+bQzF93yHuIXACPTL
+pQHhg2zMv5C7BAbuDHfbj1Cu294Z832yhSfBcziWGskOvl3es2EcHytbS9c9P+0z
+Mpka7zGC1FHrvLb/FoduH86TeZt0QjZ6pcplNzoaxDnDvzTJ6CC2Eny+qH/APFCu
+VUv5/wjwF+HPm8Pup2ARj9cEp92+0qcerfHacNq5hMeGZdbA/dzdUR/5z5zXdxAk
+nl8mcfGb0eMNSTXQmmB/i4AecNnr72uYjzlaXUGYN7Nrb6XouG0pnh0/BBtWWp0U
+ShIPpWEAqs7RJBj6+1ZUYXZ4ObrCw962DxhN2p19Hxw9LtuUUcLqqTPrFXYvwO4t
+ouj7KJnAkaTUfXGdEaFVtFig1EA30WzJY2X1vAQ7hVnniCjgaXAGqjsU6sklNM9n
+xDx5rFCCCEtj9Kh8UHjGK2QqgP5kwgttjOApQMaCoezMfK4KD7WpOXU=
+-----END CERTIFICATE-----

salt/ca/files/blocklist/emsign-root-cs-ca-g2.pem

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFdjCCA16gAwIBAgIQCIYhdXKnxAzDuy7l1y1ubDANBgkqhkiG9w0BAQwFADBV
+MQswCQYDVQQGEwJJTjElMCMGA1UECgwcZU11ZGhyYSBUZWNobm9sb2dpZXMgTGlt
+aXRlZDEfMB0GA1UEAwwWZW1TaWduIFJvb3QgQ1MgQ0EgLSBHMjAeFw0yNDAyMDgw
+ODE1MzlaFw00OTAyMDcwODE1MzlaMFUxCzAJBgNVBAYTAklOMSUwIwYDVQQKDBxl
+TXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVkMR8wHQYDVQQDDBZlbVNpZ24gUm9v
+dCBDUyBDQSAtIEcyMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA4CVO
+xuBjdLJXpkrjosA+7ygTGaYzufUJrhQOeIZEuCqeCPI8hDqIu+MIqEqOlgDFetVM
+3Cor3CGBlGJzfzzlI31vyElvQq88jjCfo12Ghqcf+NAZDFduJk5+NgC0z3lfdiB6
+NNzxbfVkA+X6oDNNRioVUZsVFuIzT9Qt0Sh4KwQPRFBc/wI2yGpDuKHNsLehcECx
+Uw00m33xGYh9UkeL3reOmAaLYWZiO+Pd2vxPtrkXI4wyN4BMlESJ73uP0tyX9lq4
+M4yN2yh3i+ETI8/jzPjxHD7QFaVkCoS+uj2YmBa3mytlqlBgyOoKueNeotX8hfzB
+FNm962abpV4mN0SoC6hjkYv8IayseRPFv9VI0meQXg9OLRoNvBOqG5kw937FH0Li
+q3CIzKyvdyeYRT0LJdWA2iuB9l+4zzIfCwGyT4M4VDmJuPBhrTmjXKae6AUtm1eO
+q2f8gftA6J6AlbD0gRd/t8CrFssFNm+LYAEd36kWRBuolbaOfbSbEww84xNAgj4U
+SPwYjtNDsCkQjtgSOaDtDOmolN+Mbk528Y+LHMlbd4T3Xmp+VIWp5+70GhBDOA91
+hhtims+hHkXjRI2U7WrJLcTlcF8ykiymRbFKhwOBvTmtZNbHmDI7scU4wWUAsmW6
+tWaeeoM50KBl8ai28A6yBGYMO//03Ubvqiqst80CAwEAAaNCMEAwHQYDVR0OBBYE
+FOYcxqontIvyvYVkX6/0unKm0AF1MA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8E
+BTADAQH/MA0GCSqGSIb3DQEBDAUAA4ICAQCv5Asord7fdQMT30emwb+oLuDUxdmJ
+DQHnzxIXoNvymDTbNHF9LqjeXc+X+XcjIV8PTp4o2I9wyVppHmcaVgRjN/G9FxeN
+5tCQ97v5sdx4eAdPljK1xqg7Vhfmzcp+7rSjJG7Sn9gMlZx81kL3Oa2qZj+W5Fi6
+tN4vaR9OLEyvKKHDG/eFxS/qghD1d/Clkb7UGGjBcWJlaZTWVALMKG+QCxiMoELi
+oeKaoBbZoXooxfF0mWdkIl9RWNU4r9O/yt66LBE9rj49dX19L+rGg71/4bTi5ua+
++Wz60XaOc2CdHJaBSols5hWEx/LBjS7UzfU3uJfnpQXVDi0US5sFzIHIlS6kjh7h
+YuF4+/9ZAl4Bx3w2TNN7pVm7I39pgYl7XN6KZRwfkAv4aRswPqhWbQfr3+qfcC93
+cgTyFy2RG5YE/OfmpSs6VtYKawqsdEODz/p2nhZfSDrVDnEPvERqahttY5+gaWNc
+2fbibgKMfuejTZaJ3yttPslVSko5SDs0xKU423swu34sb/fdISqtOu4NzPCZiHSG
+ladaoITmSA6UWFk4fdqaLBI+xSxY5Et3S1APp5r3j/l1KrWPNbb1/+ydHN+YLbh8
+hCBIecKdk6VbnnFRlulm0d2RSgpfCRrWLomN8JHCxCMVv/hBQXz3AXGM1ZlfYTyl
+E1BpDby0G2eEkw==
+-----END CERTIFICATE-----

salt/ca/init.sls

@@ -10,3 +10,4 @@ include:
   - ca.server
 {% endif %}
   - ca.trustca
+  - ca.trust

salt/ca/trust.map.jinja

@@ -0,0 +1,12 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{# Containers mount the extracted directory rather than single files: update-ca-trust replaces
+   files by rename, so a file bind mount would keep pointing at the old inode. #}
+{% set CATRUST = {
+    'dir': '/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro',
+    'java': '/etc/pki/ca-trust/extracted/java/cacerts',
+    'pem': '/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem'
+} %}

salt/ca/trust.sls

@@ -0,0 +1,37 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+ca_trust_anchor:
+  file.managed:
+    - name: /etc/pki/ca-trust/source/anchors/so-intca.crt
+    - source: salt://ca/files/ca.crt
+    - show_changes: False
+    - makedirs: True
+
+ca_trust_legacy_anchor:
+  file.absent:
+    - name: /etc/pki/ca-trust/source/anchors/ca.crt
+
+ca_trust_legacy_bundles:
+  file.absent:
+    - name: /opt/so/conf/ca
+
+# clean removes blocklist entries that are no longer shipped
+ca_trust_blocklist:
+  file.recurse:
+    - name: /etc/pki/ca-trust/source/blocklist
+    - source: salt://ca/files/blocklist
+    - clean: True
+    - makedirs: True
+    - file_mode: '0644'
+    - dir_mode: '0755'
+
+ca_trust_extract:
+  cmd.run:
+    - name: update-ca-trust extract
+    - onchanges:
+      - file: ca_trust_anchor
+      - file: ca_trust_legacy_anchor
+      - file: ca_trust_blocklist

salt/common/init.sls

@@ -117,14 +117,25 @@ elastic_curl_config:
 {% endif %}
 
 
+# A non-root owner here can chmod the directory and replace any script in it, including
+# the root-owned ones. 555 is the mode the filesystem RPM ships; root ignores it anyway.
+usr_sbin_perms:
+  file.directory:
+    - name: /usr/sbin
+    - user: root
+    - group: root
+    - mode: 555
+
 common_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://common/tools/sbin
-    - user: 939
-    - group: 939
+    - user: root
+    - group: root
     - file_mode: 755
     - show_changes: False
+    - require:
+      - file: usr_sbin_perms
 {% if GLOBALS.role == 'so-heavynode' %}
     - exclude_pat:
       - so-pcap-import
@@ -159,8 +170,8 @@ common_sbin_jinja:
   file.recurse:
     - name: /usr/sbin
     - source: salt://common/tools/sbin_jinja
-    - user: 939
-    - group: 939 
+    - user: root
+    - group: root
     - file_mode: 755
     - template: jinja
     - show_changes: False
@@ -173,6 +184,8 @@ so-status_script:
   file.managed:
     - name: /usr/sbin/so-status
     - source: salt://common/tools/sbin/so-status
+    - user: root
+    - group: root
     - mode: 755
 
 {% if GLOBALS.is_sensor %}
@@ -204,9 +217,11 @@ sostatus_log:
     - replace: False
 
 # Install sostatus check cron. This is used to populate Grid.
+# telegraf reads status.log on the same minute boundary this runs, so write aside and rename
+# rather than truncating the file it is reading
 so-status_check_cron:
   cron.present:
-    - name: '/usr/sbin/so-status -j > /opt/so/log/sostatus/status.log 2>&1'
+    - name: '/usr/sbin/so-status -j > /opt/so/log/sostatus/status.log.tmp 2>&1; mv -f /opt/so/log/sostatus/status.log.tmp /opt/so/log/sostatus/status.log'
     - identifier: so-status_check_cron
     - user: root
     - minute: '*/1'

salt/common/soup_scripts.sls

@@ -18,47 +18,61 @@ copy_so-common_common_tools_sbin:
     - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-common
     - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-common
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_so-image-common_common_tools_sbin:
   file.copy:
     - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-image-common
     - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-image-common
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_soup_manager_tools_sbin:
   file.copy:
     - name: /opt/so/saltstack/default/salt/manager/tools/sbin/soup
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/soup
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_so-firewall_manager_tools_sbin:
   file.copy:
     - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-firewall
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_so-yaml_manager_tools_sbin:
   file.copy:
     - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-yaml.py
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-yaml.py
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_so-repo-sync_manager_tools_sbin:
   file.copy:
     - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-repo-sync
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_bootstrap-salt_manager_tools_sbin:
   file.copy:
     - name: /opt/so/saltstack/default/salt/salt/scripts/bootstrap-salt.sh
     - source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 644
 
 # This section is used to put the new script in place so that it can be called during soup.
 # It is faster than calling the states that normally manage them to put them in place.
@@ -67,46 +81,60 @@ copy_so-common_sbin:
     - name: /usr/sbin/so-common
     - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-common
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_so-image-common_sbin:
   file.copy:
     - name: /usr/sbin/so-image-common
     - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-image-common
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_soup_sbin:
   file.copy:
     - name: /usr/sbin/soup
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/soup
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_so-firewall_sbin:
   file.copy:
     - name: /usr/sbin/so-firewall
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_so-yaml_sbin:
   file.copy:
     - name: /usr/sbin/so-yaml.py
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-yaml.py
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_so-repo-sync_sbin:
   file.copy:
     - name: /usr/sbin/so-repo-sync
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_bootstrap-salt_sbin:
   file.copy:
     - name: /usr/sbin/bootstrap-salt.sh
     - source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755

salt/common/tools/sbin/so-common

@@ -240,7 +240,8 @@ copy_new_files() {
   cd $UPDATE_DIR
   rsync -a salt $DEFAULT_SALT_DIR/ --delete "${EXCLUDE_ARGS[@]}"
   rsync -a pillar $DEFAULT_SALT_DIR/ --delete "${EXCLUDE_ARGS[@]}"
-  chown -R socore:socore $DEFAULT_SALT_DIR/
+  # Root-executed code; SOC only needs to read it. Local dirs stay socore-owned.
+  chown -R root:root $DEFAULT_SALT_DIR/
   cd /tmp
 }
 
@@ -418,6 +419,19 @@ is_single_node_grid() {
 	grep "role: so-" /etc/salt/grains | grep -E "eval|standalone|import" &> /dev/null
 }
 
+remove_elasticsearch_index_template() {
+  local template_name=$1
+  local reason=${2:-"Removing index template for upgrade"}
+
+  # check if template exists
+  if so-elasticsearch-query "_index_template/$template_name" --fail --retry 3 --retry-delay 5 >/dev/null 2>&1; then
+    echo "Removing Elasticsearch index template: $template_name ($reason)"
+    if ! so-elasticsearch-query "_index_template/$template_name" -XDELETE --fail --retry 3 --retry-delay 5; then
+      return 1
+    fi
+  fi
+}
+
 initialize_elasticsearch_indices() {
     local index_names=$1
     local default_entry=${2:-'{"@timestamp":"0"}'}

salt/common/tools/sbin/so-common-status-check

@@ -9,6 +9,7 @@ import sys
 import subprocess
 import os
 import json
+import tempfile
 
 sys.path.append('/opt/saltstack/salt/lib/python3.10/site-packages/')
 import salt.config
@@ -17,6 +18,21 @@ import salt.loader
 __opts__ = salt.config.minion_config('/etc/salt/minion')
 __grains__ = salt.loader.grains(__opts__)
 
+def write_atomic(path, value):
+  # telegraf reads these files on its own schedule; replacing them by rename means it never
+  # reads a truncated file and reports an empty value as if it were real
+  directory = os.path.dirname(path)
+  handle, temp = tempfile.mkstemp(dir=directory)
+  try:
+    with os.fdopen(handle, 'w') as f:
+      f.write(str(value))
+    os.chmod(temp, 0o644)
+    os.replace(temp, path)
+  except Exception:
+    os.path.exists(temp) and os.unlink(temp)
+    raise
+
+
 def check_needs_restarted():
   osfam = __grains__['os_family']
   val = '0'
@@ -34,8 +50,7 @@ def check_needs_restarted():
   else:
     fail("Unsupported OS")
 
-  with open(outfile, 'w') as f:
-    f.write(val)
+  write_atomic(outfile, val)
 
 def check_for_fps():
     feat = 'fps'
@@ -56,8 +71,7 @@ def check_for_fps():
           # Unknown, so assume 0
           fps = 0
 
-    with open('/opt/so/log/sostatus/fps_enabled', 'w') as f:
-       f.write(str(fps))
+    write_atomic('/opt/so/log/sostatus/fps_enabled', fps)
 
 def check_for_lks():
     feat = 'Lks'
@@ -80,8 +94,7 @@ def check_for_lks():
                                 lks = 1
         if lks:
             break
-    with open('/opt/so/log/sostatus/lks_enabled', 'w') as f:
-       f.write(str(lks))
+    write_atomic('/opt/so/log/sostatus/lks_enabled', lks)
 
 def fail(msg):
   print(msg, file=sys.stderr)

salt/common/tools/sbin/so-log-check

@@ -134,6 +134,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Redis may have been restarted" # Redis likely restarted by salt
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|file already closed"          # Go logging race condition during container restart
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|relation \"audit_settings\" does not exist" # salt checking for changes before SOC starts
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error in plugin: elasticsearch: Unable to retrieve master node information" # expected error while ES is upgrading/electing a master 
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
@@ -176,6 +177,8 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unexpected authorization header" # expected WARN log lines indicating invalid auth header
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Missing ory_kratos_session cookie" # expected WARN log lines indicating invalid auth header
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Static assets preprocessor only supports GET and HEAD requests" # expected WARN log lines indicating invalid auth header
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|respondError"                 # respondError is a function name, output via http middleware as standard request logging
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|GET /kibana/"                 # Ignore Kibana queries with triggered words
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
@@ -239,7 +242,8 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal"           # docker container getting recycled
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown"             # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown"             # so-dockerregistry logs a tag lookup miss during image copy; not tied to one docker version
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Could not index event to Elasticsearch.*\"version\" => \"9.0.8\"" # Expected during Elastic upgrade temporarily, as policies referencing older pipelines are updated
 fi
 
 RESULT=0
@@ -304,4 +308,4 @@ else
     echo -e "\nResult: One or more errors found"
 fi
 
-exit $RESULT
+exit $RESULT

salt/common/tools/sbin/so-sensor-clean

@@ -8,21 +8,37 @@
 # Elastic License 2.0.
 
 
-SENSOR_DIR='/nsm'
+SENSOR_DIR="${SENSOR_DIR:-/nsm}"
 CRIT_DISK_USAGE=90
-CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
-LOG="/opt/so/log/sensor_clean.log"
-TODAY=$(date -u "+%Y-%m-%d")
+LOG="${LOG:-/opt/so/log/sensor_clean.log}"
+LOCK="${LOCK:-/var/tmp/so-sensor-clean.lock}"
+MAX_PASSES=100
 
+ZEEK_LOGS="$SENSOR_DIR/zeek/logs"
+STRELKA_FILES="$SENSOR_DIR/strelka/processed"
+SURICATA_LOGS="$SENSOR_DIR/suricata"
+PCAPS="$SENSOR_DIR/pcapout"
+
+log() {
+	echo "$(date) - $*" >>"$LOG"
+}
+
+disk_usage() {
+	df -P "$SENSOR_DIR" | tail -1 | awk '{print $5}' | tr -d %
+}
+
+disk_avail() {
+	df -P "$SENSOR_DIR" | tail -1 | awk '{print $4}'
+}
+
+# sets REMOVED=1 if anything was actually deleted
 clean() {
 	## find the oldest Zeek logs directory
-	OLDEST_DIR=$(ls /nsm/zeek/logs/ | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
-	if [ -z "$OLDEST_DIR" -o "$OLDEST_DIR" == ".." -o "$OLDEST_DIR" == "." ]; then
-		echo "$(date) - No old Zeek logs available to clean up in /nsm/zeek/logs/" >>$LOG
-		#exit 0
-	else
-		echo "$(date) - Removing directory: /nsm/zeek/logs/$OLDEST_DIR" >>$LOG
-		rm -rf /nsm/zeek/logs/"$OLDEST_DIR"
+	OLDEST_DIR=$(ls "$ZEEK_LOGS" 2>/dev/null | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
+	if [ -n "$OLDEST_DIR" ]; then
+		log "Removing directory: $ZEEK_LOGS/$OLDEST_DIR"
+		rm -rf "$ZEEK_LOGS/$OLDEST_DIR"
+		REMOVED=1
 	fi
 
 	## Remarking for now, as we are moving extracted files to /nsm/strelka/processed
@@ -43,58 +59,73 @@ clean() {
 	#fi
 
 	## Clean up Zeek extracted files processed by Strelka
-	STRELKA_FILES='/nsm/strelka/processed'
-	OLDEST_STRELKA=$(find $STRELKA_FILES -type f -printf '%T+ %p\n' | sort -n | head -n 1)
-	if [ -z "$OLDEST_STRELKA" -o "$OLDEST_STRELKA" == ".." -o "$OLDEST_STRELKA" == "." ]; then
-		echo "$(date) - No old files available to clean up in $STRELKA_FILES" >>$LOG
-	else
+	OLDEST_STRELKA=$(find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
+	if [ -n "$OLDEST_STRELKA" ]; then
 		OLDEST_STRELKA_DATE=$(echo $OLDEST_STRELKA | awk '{print $1}' | cut -d+ -f1)
-		OLDEST_STRELKA_FILE=$(echo $OLDEST_STRELKA | awk '{print $2}')
-		echo "$(date) - Removing extracted files for $OLDEST_STRELKA_DATE" >>$LOG
-		find $STRELKA_FILES -type f -printf '%T+ %p\n' | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
-			echo "$(date) - Removing file: $FILE" >>$LOG
+		log "Removing extracted files for $OLDEST_STRELKA_DATE"
+		REMOVED=1
+		find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
+			log "Removing file: $FILE"
 			rm -f "$FILE"
 		done
 	fi
 
 	## Clean up Suricata log files
-	SURICATA_LOGS='/nsm/suricata'
-	OLDEST_SURICATA=$(find $SURICATA_LOGS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
-	if [[ -z "$OLDEST_SURICATA" ]] || [[ "$OLDEST_SURICATA" == ".." ]] || [[ "$OLDEST_SURICATA" == "." ]]; then
-		echo "$(date) - No old files available to clean up in $SURICATA_LOGS" >>$LOG
-	else
+	OLDEST_SURICATA=$(find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
+	if [ -n "$OLDEST_SURICATA" ]; then
 		OLDEST_SURICATA_DATE=$(echo $OLDEST_SURICATA | awk '{print $1}' | cut -d+ -f1)
-		OLDEST_SURICATA_FILE=$(echo $OLDEST_SURICATA | awk '{print $2}')
-		echo "$(date) - Removing logs for $OLDEST_SURICATA_DATE" >>$LOG
-		find $SURICATA_LOGS -type f -printf '%T+ %p\n' | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
-			echo "$(date) - Removing file: $FILE" >>$LOG
+		log "Removing logs for $OLDEST_SURICATA_DATE"
+		REMOVED=1
+		find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
+			log "Removing file: $FILE"
 			rm -f "$FILE"
 		done
 	fi
 
 	## Clean up extracted pcaps
-	PCAPS='/nsm/pcapout'
-	OLDEST_PCAP=$(find $PCAPS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
-	if [ -z "$OLDEST_PCAP" -o "$OLDEST_PCAP" == ".." -o "$OLDEST_PCAP" == "." ]; then
-		echo "$(date) - No old files available to clean up in $PCAPS" >>$LOG
-	else
+	OLDEST_PCAP=$(find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
+	if [ -n "$OLDEST_PCAP" ]; then
 		OLDEST_PCAP_DATE=$(echo $OLDEST_PCAP | awk '{print $1}' | cut -d+ -f1)
-		OLDEST_PCAP_FILE=$(echo $OLDEST_PCAP | awk '{print $2}')
-		echo "$(date) - Removing extracted files for $OLDEST_PCAP_DATE" >>$LOG
-		find $PCAPS -type f -printf '%T+ %p\n' | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
-			echo "$(date) - Removing file: $FILE" >>$LOG
+		log "Removing extracted files for $OLDEST_PCAP_DATE"
+		REMOVED=1
+		find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
+			log "Removing file: $FILE"
 			rm -f "$FILE"
 		done
 	fi
 }
 
-# Check to see if we are already running
-NUM_RUNNING=$(pgrep -cf "/bin/bash /usr/sbin/so-sensor-clean")
-[ "$NUM_RUNNING" -gt 1 ] && echo "$(date) - $NUM_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0
-
-if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then
-	while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
-		clean
-		CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
-	done
+# Only one instance at a time; the lock is the fd, so it releases on any exit
+exec 9>"$LOCK" || exit 1
+if ! flock -n 9; then
+	log "another so-sensor-clean is already running (lock $LOCK held); exiting"
+	exit 0
 fi
+
+CUR_USAGE=$(disk_usage)
+[ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ] || exit 0
+
+log "$SENSOR_DIR at ${CUR_USAGE}% (threshold ${CRIT_DISK_USAGE}%); starting cleanup"
+
+PASS=0
+while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
+	PASS=$((PASS + 1))
+	if [ "$PASS" -gt "$MAX_PASSES" ]; then
+		log "stopping after $MAX_PASSES passes; $SENSOR_DIR still at ${CUR_USAGE}%"
+		break
+	fi
+
+	REMOVED=0
+	BEFORE=$(disk_avail)
+	clean
+	CUR_USAGE=$(disk_usage)
+
+	if [ "$REMOVED" -eq 0 ]; then
+		log "nothing left to remove in $ZEEK_LOGS, $STRELKA_FILES, $SURICATA_LOGS, $PCAPS; $SENSOR_DIR still at ${CUR_USAGE}% - space is consumed outside of NSM cleanup scope"
+		break
+	fi
+	if [ "$(disk_avail)" -le "$BEFORE" ]; then
+		log "pass $PASS freed no space; $SENSOR_DIR still at ${CUR_USAGE}% - stopping until next run"
+		break
+	fi
+done

salt/common/tools/sbin_jinja/so-raid-status

@@ -125,4 +125,6 @@ else
   RAIDSTATUS=1
 fi
 
-echo "nsmraid=$RAIDSTATUS" > /opt/so/log/raid/status.log
+# telegraf reads this file; write aside and rename so it never sees a half-written file
+echo "nsmraid=$RAIDSTATUS" > /opt/so/log/raid/status.log.tmp
+mv -f /opt/so/log/raid/status.log.tmp /opt/so/log/raid/status.log

salt/desktop/trusted-ca.sls

@@ -1,18 +1,10 @@
- {% from 'vars/globals.map.jinja' import GLOBALS %}
+{% from 'vars/globals.map.jinja' import GLOBALS %}
 
 {# we only want this state to run it is CentOS #}
 {% if GLOBALS.os == 'OEL' %}
 
-trusted_ca:
-  file.managed:
-    - name: /etc/pki/ca-trust/source/anchors/ca.crt
-    - source: salt://ca/files/ca.crt
-
-update_ca_certs:
-  cmd.run:
-    - name: update-ca-trust
-    - onchanges:
-      - file: trusted_ca
+include:
+  - ca.trust
 
 {% else %}
 

salt/docker/defaults.yaml

@@ -1,6 +1,12 @@
 docker:
   range: '172.17.1.0/24'
   gateway: '172.17.1.1'
+  networks:
+    sobridge: {}
+    soauth:
+      range: '172.17.2.0/24'
+      gateway: '172.17.2.1'
+      manager_only: True
   ulimits:
     - name: nofile
       soft: 1048576
@@ -58,18 +64,18 @@ docker:
       ulimits: []
     'so-kratos':
       final_octet: 28
+      networks: ['soauth']
       port_bindings:
         - 0.0.0.0:4433:4433
-        - 0.0.0.0:4434:4434
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
       ulimits: []
     'so-hydra':
       final_octet: 30
+      networks: ['soauth']
       port_bindings:
         - 0.0.0.0:4444:4444
-        - 0.0.0.0:4445:4445
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
@@ -128,6 +134,7 @@ docker:
       ulimits: []
     'so-soc':
       final_octet: 34
+      networks: ['sobridge', 'soauth']
       port_bindings:
         - 0.0.0.0:9822:9822
       custom_bind_mounts: []

salt/docker/docker.map.jinja

@@ -1,8 +1,26 @@
 {% import_yaml 'docker/defaults.yaml' as DOCKERDEFAULTS %}
 {% set DOCKERMERGED = salt['pillar.get']('docker', DOCKERDEFAULTS.docker, merge=True) %}
-{% set RANGESPLIT = DOCKERMERGED.range.split('.') %}
-{% set FIRSTTHREE = RANGESPLIT[0] ~ '.' ~ RANGESPLIT[1] ~ '.' ~ RANGESPLIT[2] ~ '.' %}
+
+{% if DOCKERMERGED.networks.sobridge is not mapping %}
+{%   do DOCKERMERGED.networks.update({'sobridge': {}}) %}
+{% endif %}
+{% do DOCKERMERGED.networks['sobridge'].update({'range': DOCKERMERGED.range, 'gateway': DOCKERMERGED.gateway}) %}
+
+{% for netname, net in DOCKERMERGED.networks.items() %}
+{%   set RANGESPLIT = net.range.split('.') %}
+{%   do net.update({'prefix': RANGESPLIT[0] ~ '.' ~ RANGESPLIT[1] ~ '.' ~ RANGESPLIT[2] ~ '.'}) %}
+{% endfor %}
 
 {% for container, vals in DOCKERMERGED.containers.items() %}
-{%   do DOCKERMERGED.containers[container].update({'ip': FIRSTTHREE ~ DOCKERMERGED.containers[container].final_octet}) %}
+{%   set CONTAINER_NETS = vals.get('networks', ['sobridge']) %}
+{%   set IPS = {} %}
+{%   for netname in CONTAINER_NETS %}
+{%     do IPS.update({netname: DOCKERMERGED.networks[netname].prefix ~ vals.final_octet}) %}
+{%   endfor %}
+{%   do DOCKERMERGED.containers[container].update({
+         'networks': CONTAINER_NETS,
+         'ips': IPS,
+         'network': CONTAINER_NETS[0],
+         'ip': IPS[CONTAINER_NETS[0]]
+       }) %}
 {% endfor %}

salt/docker/init.sls

@@ -18,10 +18,10 @@ dockergroup:
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 2.2.1-1.el9
-      - docker-ce: 3:29.2.1-1.el9
-      - docker-ce-cli: 1:29.2.1-1.el9
-      - docker-ce-rootless-extras: 29.2.1-1.el9
+      - containerd.io: 2.3.6-1.el9
+      - docker-ce: 3:29.8.1-1.el9
+      - docker-ce-cli: 1:29.8.1-1.el9
+      - docker-ce-rootless-extras: 29.8.1-1.el9
     - hold: True
     - update_holds: True
 
@@ -71,15 +71,19 @@ dockerreserveports:
     - source: salt://common/files/99-reserved-ports.conf
     - name: /etc/sysctl.d/99-reserved-ports.conf
 
-sos_docker_net:
+{% for NETNAME, NETWORK in DOCKERMERGED.networks.items() %}
+{% if not NETWORK.get('manager_only') or GLOBALS.get('is_manager', False) %}
+sos_docker_net_{{ NETNAME }}:
   docker_network.present:
-    - name: sobridge
-    - subnet: {{ DOCKERMERGED.range }}
-    - gateway: {{ DOCKERMERGED.gateway }}
+    - name: {{ NETNAME }}
+    - subnet: {{ NETWORK.range }}
+    - gateway: {{ NETWORK.gateway }}
     - options:
-        com.docker.network.bridge.name: 'sobridge'
+        com.docker.network.bridge.name: '{{ NETNAME }}'
         com.docker.network.driver.mtu: '1500'
         com.docker.network.bridge.enable_ip_masquerade: 'true'
         com.docker.network.bridge.enable_icc: 'true'
         com.docker.network.bridge.host_binding_ipv4: '0.0.0.0'
-    - unless: ip l | grep sobridge
+    - unless: ip l | grep {{ NETNAME }}
+{% endif %}
+{% endfor %}

salt/docker/soc_docker.yaml

@@ -7,6 +7,40 @@ docker:
     description: Default docker IP range for containers.
     helpLink: docker
     advanced: True
+  networks:
+    sobridge:
+      description: |
+        The default docker network, carrying most containers. Its range and gateway are taken
+        from the docker.range and docker.gateway settings above rather than set here.
+      helpLink: docker
+      readonly: True
+      advanced: True
+      global: True
+    soauth:
+      range:
+        description: |
+          IP range for the soauth docker network, an isolated network for the authentication
+          services, so that the Kratos and Hydra admin APIs are only reachable from the
+          containers placed on it.
+        helpLink: docker
+        readonly: True
+        advanced: True
+        global: True
+      gateway:
+        description: Gateway for the soauth docker network.
+        helpLink: docker
+        readonly: True
+        advanced: True
+        global: True
+      manager_only:
+        description: |
+          Limits the soauth network to grid members running the authentication containers,
+          instead of creating it on every node.
+        helpLink: docker
+        readonly: True
+        advanced: True
+        global: True
+        forcedType: bool
   ulimits:
     description: |
       Default ulimit settings applied to all containers via the Docker daemon. Each entry specifies a resource name (e.g. nofile, memlock, core, nproc) with soft and hard limits. Individual container ulimits override these defaults. Valid resource names include: cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime.
@@ -34,6 +68,16 @@ docker:
         readonly: True
         advanced: True
         global: True
+      networks:
+        description: |
+          Docker networks this container is attached to. The first entry is the container's
+          primary network and determines the address its published ports are forwarded to.
+          Defaults to sobridge when unset.
+        helpLink: docker
+        readonly: True
+        advanced: True
+        global: True
+        forcedType: "[]string"
       port_bindings:
         description: List of port bindings for the container.
         helpLink: docker

salt/elastalert/config.sls

@@ -21,6 +21,7 @@ elastalert:
     - gid: 933
     - home: /opt/so/conf/elastalert
     - createhome: False
+    - shell: /sbin/nologin
 
 elastalogdir:
   file.directory:
@@ -33,8 +34,8 @@ elastalert_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://elastalert/tools/sbin
-    - user: 933
-    - group: 939
+    - user: root
+    - group: root
     - file_mode: 755
 
 #elastalert_sbin_jinja:

salt/elastalert/defaults.yaml

@@ -10,7 +10,7 @@ elastalert:
     buffer_time:
       minutes: 10
     old_query_limit:
-      minutes: 5
+      minutes: 1440
     es_port: 9200
     es_conn_timeout: 55
     max_query_size: 5000

salt/elastalert/enabled.sls

@@ -6,9 +6,11 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 
 include:
+  - ca.trust
   - elastalert.config
   - elastalert.sostatus
 
@@ -28,6 +30,7 @@ so-elastalert:
         - ipv4_address: {{ DOCKERMERGED.containers['so-elastalert'].ip }}
     - detach: True
     - binds:
+      - {{ CATRUST.dir }}
       - /opt/so/rules/elastalert:/opt/elastalert/rules/:ro
       - /opt/so/log/elastalert:/var/log/elastalert:rw
       - /opt/so/conf/elastalert/modules/:/opt/elastalert/modules/:ro
@@ -46,8 +49,9 @@ so-elastalert:
       - {{ XTRAHOST }}
         {% endfor %}
       {% endif %}
-      {% if DOCKERMERGED.containers['so-elastalert'].extra_env %}
     - environment:
+      - REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt
+      {% if DOCKERMERGED.containers['so-elastalert'].extra_env %}
         {% for XTRAENV in DOCKERMERGED.containers['so-elastalert'].extra_env %}
       - {{ XTRAENV }}
         {% endfor %}
@@ -65,6 +69,7 @@ so-elastalert:
       - file: elastacustmodulesdir
       - file: elastaconf
     - watch:
+      - cmd: ca_trust_extract
       - file: elastaconf
     - onlyif:
       - "so-elasticsearch-query / | jq -r '.version.number[0:1]' | grep -q 9" {# only run this state if elasticsearch is version 9 #}

salt/elastalert/files/modules/so/conftest.py

@@ -0,0 +1,80 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+from datetime import datetime
+import json
+import logging
+import sys
+import types
+
+# stand-ins when ElastAlert isn't installed (CI)
+try:
+    import elastalert.alerts  # noqa: F401
+except ImportError:
+
+    class Alerter:
+        def __init__(self, rule):
+            self.rule = rule
+
+    class DateTimeEncoder(json.JSONEncoder):
+        def default(self, obj):
+            return obj.isoformat() if hasattr(obj, 'isoformat') else json.JSONEncoder.default(self, obj)
+
+    class EAException(Exception):
+        pass
+
+    def lookup_es_key(doc, term):
+        for part in term.split('.'):
+            if not isinstance(doc, dict) or part not in doc:
+                return None
+            doc = doc[part]
+        return doc
+
+    def ts_to_dt(value):
+        return value if isinstance(value, datetime) else datetime.fromisoformat(value)
+
+    def elasticsearch_client(conf):
+        return None  # tests set the alerter's client
+
+    alerts = types.ModuleType('elastalert.alerts')
+    alerts.Alerter = Alerter
+    alerts.DateTimeEncoder = DateTimeEncoder
+    util = types.ModuleType('elastalert.util')
+    util.EAException = EAException
+    util.elastalert_logger = logging.getLogger('elastalert')
+    util.lookup_es_key = lookup_es_key
+    util.ts_to_dt = ts_to_dt
+    util.elasticsearch_client = elasticsearch_client
+    package = types.ModuleType('elastalert')
+    package.alerts = alerts
+    package.util = util
+    sys.modules.update({'elastalert': package, 'elastalert.alerts': alerts, 'elastalert.util': util})
+
+# stand-ins when elasticsearch-py isn't installed (CI)
+try:
+    import elasticsearch.exceptions  # noqa: F401
+except ImportError:
+
+    class ElasticsearchException(Exception):
+        pass
+
+    class TransportError(ElasticsearchException):
+        pass
+
+    class ConnectionError(TransportError):
+        pass
+
+    class ConflictError(TransportError):
+        pass
+
+    class RequestError(TransportError):
+        pass
+
+    exceptions = types.ModuleType('elasticsearch.exceptions')
+    for cls in (ElasticsearchException, TransportError, ConnectionError, ConflictError, RequestError):
+        setattr(exceptions, cls.__name__, cls)
+    es_package = types.ModuleType('elasticsearch')
+    es_package.exceptions = exceptions
+    sys.modules.update({'elasticsearch': es_package, 'elasticsearch.exceptions': exceptions})

salt/elastalert/files/modules/so/securityonion-es.py

@@ -1,63 +1,269 @@
 # -*- coding: utf-8 -*-
 
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-
-from time import gmtime, strftime
-import requests,json
-from elastalert.alerts import Alerter
+from datetime import datetime, timezone
+import hashlib
+import ipaddress
+import json
+import re
+import uuid
 
 import urllib3
+from elasticsearch.exceptions import ConflictError, ElasticsearchException, RequestError
+from elastalert.alerts import Alerter, DateTimeEncoder
+from elastalert.util import EAException, elastalert_logger, elasticsearch_client, lookup_es_key, ts_to_dt
+
+# grid runs verify_certs: false; also quiets ElastAlert's own queries
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
+ALERT_INDEX = 'logs-detections.alerts-so'
+# ES error text kept in logs and alerts
+ERROR_TEXT_LIMIT = 500
+# a match missing backend columns (window_start, count, @timestamp); the alert is still written
+MATCH_ERRORS = (KeyError, TypeError, ValueError)
+
+
 class SecurityOnionESAlerter(Alerter):
     """
     Use matched data to create alerts in Elasticsearch.
     """
 
-    required_options = set(['detection_title', 'sigma_level'])
-    optional_fields = ['sigma_category', 'sigma_product', 'sigma_service']
+    required_options = {'detection_title', 'sigma_level'}
+    optional_fields = ['sigma_category', 'sigma_product', 'sigma_service', 'sigma_correlation']
+
+    # count column and default summary per type; stored alert data, so not localized
+    CORRELATION_COUNTS = {
+        'event_count': ('event_count', '%count% events'),
+        'value_count': ('value_count', '%count% distinct values'),
+        'temporal': ('event_type_count', '%count% correlated rules matched'),
+        'value_sum': ('value_sum', 'total %count%'),
+        'value_avg': ('value_avg', 'average %count%'),
+        'value_percentile': ('value_percentile', 'percentile %count%'),
+        'value_median': ('value_median', 'median %count%'),
+    }
+    PLACEHOLDER = re.compile(r'%([^%\s]+)%')
+    # group-by fields copied into ECS related.*
+    RELATED_USERS = {'user.name', 'winlog.event_data.TargetUserName', 'winlog.event_data.SubjectUserName'}
+    RELATED_HOSTS = {'host.name', 'host.hostname', 'winlog.computer_name'}
+
+    def __init__(self, rule):
+        super().__init__(rule)
+        # uses the grid's TLS, auth and timeout settings
+        self.es = elasticsearch_client(rule)
+
+    @property
+    def is_correlation(self):
+        return bool(self.rule.get('sigma_correlation'))
+
+    def query_keys(self):
+        """compound_query_key holds the list; query_key is flattened to a string."""
+        if self.rule.get('compound_query_key'):
+            return self.rule['compound_query_key']
+        if self.rule.get('query_key'):
+            return [self.rule['query_key']]
+        return []
+
+    def alert_id(self, match):
+        """Stable id: window end + group values for correlations, source _id otherwise; random without one."""
+        if self.is_correlation:
+            # ungrouped rows have a hashed _id that changes with the count
+            values = ''.join(f"|{lookup_es_key(match, k)}" for k in self.query_keys())
+            key = f"{self.rule['detection_public_id']}|{ts_to_dt(match['@timestamp']).isoformat()}{values}"
+        elif match.get('_id'):
+            key = f"{self.rule['detection_public_id']}|{match['_id']}"
+        else:
+            return uuid.uuid4().hex
+
+        return hashlib.sha256(key.encode('utf-8')).hexdigest()
+
+    def group(self, match):
+        """Group-by values, joined like ElastAlert's realert key."""
+        return ', '.join(str(lookup_es_key(match, k)) for k in self.query_keys())
+
+    def related_bucket(self, key):
+        if key == 'ip' or key.endswith('.ip'):
+            return 'ip'
+        if key in self.RELATED_USERS or key.endswith('.user.name'):
+            return 'user'
+        if key in self.RELATED_HOSTS:
+            return 'hosts'
+        return None
+
+    @staticmethod
+    def valid_ip(value):
+        try:
+            ipaddress.ip_address(value)
+            return True
+        except ValueError:
+            return False
+
+    def related(self, match):
+        """ECS related.* from group-by values; skips invalid IPs."""
+        related = {}
+        for key in self.query_keys():
+            bucket = self.related_bucket(key)
+            if not bucket:
+                continue
+            # original spellings of a lowercased group
+            value = lookup_es_key(match, f"{key}_spellings")
+            if value is None:
+                value = lookup_es_key(match, key)
+            for v in value if isinstance(value, list) else [value]:
+                if v is None or (bucket == 'ip' and not self.valid_ip(str(v))):
+                    continue
+                # dict: ordered and deduped
+                related.setdefault(bucket, {})[str(v)] = None
+        return {bucket: list(values) for bucket, values in related.items()}
+
+    def event_data(self, match):
+        """The match minus the compound query_key field, which ES would map by its last part."""
+        if not self.rule.get('compound_query_key'):
+            return match
+        return {k: v for k, v in match.items() if k != self.rule['query_key']}
+
+    @staticmethod
+    def format_value(value):
+        if isinstance(value, list):
+            shown = ', '.join(str(v) for v in value[:3])
+            return shown if len(value) <= 3 else f"{shown} and {len(value) - 3} more"
+        return str(value)
+
+    @staticmethod
+    def format_count(value):
+        if isinstance(value, float) and not value.is_integer():
+            return f"{value:,.2f}"
+        if isinstance(value, (int, float)):
+            return f"{int(value):,}"
+        return str(value)
+
+    @staticmethod
+    def format_duration(seconds):
+        for unit, size in (('hour', 3600), ('minute', 60)):
+            if seconds >= 2 * size:
+                return f"{seconds // size} {unit}s"
+        return f"{seconds} second{'' if seconds == 1 else 's'}"
+
+    def summary(self, match):
+        """One-line correlation summary."""
+        column, label = self.CORRELATION_COUNTS.get(self.rule['sigma_correlation'], (None, '%count%'))
+        start = ts_to_dt(match['window_start'])
+        end = ts_to_dt(match['@timestamp'])
+        values = {
+            'count': self.format_count(match.get(column)),
+            'start': start.strftime('%Y-%m-%d %H:%M:%S UTC'),
+            'end': end.strftime('%Y-%m-%d %H:%M:%S UTC'),
+            'duration': self.format_duration(int((end - start).total_seconds())),
+        }
+
+        template = self.rule.get('summary_template')
+        if not template:
+            groups = ', '.join(f"{k} %{k}%" for k in self.query_keys())
+            template = f"{label} for {groups} in %duration%" if groups else f"{label} in %duration%"
+
+        def fill(m):
+            if m[1] in values:
+                return values[m[1]]
+            value = lookup_es_key(match, m[1])
+            # unknown placeholders stay visible so typos show
+            return m[0] if value is None else self.format_value(value)
+
+        return self.PLACEHOLDER.sub(fill, template)
 
     def alert(self, matches):
         for match in matches:
-            timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S"'.000Z', gmtime())
-            headers = {"Content-Type": "application/json"}
+            try:
+                alert_id = self.alert_id(match)
+            except MATCH_ERRORS as e:
+                elastalert_logger.warning("Writing alert for rule %s without a stable id, so a retry may duplicate it: %r",
+                                          self.rule['detection_public_id'], e)
+                alert_id = uuid.uuid4().hex
+            try:
+                self.write(alert_id, self.payload(match))
+            except ElasticsearchException as e:
+                # EAException makes ElastAlert retry
+                raise EAException(f"Unable to write the alert to Elasticsearch: {str(e)[:ERROR_TEXT_LIMIT]}") from e
 
-            creds = None
-            if 'es_username' in self.rule and 'es_password' in self.rule:
-                creds = (self.rule['es_username'], self.rule['es_password'])
+    def payload(self, match):
+        rule_info = {
+            "name": self.rule['detection_title'],
+            "uuid": self.rule['detection_public_id']
+        }
 
-            # Start building the rule dict
-            rule_info = {
-                "name": self.rule['detection_title'],
-                "uuid": self.rule['detection_public_id']
-            }
+        # Add optional fields if they are present in the rule
+        for field in self.optional_fields:
+            rule_key = field.split('_')[-1]  # Assumes field format "sigma_<key>"
+            if field in self.rule:
+                rule_info[rule_key] = self.rule[field]
 
-            # Add optional fields if they are present in the rule
-            for field in self.optional_fields:
-                rule_key = field.split('_')[-1]  # Assumes field format "sigma_<key>"
-                if field in self.rule:
-                    rule_info[rule_key] = self.rule[field]
+        event_info = {
+            "kind": "alert",
+            "severity": self.rule['event.severity'],
+            "module": self.rule['event.module'],
+            "dataset": self.rule['event.dataset'],
+            "severity_label": self.rule['sigma_level']
+        }
 
-            # Construct the payload with the conditional rule_info
-            payload = {
-                "tags": "alert",
-                "rule": rule_info,
-                "event": {
-                    "severity": self.rule['event.severity'],
-                    "module": self.rule['event.module'],
-                    "dataset": self.rule['event.dataset'],
-                    "severity_label": self.rule['sigma_level']
-                },
-                "sigma_level": self.rule['sigma_level'],
-                "event_data": match,
-                "@timestamp": timestamp
-            }
-            url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/logs-detections.alerts-so/_doc/"
-            requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds)
+        payload = {
+            "tags": ["alert"],
+            "rule": rule_info,
+            "event": event_info,
+            "sigma_level": self.rule['sigma_level'],
+            "event_data": self.event_data(match),
+            "@timestamp": datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.000Z')
+        }
+
+        if self.is_correlation:
+            keys = self.query_keys()
+            try:
+                # built before any is added, so a failure adds none
+                reason = self.summary(match)
+                labels = {"correlation_group_by": ', '.join(keys), "correlation_group": self.group(match)} if keys else None
+                related = self.related(match)
+            except MATCH_ERRORS as e:
+                elastalert_logger.warning("Writing alert for rule %s without its correlation summary: %r",
+                                          self.rule['detection_public_id'], e)
+            else:
+                payload["event"]["reason"] = reason
+                if labels:
+                    payload["labels"] = labels
+                if related:
+                    payload["related"] = related
+
+        return payload
+
+    def write(self, alert_id, payload):
+        try:
+            self.create(alert_id, payload)
+        except RequestError as e:
+            # mapping rejections come from event_data; retry it as text
+            rejection = str(e)[:ERROR_TEXT_LIMIT]
+            try:
+                self.create(alert_id, self.without_event_data(payload, rejection))
+            except RequestError as again:
+                elastalert_logger.error("Dropping alert %s for rule %s, rejected by Elasticsearch even without its event data: %s; first rejection: %s",
+                                        alert_id, self.rule['detection_public_id'], str(again)[:ERROR_TEXT_LIMIT], rejection)
+                return
+            elastalert_logger.warning("Stored alert %s for rule %s with its event data as text, rejected by Elasticsearch: %s",
+                                      alert_id, self.rule['detection_public_id'], rejection)
+
+    def create(self, alert_id, payload):
+        try:
+            self.es.create(index=ALERT_INDEX, id=alert_id, body=payload)
+        except ConflictError:
+            pass  # a repeat id is already stored
+
+    @staticmethod
+    def without_event_data(payload, rejection):
+        """Moves event_data to event.original; the tag keeps Fleet's final pipeline from dropping it."""
+        fallback = {k: v for k, v in payload.items() if k != 'event_data'}
+        fallback['event'] = dict(payload['event'], original=json.dumps(payload['event_data'], cls=DateTimeEncoder))
+        fallback['error'] = {'message': f"event_data rejected by Elasticsearch: {rejection}"}
+        fallback['tags'] = payload['tags'] + ['preserve_original_event']
+        return fallback
 
     def get_info(self):
         return {'type': 'SecurityOnionESAlerter'}

salt/elastalert/files/modules/so/securityonion-es_test.py

@@ -0,0 +1,260 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+import copy
+from datetime import datetime, timezone
+import importlib.util
+import json
+import os
+import unittest
+from unittest.mock import MagicMock
+
+from elasticsearch.exceptions import ConflictError, ConnectionError, RequestError
+
+spec = importlib.util.spec_from_file_location('securityonion_es', os.path.join(os.path.dirname(__file__), 'securityonion-es.py'))
+es = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(es)
+
+BASE_RULE = {
+    'detection_title': 'Many Failed Network Logons To One Host From One Source',
+    'detection_public_id': '35a42db6-6629-45af-b8aa-e1fa33c28ef5',
+    'sigma_level': 'medium',
+    'sigma_correlation': 'event_count',
+    'event.severity': 3,
+    'event.module': 'sigma',
+    'event.dataset': 'sigma.alert',
+    'summary_template': '%count% failed network logons to %host.name% from %source.ip% in %duration%',
+}
+
+PLAIN_RULE = {k: v for k, v in BASE_RULE.items() if k not in ('sigma_correlation', 'summary_template')}
+
+
+def correlation_match():
+    return {
+        'event_count': 3561,
+        'window_start': '2026-09-30T18:05:10+00:00',
+        '@timestamp': '2026-09-30T18:07:53+00:00',
+        'host': {'name': 'host-01'},
+        'source': {'ip': '192.0.2.10'},
+        '_id': '6d1c',
+        'num_hits': 1,
+        'num_matches': 1,
+    }
+
+
+class TestSecurityOnionESAlerter(unittest.TestCase):
+
+    def creates(self, rule, match, effects=None):
+        """Run alert(); return (body, id) of each create."""
+        alerter = es.SecurityOnionESAlerter(rule)
+        alerter.es = MagicMock()
+        alerter.es.create.side_effect = effects
+        alerter.alert([match])
+        calls = alerter.es.create.call_args_list
+        self.assertTrue(all(c.kwargs['index'] == 'logs-detections.alerts-so' for c in calls))
+        return [(json.loads(json.dumps(c.kwargs['body'], cls=es.DateTimeEncoder)), c.kwargs['id']) for c in calls]
+
+    def send(self, rule, match):
+        """Run alert(); return the payload it wrote and its id."""
+        (payload, alert_id), = self.creates(rule, match)
+        return payload, alert_id
+
+    def test_compound_query_key_left_out_of_event_data(self):
+        rule = dict(BASE_RULE, compound_query_key=['host.name', 'source.ip'], query_key='host.name,source.ip')
+        match = correlation_match()
+        match['host.name,source.ip'] = 'host-01, 192.0.2.10'
+        original = copy.deepcopy(match)
+
+        payload, _ = self.send(rule, match)
+
+        self.assertNotIn('host.name,source.ip', payload['event_data'])
+        self.assertEqual(payload['event_data']['host'], {'name': 'host-01'})
+        self.assertEqual(payload['event_data']['source'], {'ip': '192.0.2.10'})
+        self.assertEqual(payload['rule'], {'name': BASE_RULE['detection_title'], 'uuid': BASE_RULE['detection_public_id'], 'correlation': 'event_count'})
+        self.assertEqual(payload['labels'], {'correlation_group_by': 'host.name, source.ip', 'correlation_group': 'host-01, 192.0.2.10'})
+        self.assertEqual(payload['related'], {'hosts': ['host-01'], 'ip': ['192.0.2.10']})
+        self.assertEqual(payload['event']['kind'], 'alert')
+        self.assertEqual(payload['event']['reason'], '3,561 failed network logons to host-01 from 192.0.2.10 in 2 minutes')
+        # ElastAlert reuses the match
+        self.assertEqual(match, original)
+
+    def test_single_query_key(self):
+        rule = dict(BASE_RULE, query_key='user.name', summary_template='%count% failed SOC logins for %user.name%')
+        match = {'event_count': 3, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00',
+                 'user': {'name': 'user@example.invalid'}}
+
+        payload, _ = self.send(rule, match)
+
+        self.assertEqual(payload['labels'], {'correlation_group_by': 'user.name', 'correlation_group': 'user@example.invalid'})
+        self.assertEqual(payload['related'], {'user': ['user@example.invalid']})
+        self.assertEqual(payload['event']['reason'], '3 failed SOC logins for user@example.invalid')
+        self.assertEqual(payload['event_data'], match)
+
+    def test_related_buckets(self):
+        rule = dict(BASE_RULE, compound_query_key=['winlog.event_data.TargetUserName', 'source.ip', 'dns.highest_registered_domain'],
+                    query_key='winlog.event_data.TargetUserName,source.ip,dns.highest_registered_domain')
+        match = correlation_match()
+        match.update({'winlog': {'event_data': {'TargetUserName': ['admin1', 'svc', 'admin1']}},
+                      'source': {'ip': 'not-an-ip'}, 'dns': {'highest_registered_domain': 'example.com'}})
+
+        payload, _ = self.send(rule, match)
+
+        # deduped; invalid IP skipped; domain stays in the group only
+        self.assertEqual(payload['related'], {'user': ['admin1', 'svc']})
+        self.assertEqual(payload['labels']['correlation_group'], "['admin1', 'svc', 'admin1'], not-an-ip, example.com")
+
+        payload, _ = self.send(dict(BASE_RULE, query_key='dns.highest_registered_domain'), match)
+
+        self.assertNotIn('related', payload)
+
+    def test_related_uses_original_spellings(self):
+        rule = dict(BASE_RULE, query_key='user.name', summary_template=None)
+        match = {'event_count': 3, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00',
+                 'user': {'name': 'admin', 'name_spellings': ['Admin', 'admin', 'ADMIN']}}
+
+        payload, _ = self.send(rule, match)
+
+        # the group shows the lowercased value; related.user finds every spelling
+        self.assertEqual(payload['labels']['correlation_group'], 'admin')
+        self.assertEqual(payload['related'], {'user': ['Admin', 'admin', 'ADMIN']})
+        self.assertEqual(payload['event']['reason'], '3 events for user.name admin in 8 seconds')
+
+    def test_plain_rule_has_no_correlation_fields(self):
+        # a query_key alone (e.g. from an override) isn't a correlation
+        rule = dict(PLAIN_RULE, query_key='user.name')
+        # ElastAlert parses @timestamp for EQL hits
+        match = {'@timestamp': datetime(2026, 9, 30, 16, 50, tzinfo=timezone.utc), '_id': 'abc',
+                 'process': {'name': 'whoami.exe'}, 'user': {'name': 'user'}}
+
+        payload, alert_id = self.send(rule, match)
+
+        alerter = es.SecurityOnionESAlerter(rule)
+        self.assertEqual(payload['rule'], {'name': PLAIN_RULE['detection_title'], 'uuid': PLAIN_RULE['detection_public_id']})
+        self.assertNotIn('labels', payload)
+        self.assertNotIn('related', payload)
+        self.assertNotIn('reason', payload['event'])
+        self.assertEqual(payload['event']['kind'], 'alert')
+        self.assertEqual(payload['event_data'], dict(match, **{'@timestamp': '2026-09-30T16:50:00+00:00'}))
+        self.assertEqual(alert_id, alerter.alert_id(match))
+        self.assertNotEqual(alerter.alert_id(match), alerter.alert_id(dict(match, _id='abd')))
+        self.assertNotEqual(alerter.alert_id(match), es.SecurityOnionESAlerter(dict(rule, detection_public_id='other')).alert_id(match))
+        # without an _id, never deduplicated
+        self.assertNotEqual(alerter.alert_id({'_id': None}), alerter.alert_id({'_id': None}))
+
+    def test_ungrouped_correlation_id_ignores_row_hash(self):
+        rule = dict(BASE_RULE, summary_template=None)
+        first = {k: v for k, v in correlation_match().items() if k not in ('host', 'source')}
+        # ES|QL hashes the row into _id, so a later count changes it
+        later = dict(first, event_count=3600, _id='9f2a')
+
+        payload, alert_id = self.send(rule, first)
+
+        alerter = es.SecurityOnionESAlerter(rule)
+        self.assertEqual(alerter.alert_id(first), alerter.alert_id(later))
+        self.assertNotEqual(alerter.alert_id(first), alerter.alert_id(dict(first, **{'@timestamp': '2026-09-30T18:09:00+00:00'})))
+        self.assertEqual(alert_id, alerter.alert_id(first))
+        self.assertEqual(payload['event']['reason'], '3,561 events in 2 minutes')
+        self.assertNotIn('labels', payload)
+
+    def test_temporal_count_column(self):
+        rule = dict(BASE_RULE, sigma_correlation='temporal', summary_template=None)
+        match = {'event_type_count': 2, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00'}
+
+        payload, _ = self.send(rule, match)
+
+        self.assertEqual(payload['event']['reason'], '2 correlated rules matched in 8 seconds')
+
+    def test_grouped_correlation_id_unchanged(self):
+        """Ids of alerts already written must not change."""
+        rule = dict(BASE_RULE, compound_query_key=['host.name', 'source.ip'], query_key='host.name,source.ip')
+        key = f"{BASE_RULE['detection_public_id']}|2026-09-30T18:07:53+00:00|host-01|192.0.2.10"
+
+        self.assertEqual(es.SecurityOnionESAlerter(rule).alert_id(correlation_match()), es.hashlib.sha256(key.encode()).hexdigest())
+
+    def test_rejected_event_data_is_kept_as_text(self):
+        rule = dict(BASE_RULE, query_key='user.name', summary_template='%count% failed SOC logins for %user.name%')
+        match = {'event_count': 3, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00',
+                 'user': {'name': 'user@example.invalid'}}
+        rejected = RequestError(400, 'document_parsing_exception', {})
+
+        (first, _), (second, _) = self.creates(rule, match, [rejected, None])
+
+        self.assertIn('event_data', first)
+        self.assertNotIn('event_data', second)
+        self.assertEqual(json.loads(second['event']['original']), match)
+        self.assertEqual(second['tags'], ['alert', 'preserve_original_event'])
+        self.assertEqual(first['tags'], ['alert'])
+        self.assertTrue(second['error']['message'].startswith('event_data rejected by Elasticsearch: '))
+        self.assertIn('document_parsing_exception', second['error']['message'])
+        # everything else carries over
+        self.assertEqual({k: v for k, v in second['event'].items() if k != 'original'}, first['event'])
+        changed = ('event_data', 'event', 'error', 'tags')
+        self.assertEqual({k: v for k, v in second.items() if k not in changed}, {k: v for k, v in first.items() if k not in changed})
+
+    def test_rejected_dates_are_kept_as_text(self):
+        match = {'@timestamp': datetime(2026, 9, 30, 16, 50, tzinfo=timezone.utc), '_id': 'abc'}
+
+        _, (second, _) = self.creates(PLAIN_RULE, match, [RequestError(400, 'document_parsing_exception', {}), None])
+
+        self.assertEqual(json.loads(second['event']['original']), {'@timestamp': '2026-09-30T16:50:00+00:00', '_id': 'abc'})
+
+    def test_rejected_twice_is_dropped_without_retry(self):
+        rejected = RequestError(400, 'document_parsing_exception', {})
+        match = {'@timestamp': '2026-09-30T16:50:00+00:00', '_id': 'abc'}
+
+        with self.assertLogs('elastalert', 'ERROR') as logs:
+            self.assertEqual(len(self.creates(PLAIN_RULE, match, [rejected, rejected])), 2)
+
+        self.assertIn('Dropping alert', logs.output[0])
+
+    def test_write_failure_is_retried(self):
+        match = {'@timestamp': '2026-09-30T16:50:00+00:00', '_id': 'abc'}
+
+        with self.assertRaisesRegex(es.EAException, 'Unable to write the alert to Elasticsearch'):
+            self.creates(PLAIN_RULE, match, [ConnectionError('N/A', 'refused', None)])
+
+    def test_repeat_id_is_ignored(self):
+        match = {'@timestamp': '2026-09-30T16:50:00+00:00', '_id': 'abc'}
+
+        self.assertEqual(len(self.creates(PLAIN_RULE, match, [ConflictError(409, 'version_conflict_engine_exception', {})])), 1)
+
+    def test_correlation_fields_are_optional(self):
+        rule = dict(BASE_RULE, query_key='source.ip')
+        # no window_start: the summary cannot be built
+        match = {k: v for k, v in correlation_match().items() if k != 'window_start'}
+
+        with self.assertLogs('elastalert', 'WARNING') as logs:
+            payload, _ = self.send(rule, match)
+
+        self.assertIn('without its correlation summary', logs.output[0])
+        self.assertEqual(payload['event']['kind'], 'alert')
+        self.assertNotIn('reason', payload['event'])
+        self.assertNotIn('labels', payload)
+        self.assertNotIn('related', payload)
+
+    def test_unstable_id_still_writes(self):
+        # no @timestamp: the window end is unknown
+        match = {k: v for k, v in correlation_match().items() if k not in ('@timestamp', 'window_start')}
+
+        with self.assertLogs('elastalert', 'WARNING') as logs:
+            payload, alert_id = self.send(BASE_RULE, match)
+
+        self.assertIn('without a stable id', logs.output[0])
+        self.assertEqual(len(alert_id), 32)
+        self.assertEqual(payload['event_data'], match)
+
+    def test_summary_formats_values(self):
+        rule = dict(BASE_RULE, sigma_correlation='value_avg', query_key='source.ip',
+                    summary_template='%count% for %source.ip% to %destination.port% %no.such.field%')
+        match = {'value_avg': 2.5, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00',
+                 'source': {'ip': '192.0.2.10'}, 'destination': {'port': [22, 80, 443, 8080, 8443]}}
+
+        payload, _ = self.send(rule, match)
+
+        self.assertEqual(payload['event']['reason'], '2.50 for 192.0.2.10 to 22, 80, 443 and 2 more %no.such.field%')
+        self.assertEqual(es.SecurityOnionESAlerter.format_count('n/a'), 'n/a')
+
+    def test_get_info(self):
+        self.assertEqual(es.SecurityOnionESAlerter(PLAIN_RULE).get_info(), {'type': 'SecurityOnionESAlerter'})

salt/elastalert/soc_elastalert.yaml

@@ -120,7 +120,7 @@ elastalert:
         helpLink: elastalert
     old_query_limit:
       minutes:
-        description: Amount of time in minutes between queries to start at the most recently run query.
+        description: How long ElastAlert can be down, in minutes, and still resume each rule where it stopped. After a longer outage, rules restart from now and skip the gap.
         global: True
         helpLink: elastalert
     es_conn_timeout:

salt/elastic-fleet-package-registry/config.sls

@@ -19,6 +19,7 @@ elastic-agent-pr:
     - gid: 948
     - home: /opt/so/conf/elastic-fleet-pr
     - createhome: False
+    - shell: /sbin/nologin
 
 {% else %}
 

salt/elastic-fleet-package-registry/enabled.sls

@@ -6,9 +6,11 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 
 include:
+  - ca.trust
   - elastic-fleet-package-registry.config
   - elastic-fleet-package-registry.sostatus
 
@@ -34,8 +36,9 @@ so-elastic-fleet-package-registry:
       {% for BINDING in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].port_bindings %}
       - {{ BINDING }}
       {% endfor %}
-    {% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
     - binds:
+      - {{ CATRUST.pem }}:/etc/ssl/certs/ca-certificates.crt:ro
+    {% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
         {% for BIND in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
       - {{ BIND }}
         {% endfor %}
@@ -52,6 +55,8 @@ so-elastic-fleet-package-registry:
       - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
     {%   endfor %}
     {% endif %}
+    - watch:
+      - cmd: ca_trust_extract
 
 wait_for_so-elastic-fleet-package-registry:
   http.wait_for_successful_query:

salt/elasticagent/config.sls

@@ -20,6 +20,7 @@ elastic-agent:
     - gid: 949
     - home: /opt/so/conf/elastic-agent
     - createhome: False
+    - shell: /sbin/nologin
 
 elasticagentconfdir:
   file.directory:
@@ -39,8 +40,8 @@ elasticagent_sbin_jinja:
   file.recurse:
     - name: /usr/sbin
     - source: salt://elasticagent/tools/sbin_jinja
-    - user: 949
-    - group: 939
+    - user: root
+    - group: root
     - file_mode: 755
     - template: jinja
 

salt/elasticagent/enabled.sls

@@ -6,6 +6,7 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 
 include:
@@ -37,6 +38,7 @@ so-elastic-agent:
       - {{ BINDING }}
       {% endfor %}
     - binds:
+      - {{ CATRUST.dir }}
       - /opt/so/conf/elastic-agent/elastic-agent.yml:/usr/share/elastic-agent/elastic-agent.yml:ro
       - /opt/so/log/elasticagent:/usr/share/elastic-agent/logs
       - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro 
@@ -65,6 +67,7 @@ so-elastic-agent:
       - file: create-elastic-agent-config
       - file: trusttheca
     - watch:
+      - cmd: ca_trust_extract
       - file: create-elastic-agent-config
       - file: trusttheca
 

salt/elasticfleet/config.sls

@@ -26,13 +26,14 @@ elastic-fleet:
     - gid: 947
     - home: /opt/so/conf/elastic-fleet
     - createhome: False
+    - shell: /sbin/nologin
 
 elasticfleet_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://elasticfleet/tools/sbin
-    - user: 947
-    - group: 939
+    - user: root
+    - group: root
     - file_mode: 755
     - show_changes: False
 
@@ -40,8 +41,8 @@ elasticfleet_sbin_jinja:
   file.recurse:
     - name: /usr/sbin
     - source: salt://elasticfleet/tools/sbin_jinja
-    - user: 947
-    - group: 939 
+    - user: root
+    - group: root
     - file_mode: 755
     - template: jinja
     - exclude_pat:
@@ -81,8 +82,8 @@ eapackageupgrade:
   file.managed:
     - name: /usr/sbin/so-elastic-fleet-package-upgrade
     - source: salt://elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
-    - user: 947
-    - group: 939
+    - user: root
+    - group: root
     - mode: 755
     - template: jinja
 

salt/elasticfleet/enabled.sls

@@ -6,6 +6,7 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
 
@@ -63,6 +64,7 @@ so-elastic-fleet:
       - {{ BINDING }}
       {% endfor %}
     - binds:
+      - {{ CATRUST.dir }}
       - /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro
       - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro
       - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro
@@ -95,6 +97,7 @@ so-elastic-fleet:
     {%   endfor %}
     {% endif %}
     - watch:
+      - cmd: ca_trust_extract
       - file: trusttheca
       - x509: etc_elasticfleet_key
       - x509: etc_elasticfleet_crt

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -5,7 +5,7 @@
   "package": {
     "name": "endpoint",
     "title": "Elastic Defend",
-    "version": "9.3.1",
+    "version": "9.4.1",
     "requires_root": true
   },
   "enabled": true,

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -29,7 +29,7 @@
               "\\.gz$"
             ],
             "include_files": [],
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.20.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.3\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.20.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.20.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.3\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- add_fields:\n    target: event\n    fields:\n      dataset: windows.forwarded\n      module: windows\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: import.evtx\n- if:\n    equals:\n      winlog.channel: 'Security'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: system.security\n          module: system\n- if:\n    equals:\n      winlog.channel: 'Windows PowerShell'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell\n          module: windows\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n          module: system\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n          module: system\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n- add_fields:\n    target: data_stream\n    fields:\n        type: logs\n        dataset: import",
             "tags": [
               "import"
             ],

salt/elasticfleet/install_agent_grid.sls

@@ -14,8 +14,8 @@ so-elastic-agent-install:
   file.managed:
     - name: /usr/sbin/so-elastic-agent-install
     - source: salt://elasticfleet/tools/sbin/so-elastic-agent-install
-    - user: 947
-    - group: 939
+    - user: root
+    - group: root
     - mode: 755
     - show_changes: False
 

salt/elasticfleet/integration-defaults.map.jinja

@@ -16,7 +16,6 @@
     'awsfirehose.metrics': 'aws.cloudwatch',
     'cribl.logs': 'cribl',
     'cribl.metrics': 'cribl',
-    'sentinel_one_cloud_funnel.logins': 'sentinel_one_cloud_funnel.login',
     'azure_application_insights.app_insights': 'azure.app_insights',
     'azure_application_insights.app_state': 'azure.app_state',
     'azure_billing.billing': 'azure.billing',
@@ -31,8 +30,6 @@
     'azure_metrics.monitor': 'azure.monitor',
     'azure_metrics.storage_account': 'azure.storage_account',
     'azure_openai.metrics': 'azure.open_ai',
-    'beat.state': 'beats.stack_monitoring.state',
-    'beat.stats': 'beats.stack_monitoring.stats',
     'enterprisesearch.health': 'enterprisesearch.stack_monitoring.health',
     'enterprisesearch.stats': 'enterprisesearch.stack_monitoring.stats',
     'kibana.cluster_actions': 'kibana.stack_monitoring.cluster_actions',
@@ -41,7 +38,8 @@
     'kibana.node_rules': 'kibana.stack_monitoring.node_rules',
     'kibana.stats': 'kibana.stack_monitoring.stats',
     'kibana.status': 'kibana.stack_monitoring.status',
-    'logstash.node_cel': 'logstash.stack_monitoring.node',
+    'logstash.node': 'logstash.stack_monitoring.node',
+    'logstash.node_cel': 'logstash.node',
     'logstash.node_stats': 'logstash.stack_monitoring.node_stats',
     'synthetics.browser': 'synthetics-browser',
     'synthetics.browser_network': 'synthetics-browser.network',

salt/elasticfleet/manager.sls

@@ -68,6 +68,24 @@ so-elastic-fleet-package-upgrade:
     - require:
       - http: wait_for_so-kibana
 
+# initial so-elasticsearch-templates run is earlier, but it can skip over templates that have component templates not yet installed to avoid elasticsearch rejecting the template.
+so-elasticsearch-templates-after-fleet-packages:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-templates-load
+    - cwd: /opt/so
+    - unless: test -f /opt/so/state/estemplates.txt
+    - require:
+      - cmd: so-elastic-fleet-package-upgrade
+
+so-elastic-fleet-integration-upgrade:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-integration-upgrade
+    - retry:
+        attempts: 3
+        interval: 10
+    - require:
+      - cmd: so-elastic-fleet-package-upgrade
+
 so-elastic-fleet-integrations:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-integration-policy-load
@@ -86,21 +104,13 @@ so-elastic-agent-grid-upgrade:
     - require:
       - http: wait_for_so-kibana
 
-so-elastic-fleet-integration-upgrade:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-integration-upgrade
-    - retry:
-        attempts: 3
-        interval: 10
-    - require:
-      - http: wait_for_so-kibana
-
 {# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
 so-elastic-fleet-addon-integrations:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
     - require:
       - http: wait_for_so-kibana
+      - cmd: so-elasticsearch-templates-after-fleet-packages
 
 {% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
 so-elastic-defend-manage-filters-file-watch:

salt/elasticfleet/tools/sbin/so-elastic-fleet-common

@@ -30,6 +30,56 @@ fleet_api() {
     curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/${QUERYPATH}" "$@" --retry 3 --retry-delay 10 --fail 2>/dev/null
 }
 
+elastic_fleet_require_agent_policy() {
+    local AGENT_POLICY=$1
+    local POLICY_JSON
+
+    if ! POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY") || [ -z "$POLICY_JSON" ]; then
+        echo "Error: Agent policy '$AGENT_POLICY' was not found or is not visible in the current Kibana space." >&2
+        return 1
+    fi
+
+    if ! jq -e '.item.package_policies | type == "array"' <<<"$POLICY_JSON" >/dev/null 2>&1; then
+        echo "Error: Agent policy '$AGENT_POLICY' was not found or is not visible in the current Kibana space." >&2
+        return 1
+    fi
+
+    echo "$POLICY_JSON"
+}
+
+# Print the single active enrollment token for POLICY_ID.
+# Exit 1: retryable (API failure, invalid response, no active token)
+# Exit 2: multiple active tokens - Shouldn't get into this state without manual intervention
+elastic_fleet_active_enrollment_token() {
+    local POLICY_ID=$1
+    local RESP TOKEN_COUNT API_KEY
+
+    if ! RESP=$(fleet_api "enrollment_api_keys?perPage=100" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
+        echo "Error: Failed to retrieve enrollment tokens for agent policy '$POLICY_ID'." >&2
+        return 1
+    fi
+
+    if ! jq -e '.list' <<<"$RESP" >/dev/null 2>&1; then
+        echo "Error: Invalid enrollment token response for agent policy '$POLICY_ID'." >&2
+        return 1
+    fi
+
+    TOKEN_COUNT=$(jq --arg pid "$POLICY_ID" '[.list[] | select(.policy_id == $pid and .active == true)] | length' <<<"$RESP")
+
+    if [ "${TOKEN_COUNT:-0}" -eq 0 ]; then
+        echo "Error: No active enrollment token found for agent policy '$POLICY_ID'." >&2
+        return 1
+    fi
+
+    if [ "$TOKEN_COUNT" -gt 1 ]; then
+        echo "Error: Found $TOKEN_COUNT active enrollment tokens for agent policy '$POLICY_ID'; expected exactly one." >&2
+        return 2
+    fi
+
+    API_KEY=$(jq -r --arg pid "$POLICY_ID" '.list[] | select(.policy_id == $pid and .active == true) | .api_key' <<<"$RESP")
+    echo "$API_KEY"
+}
+
 # Max number of concurrent Fleet write jobs (create/update). Override via env if needed.
 MAX_FLEET_JOBS=${MAX_FLEET_JOBS:-10}
 
@@ -62,15 +112,7 @@ elastic_fleet_load_integrations_dir() {
     i=0
 
     # Fetch the agent policy a single time; we look up integration ids locally below.
-    if ! POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY"); then
-        echo "Error: Failed to retrieve agent policy '$AGENT_POLICY'."
-        rm -f "$FAIL_FILE"
-        rm -rf "$OUT_DIR"
-        return 1
-    fi
-
-    if ! jq -e '.item.package_policies' <<<"$POLICY_JSON" >/dev/null 2>&1; then
-        echo "Error: Invalid agent policy response for '$AGENT_POLICY'."
+    if ! POLICY_JSON=$(elastic_fleet_require_agent_policy "$AGENT_POLICY"); then
         rm -f "$FAIL_FILE"
         rm -rf "$OUT_DIR"
         return 1
@@ -124,9 +166,15 @@ elastic_fleet_integration_check() {
 
     JSON_STRING=$2
 
-    NAME=$(jq -r .name $JSON_STRING)
+    NAME=$(jq -r .name "$JSON_STRING")
+    INTEGRATION_ID=""
 
-    INTEGRATION_ID=$(/usr/sbin/so-elastic-fleet-agent-policy-view "$AGENT_POLICY" | jq -r '.item.package_policies[] | select(.name=="'"$NAME"'") | .id')
+    local POLICY_JSON
+    if ! POLICY_JSON=$(elastic_fleet_require_agent_policy "$AGENT_POLICY"); then
+        return 1
+    fi
+
+    INTEGRATION_ID=$(jq -r --arg n "$NAME" '.item.package_policies[]? | select(.name==$n) | .id' <<<"$POLICY_JSON")
 
 }
 
@@ -148,7 +196,16 @@ elastic_fleet_integration_remove() {
 
     NAME=$2
 
-    INTEGRATION_ID=$(/usr/sbin/so-elastic-fleet-agent-policy-view "$AGENT_POLICY" | jq -r '.item.package_policies[] | select(.name=="'"$NAME"'") | .id')
+    local POLICY_JSON
+    if ! POLICY_JSON=$(elastic_fleet_require_agent_policy "$AGENT_POLICY"); then
+        return 1
+    fi
+
+    INTEGRATION_ID=$(jq -r --arg n "$NAME" '.item.package_policies[]? | select(.name==$n) | .id' <<<"$POLICY_JSON")
+    if [ -z "$INTEGRATION_ID" ]; then
+        echo "Error: Integration '$NAME' was not found in agent policy '$AGENT_POLICY'." >&2
+        return 1
+    fi
 
     JSON_STRING=$( jq -n \
                     --arg INTEGRATIONID "$INTEGRATION_ID" \

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-defend

@@ -13,7 +13,10 @@ ERROR=false
 for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/elastic-defend/*.json
 do
   printf "\n\nInitial Endpoints Policy - Loading $INTEGRATION\n"
-  elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
+  if ! elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"; then
+    ERROR=true
+    continue
+  fi
   if [ -n "$INTEGRATION_ID" ]; then
       printf "\n\nIntegration $NAME exists - Upgrading integration policy\n"
       if ! elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"; then

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server

@@ -7,20 +7,35 @@
 . /usr/sbin/so-elastic-fleet-common
 
 # Get all the fleet policies
-json_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -L -X GET "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true')
+if ! json_output=$(fleet_api "agent_policies" -H 'kbn-xsrf: true'); then
+    echo "Error: Failed to retrieve Fleet agent policies." >&2
+    exit 1
+fi
+
+if ! jq -e '.items' <<<"$json_output" >/dev/null 2>&1; then
+    echo "Error: Invalid Fleet agent policies response." >&2
+    exit 1
+fi
 
 # Extract the IDs that start with "FleetServer_"
-POLICY=$(echo "$json_output" | jq -r '.items[] | select(.id | startswith("FleetServer_")) | .id')
+POLICY=$(jq -r '.items[] | select(.id | startswith("FleetServer_")) | .id' <<<"$json_output")
 
 # Iterate over each ID in the POLICY variable
 for POLICYNAME in $POLICY; do
     printf "\nUpdating Policy: $POLICYNAME\n"
 
-    # First get the Integration ID
-    INTEGRATION_ID=$(/usr/sbin/so-elastic-fleet-agent-policy-view "$POLICYNAME" | jq -r '.item.package_policies[] | select(.package.name == "fleet_server") | .id')  
+    if ! POLICY_JSON=$(elastic_fleet_require_agent_policy "$POLICYNAME"); then
+        exit 1
+    fi
+
+    INTEGRATION_ID=$(jq -r '.item.package_policies[]? | select(.package.name == "fleet_server") | .id' <<<"$POLICY_JSON")
+    if [ -z "$INTEGRATION_ID" ]; then
+        echo "Error: fleet_server integration was not found in agent policy '$POLICYNAME'." >&2
+        exit 1
+    fi
 
     # Modify the default integration policy to update the policy_id and an with the correct naming
-	UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "$POLICYNAME" --arg name "fleet_server-$POLICYNAME" '
+    UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "$POLICYNAME" --arg name "fleet_server-$POLICYNAME" '
     .policy_id = $policy_id |
     .name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json)
 

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers

@@ -22,12 +22,19 @@ NUM_RUNNING=$(pgrep -cf "/bin/bash /sbin/so-elastic-agent-gen-installers")
 
 for i in {1..30}
 do
-   ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys?perPage=100" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
+   ENROLLMENTOKEN=$(elastic_fleet_active_enrollment_token "endpoints-initial")
+   TOKEN_RC=$?
+   if [ "$TOKEN_RC" -eq 2 ]; then
+     exit 1
+   fi
    FLEETHOST=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts/grid-default' | jq -r  '.item.host_urls[]' | paste -sd ',')
-if [[ $FLEETHOST ]] && [[  $ENROLLMENTOKEN ]]; then break; else sleep 10; fi
+   if [[ -n "$FLEETHOST" ]] && [[ -n "$ENROLLMENTOKEN" ]]; then
+     break
+   fi
+   sleep 10
 done
 
-if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then 
+if [[ -z "$FLEETHOST" ]] || [[ -z "$ENROLLMENTOKEN" ]]; then
   printf "\nFleet Host URL, Enrollment Token or Elastic Version empty - exiting..."
   printf "\nFleet Host: $FLEETHOST, Enrollment Token: $ENROLLMENTOKEN\n"
   exit 1
@@ -67,19 +74,25 @@ for GOOS in "${GOTARGETOS[@]}"; do
     GOARCH="amd64"
     if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi
     printf "\n\n### Generating $GOOS/$GOARCH Installer...\n"
-    docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \
+    if ! docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \
     --mount type=bind,source=/etc/pki/tls/certs/,target=/workspace/files/cert/ \
     --mount type=bind,source=/nsm/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \
     --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ \
-    {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN"  -o /output/so-elastic-agent_${GOOS}_${GOARCH}
+    {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN"  -o /output/so-elastic-agent_${GOOS}_${GOARCH}; then
+        printf "\n### ERROR: Failed to generate $GOOS/$GOARCH installer. Exiting...\n"
+        exit 1
+    fi
     printf "\n### $GOOS/$GOARCH Installer Generated...\n"
 done
 
 printf "\n\n### Generating MSI...\n"
 cp /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe
-docker run \
+if ! docker run \
 --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ -w /output \
-{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs
+{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs; then
+    printf "\n### ERROR: Failed to generate MSI. Exiting...\n"
+    exit 1
+fi
 printf "\n### MSI Generated...\n"
 
 # Verify installers were created

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade

@@ -10,6 +10,25 @@
 
 PKG_LOAD_FAILURES=0
 PKG_LOAD_FAILURES_NAMES=()
+PKG_UPGRADED=0
+
+cleanup_elasticsearch_fleet_transforms() {
+    local transforms transform_id attempt
+    
+    if ! transforms=$(so-elasticsearch-query "_transform/logs-elasticsearch.index_pivot-default-*" --retry 1 --retry-delay 5); then
+        return 0
+    fi
+
+    while IFS= read -r transform_id; do
+        [ -n "$transform_id" ] || continue
+        for attempt in {1..3}; do
+            if so-elasticsearch-query "_transform/$transform_id?force=true" -XDELETE --fail --retry 1 --retry-delay 5; then
+                break
+            fi
+            sleep 5
+        done
+    done < <(jq -r '.transforms[]?.id' <<< "$transforms")
+}
 
 {%- for PACKAGE in SUPPORTED_PACKAGES %}
 if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && LATEST_VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
@@ -17,10 +36,25 @@ if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && L
     if [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then
         echo "{{ PACKAGE }} integration version $INSTALLED_VERSION is already at the reported latest version $LATEST_VERSION, skipping upgrade."
     else
+{%- if PACKAGE == 'elasticsearch' %}
+        cleanup_elasticsearch_fleet_transforms
+{%- endif %}
         echo "Upgrading {{ PACKAGE }} package from $INSTALLED_VERSION to version $LATEST_VERSION..."
         if ! elastic_fleet_package_install "{{ PACKAGE }}" "$LATEST_VERSION"; then
             PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
             PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
+        # check that package has upgraded to the expected version after install command
+        elif ! LATEST_VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
+            echo "ERROR: Failed to get latest version information for integration {{ PACKAGE }} after upgrade attempt"
+            PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
+            PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
+        elif INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then
+            echo "{{ PACKAGE }} integration upgraded to version $LATEST_VERSION."
+            PKG_UPGRADED=$((PKG_UPGRADED + 1))
+        else
+            echo "ERROR: {{ PACKAGE }} integration still at ${INSTALLED_VERSION:-unknown}; expected $LATEST_VERSION"
+            PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
+            PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
         fi
     fi
 else
@@ -30,6 +64,11 @@ else
 fi
 {%- endfor %}
 
+if [ $PKG_UPGRADED -gt 0 ]; then
+    echo "Elasticsearch template statefiles cleared after $PKG_UPGRADED package upgrade(s), so templates can reload."
+    rm -f /opt/so/state/estemplates.txt /opt/so/state/addon_estemplates.txt
+fi
+
 if [ $PKG_LOAD_FAILURES -gt 0 ]; then
     echo "ERROR: Failed to upgrade $PKG_LOAD_FAILURES package(s):"
     for PKG in "${PKG_LOAD_FAILURES_NAMES[@]}"; do

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -202,26 +202,9 @@ fi
 ### Finalization ###
 
 # Query for Enrollment Tokens for default policies
-if ENDPOINTSENROLLMENTOKEN_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
-    ENDPOINTSENROLLMENTOKEN=$(echo "$ENDPOINTSENROLLMENTOKEN_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
-else
-    echo -e "\nFailed to query for Endpoints enrollment token"
-    exit 1
-fi
-
-if GRIDNODESENROLLMENTOKENGENERAL_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
-    GRIDNODESENROLLMENTOKENGENERAL=$(echo "$GRIDNODESENROLLMENTOKENGENERAL_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_general")) | .api_key')
-else
-    echo -e "\nFailed to query for Grid nodes - General enrollment token"
-    exit 1
-fi
-
-if GRIDNODESENROLLMENTOKENHEAVY_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
-    GRIDNODESENROLLMENTOKENHEAVY=$(echo "$GRIDNODESENROLLMENTOKENHEAVY_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_heavy")) | .api_key')
-else
-    echo -e "\nFailed to query for Grid nodes - Heavy enrollment token"
-    exit 1
-fi
+ENDPOINTSENROLLMENTOKEN=$(elastic_fleet_active_enrollment_token "endpoints-initial") || exit 1
+GRIDNODESENROLLMENTOKENGENERAL=$(elastic_fleet_active_enrollment_token "so-grid-nodes_general") || exit 1
+GRIDNODESENROLLMENTOKENHEAVY=$(elastic_fleet_active_enrollment_token "so-grid-nodes_heavy") || exit 1
 
 # Store needed data in minion pillar
 pillar_file=/opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls

salt/elasticsearch/ca.sls

@@ -1,49 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states or sls in allowed_states %}
-{%   from 'vars/globals.map.jinja' import GLOBALS %}
-
-# Move our new CA over so Elastic and Logstash can use SSL with the internal CA
-catrustdir:
-  file.directory:
-    - name: /opt/so/conf/ca
-    - user: 939
-    - group: 939
-    - makedirs: True
-
-{%   if GLOBALS.is_manager %}
-# We have to add the Manager CA to the CA list
-catrustscript:
-  cmd.script:
-    - source: salt://elasticsearch/tools/sbin_jinja/so-catrust
-    - template: jinja
-    - cwd: /opt/so
-    - defaults:
-        GLOBALS: {{ GLOBALS }}
-{%   endif %}
-
-elasticsearch_cacerts:
-  file.managed:
-    - name: /opt/so/conf/ca/cacerts
-    - source: salt://elasticsearch/cacerts
-    - user: 939
-    - group: 939
-
-elasticsearch_capems:
-  file.managed:
-    - name: /opt/so/conf/ca/tls-ca-bundle.pem
-    - source: salt://elasticsearch/tls-ca-bundle.pem
-    - user: 939
-    - group: 939
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/elasticsearch/cluster.sls

@@ -98,6 +98,13 @@ so-es-cluster-settings:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
       - http: wait_for_so-elasticsearch
+
+so-elasticsearch-system-indices-patch:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-system-indices-patch
+    - require:
+      - http: wait_for_so-elasticsearch
+      - file: so-elasticsearch-system-indices-patch-script
 {%  endif %}
 
 # heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)

salt/elasticsearch/config.sls

@@ -32,24 +32,35 @@ elasticsearch:
     - gid: 930
     - home: /opt/so/conf/elasticsearch
     - createhome: False
+    - shell: /sbin/nologin
 
 elasticsearch_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://elasticsearch/tools/sbin
-    - user: 930
-    - group: 939
+    - user: root
+    - group: root
     - file_mode: 755
     - exclude_pat:
       - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state
+      - so-elasticsearch-system-indices-patch
+    - show_changes: False
+
+so-elasticsearch-system-indices-patch-script:
+  file.managed:
+    - name: /usr/sbin/so-elasticsearch-system-indices-patch
+    - source: salt://elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch
+    - user: root
+    - group: root
+    - mode: 755
     - show_changes: False
 
 elasticsearch_sbin_jinja:
   file.recurse:
     - name: /usr/sbin
     - source: salt://elasticsearch/tools/sbin_jinja
-    - user: 939
-    - group: 939 
+    - user: root
+    - group: root
     - file_mode: 755
     - template: jinja
     - exclude_pat:
@@ -62,8 +73,8 @@ so-elasticsearch-ilm-policy-load-script:
   file.managed:
     - name: /usr/sbin/so-elasticsearch-ilm-policy-load
     - source: salt://elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load
-    - user: 930
-    - group: 939
+    - user: root
+    - group: root
     - mode: 754
     - template: jinja
     - defaults:
@@ -74,8 +85,8 @@ so-elasticsearch-pipelines-script:
   file.managed:
     - name: /usr/sbin/so-elasticsearch-pipelines
     - source: salt://elasticsearch/tools/sbin/so-elasticsearch-pipelines
-    - user: 930
-    - group: 939
+    - user: root
+    - group: root
     - mode: 754
     - show_changes: False
 

salt/elasticsearch/defaults.yaml

@@ -1,7 +1,7 @@
 elasticsearch:
   enabled: false
   esheap: '600m'
-  version: 9.3.7
+  version: 9.4.8
   index_clean: true
   data_retention_method: DLM
   vm:
@@ -1160,6 +1160,7 @@ elasticsearch:
         - so-fleet_agent_id_verification-1
         - so-logs-mappings
         - so-logs-settings
+        - detections-alerts-mappings
         data_stream:
           allow_custom_routing: false
           hidden: false
@@ -3309,6 +3310,7 @@ elasticsearch:
         composed_of:
         - event-mappings
         - logs-system.security@package
+        - so-fleet_system.security_caseless-1
         - logs-system.security@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -4175,6 +4177,7 @@ elasticsearch:
       index_template:
         composed_of:
         - logs-windows.forwarded@package
+        - so-fleet_process_caseless-1
         - logs-windows.forwarded@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -4224,6 +4227,7 @@ elasticsearch:
       index_template:
         composed_of:
         - logs-windows.powershell@package
+        - so-fleet_process_caseless-1
         - logs-windows.powershell@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -4273,6 +4277,7 @@ elasticsearch:
       index_template:
         composed_of:
         - logs-windows.powershell_operational@package
+        - so-fleet_process_caseless-1
         - logs-windows.powershell_operational@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -4322,6 +4327,7 @@ elasticsearch:
       index_template:
         composed_of:
         - logs-windows.sysmon_operational@package
+        - so-fleet_process_caseless-1
         - logs-windows.sysmon_operational@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1

salt/elasticsearch/enabled.sls

@@ -6,6 +6,7 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
@@ -13,7 +14,6 @@
 
 include:
   - ca
-  - elasticsearch.ca
   - elasticsearch.ssl
   - elasticsearch.config
   - elasticsearch.sostatus
@@ -63,15 +63,14 @@ so-elasticsearch:
       - {{ BINDING }}
       {% endfor %}
     - binds:
+      - {{ CATRUST.dir }}
       - /opt/so/conf/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
       - /opt/so/conf/elasticsearch/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro
       - /nsm/elasticsearch:/usr/share/elasticsearch/data:rw
       - /opt/so/log/elasticsearch:/var/log/elasticsearch:rw
-      - /opt/so/conf/ca/cacerts:/usr/share/elasticsearch/jdk/lib/security/cacerts:ro
       - /etc/pki/tls/certs/intca.crt:/usr/share/elasticsearch/config/ca.crt:ro
       - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro
       - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro
-      - /etc/pki/elasticsearch.p12:/usr/share/elasticsearch/config/elasticsearch.p12:ro
       - /opt/so/conf/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles:ro
       - /opt/so/conf/elasticsearch/users:/usr/share/elasticsearch/config/users:ro
       {% if ELASTICSEARCHMERGED.config.path.get('repo', False) %}
@@ -85,21 +84,19 @@ so-elasticsearch:
         {% endfor %}
       {% endif %}
     - watch:
+      - cmd: ca_trust_extract
       - file: trusttheca
       - x509: elasticsearch_crt
       - x509: elasticsearch_key
-      - file: elasticsearch_cacerts
       - file: esyml
     - require:
       - file: trusttheca
       - x509: elasticsearch_crt
       - x509: elasticsearch_key
-      - file: elasticsearch_cacerts
       - file: esyml
       - file: eslog4jfile
       - file: nsmesdir
       - file: eslogdir
-      - file: elasticp12perms
       - cmd: auth_users_roles_inode
       - cmd: auth_users_inode
 

salt/elasticsearch/files/ingest/global@custom

@@ -1,5 +1,5 @@
 {
-    "version": 3,
+    "version": 4,
     "_meta": {
         "managed_by": "securityonion",
         "managed": true
@@ -99,7 +99,7 @@
         },
         {
             "set": {
-                "if": "ctx.tags != null && ctx.tags.contains('import')",
+                "if": "ctx.tags != null && ctx.tags.contains('import') && ctx._index != null && ctx._index.startsWith('logs-import-')",
                 "override": true,
                 "field": "data_stream.dataset",
                 "value": "import"
@@ -107,7 +107,7 @@
         },
         {
             "set": {
-                "if": "ctx.tags != null && ctx.tags.contains('import')",
+                "if": "ctx.tags != null && ctx.tags.contains('import') && ctx._index != null && ctx._index.startsWith('logs-import-')",
                 "override": true,
                 "field": "data_stream.namespace",
                 "value": "so"
@@ -177,6 +177,32 @@
                 "description": "Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"
             }
         },
+        {
+            "trim": {
+                "if": "ctx.event?.dataset == 'system.auth'",
+                "description": "Grok leaves a leading space on 'invalid user' names (elastic/integrations#12174) and, before 2.23.2, sudo padding",
+                "field": "user.name",
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        },
+        {
+            "trim": {
+                "if": "ctx.event?.dataset == 'system.auth'",
+                "description": "Appended from the untrimmed user.name",
+                "field": "related.user",
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        },
+        {
+            "script": {
+                "if": "ctx.event?.dataset == 'system.auth' && ctx.related?.user instanceof List",
+                "description": "Dedupe after trimming",
+                "source": "ctx.related.user = new ArrayList(new LinkedHashSet(ctx.related.user));",
+                "ignore_failure": true
+            }
+        },
         {
             "script": {
                 "description": "Snapshot event.ingested into _tmp.event_ingested_pre_fleet before .fleet_final_pipeline-1 overwrites it with ES ingest time",

salt/elasticsearch/files/ingest/import.evtx

@@ -0,0 +1,31 @@
+{
+  "description" : "import.evtx: normalize imported EVTX and reroute to logs-<dataset>-import",
+  "processors" : [
+    { "script":  {
+        "description": "Host from the event, not the importing node",
+        "lang": "painless",
+        "source": "Map host = ['os': ['type': 'windows', 'family': 'windows', 'platform': 'windows']]; def cn = ctx.winlog?.computer_name; if (cn != null && cn.toString().length() > 0) { String name = cn.toString(); int dot = name.indexOf('.'); if (dot > 0) { name = name.substring(0, dot); } host.put('hostname', name); host.put('name', name.toLowerCase()); } ctx.host = host;"
+    } },
+    { "script":  {
+        "description": "String event IDs, as Winlogbeat sends",
+        "lang": "painless",
+        "source": "if (ctx.winlog?.event_id != null) { ctx.winlog.event_id = ctx.winlog.event_id.toString(); } if (ctx.event?.code != null) { ctx.event.code = ctx.event.code.toString(); }"
+    } },
+    { "script":  {
+        "description": "Unnamed <Data> to param1..N, as Winlogbeat",
+        "lang": "painless",
+        "if": "ctx.winlog?.event_data?.Data instanceof Map && ctx.winlog.event_data.Data['#text'] != null",
+        "source": "def t = ctx.winlog.event_data.Data['#text']; List vals = t instanceof List ? t : [t]; for (int i = 0; i < vals.size(); i++) { ctx.winlog.event_data['param' + (i + 1)] = vals.get(i); } ctx.winlog.event_data.remove('Data');"
+    } },
+    { "script":  {
+        "description": "String values and LF line endings, as Winlogbeat",
+        "lang": "painless",
+        "if": "ctx.winlog?.event_data instanceof Map || ctx.winlog?.user_data instanceof Map",
+        "source": "String lf = String.valueOf((char) 10); String crlf = String.valueOf((char) 13) + lf; for (def key : ['event_data', 'user_data']) { def m = ctx.winlog[key]; if (!(m instanceof Map)) { continue; } for (def e : m.entrySet()) { def v = e.getValue(); if (v instanceof String) { e.setValue(v.replace(crlf, lf)); } else if (v instanceof Number || v instanceof Boolean) { e.setValue(v.toString()); } } }"
+    } },
+    { "set":     { "description": "event.kind, as Winlogbeat", "field": "event.kind", "value": "event", "override": false } },
+    { "set":     { "field": "data_stream.dataset", "copy_from": "event.dataset", "override": true, "ignore_empty_value": true } },
+    { "set":     { "field": "data_stream.namespace", "value": "import", "override": true } },
+    { "reroute": { "dataset": "{{data_stream.dataset}}", "namespace": "{{data_stream.namespace}}" } }
+  ]
+}

salt/elasticsearch/files/ingest/suricata.fileinfo

@@ -5,7 +5,8 @@
     { "rename": 	{ "field": "message2.proto", 			"target_field": "network.transport",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.fileinfo.filename", 	"target_field": "file.name",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.fileinfo.gaps", 		"target_field": "file.bytes.missing",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fileinfo.gaps", 		"target_field": "suricata.fileinfo.gaps",	"ignore_missing": true 	} },
+    { "set":		{ "if": "ctx.suricata?.fileinfo?.gaps == false", 	"field": "file.bytes.missing",	"value": 0 		} },
     { "rename": 	{ "field": "message2.fileinfo.magic", 		"target_field": "file.mime_type",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.fileinfo.md5", 		"target_field": "hash.md5",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.fileinfo.sha1", 		"target_field": "hash.sha1",		"ignore_missing": true 	} },

salt/elasticsearch/ssl.sls

@@ -38,10 +38,6 @@ elasticsearch_crt:
     - retry:
         attempts: 5
         interval: 30
-  cmd.run:
-    - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12 -nodes -passout pass:"
-    - onchanges:
-      - x509: /etc/pki/elasticsearch.key
 
 elastickeyperms:
   file.managed:
@@ -50,13 +46,6 @@ elastickeyperms:
     - mode: 640
     - group: 930
     
-elasticp12perms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/elasticsearch.p12
-    - mode: 640
-    - group: 930
-
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/elasticsearch/templates/component/elastic-agent/so-fleet_process_caseless-1.json

@@ -0,0 +1,123 @@
+{
+  "_meta": {
+    "managed_by": "security_onion",
+    "managed": true,
+    "description": "Adds .caseless for Lucene queries. Restates each field's package type and .text."
+  },
+  "template": {
+    "mappings": {
+      "properties": {
+        "process": {
+          "properties": {
+            "executable": {
+              "type": "keyword",
+              "ignore_above": 1024,
+              "fields": {
+                "caseless": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "normalizer": "lowercase"
+                },
+                "text": {
+                  "type": "match_only_text"
+                }
+              }
+            },
+            "name": {
+              "type": "keyword",
+              "ignore_above": 1024,
+              "fields": {
+                "caseless": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "normalizer": "lowercase"
+                },
+                "text": {
+                  "type": "match_only_text"
+                }
+              }
+            },
+            "command_line": {
+              "type": "wildcard",
+              "ignore_above": 1024,
+              "fields": {
+                "caseless": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "normalizer": "lowercase"
+                },
+                "text": {
+                  "type": "match_only_text"
+                }
+              }
+            },
+            "parent": {
+              "properties": {
+                "executable": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "fields": {
+                    "caseless": {
+                      "type": "keyword",
+                      "ignore_above": 1024,
+                      "normalizer": "lowercase"
+                    },
+                    "text": {
+                      "type": "match_only_text"
+                    }
+                  }
+                },
+                "name": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "fields": {
+                    "caseless": {
+                      "type": "keyword",
+                      "ignore_above": 1024,
+                      "normalizer": "lowercase"
+                    },
+                    "text": {
+                      "type": "match_only_text"
+                    }
+                  }
+                },
+                "command_line": {
+                  "type": "wildcard",
+                  "ignore_above": 1024,
+                  "fields": {
+                    "caseless": {
+                      "type": "keyword",
+                      "ignore_above": 1024,
+                      "normalizer": "lowercase"
+                    },
+                    "text": {
+                      "type": "match_only_text"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
+        "file": {
+          "properties": {
+            "path": {
+              "type": "keyword",
+              "ignore_above": 1024,
+              "fields": {
+                "caseless": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "normalizer": "lowercase"
+                },
+                "text": {
+                  "type": "match_only_text"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/elastic-agent/so-fleet_system.security_caseless-1.json

@@ -0,0 +1,80 @@
+{
+  "_meta": {
+    "managed_by": "security_onion",
+    "managed": true,
+    "description": "Adds .caseless for Lucene queries. Keeps each field's existing keyword type."
+  },
+  "template": {
+    "mappings": {
+      "properties": {
+        "process": {
+          "properties": {
+            "command_line": {
+              "type": "keyword",
+              "ignore_above": 1024,
+              "fields": {
+                "caseless": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "normalizer": "lowercase"
+                }
+              }
+            },
+            "parent": {
+              "properties": {
+                "executable": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "fields": {
+                    "caseless": {
+                      "type": "keyword",
+                      "ignore_above": 1024,
+                      "normalizer": "lowercase"
+                    }
+                  }
+                },
+                "name": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "fields": {
+                    "caseless": {
+                      "type": "keyword",
+                      "ignore_above": 1024,
+                      "normalizer": "lowercase"
+                    }
+                  }
+                },
+                "command_line": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "fields": {
+                    "caseless": {
+                      "type": "keyword",
+                      "ignore_above": 1024,
+                      "normalizer": "lowercase"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
+        "file": {
+          "properties": {
+            "path": {
+              "type": "keyword",
+              "ignore_above": 1024,
+              "fields": {
+                "caseless": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "normalizer": "lowercase"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/so/detection-mappings.json

@@ -154,6 +154,20 @@
             }
           }
         }
+      },
+      "runtime": {
+        "so_detection.sigmaKind": {
+          "type": "keyword",
+          "script": {
+            "source": "if (doc['so_detection.language'].size() == 0 || doc['so_detection.language'].value != 'sigma') { return; } def d = params._source.so_detection; if (d == null || d.content == null) { return; } String c = d.content; String NL = String.valueOf((char) 10); emit(c.startsWith('correlation:') || c.contains(NL + 'correlation:') ? 'correlation' : 'single');"
+          }
+        },
+        "so_detection.sigmaCorrelationType": {
+          "type": "keyword",
+          "script": {
+            "source": "if (doc['so_detection.language'].size() == 0 || doc['so_detection.language'].value != 'sigma') { return; } def d = params._source.so_detection; if (d == null || d.content == null) { return; } String c = d.content; String NL = String.valueOf((char) 10); int i = c.startsWith('correlation:') ? 0 : c.indexOf(NL + 'correlation:'); if (i < 0) { return; } i = c.indexOf(NL, i + 1); while (i >= 0 && i + 1 < c.length()) { int e = c.indexOf(NL, i + 1); String line = e < 0 ? c.substring(i + 1) : c.substring(i + 1, e); if (line.length() > 0 && !Character.isWhitespace(line.charAt(0)) && line.charAt(0) != (char) 35) { return; } String t = line.trim(); if (t.startsWith('type:')) { String v = t.substring(5).trim(); int h = v.indexOf(' ' + String.valueOf((char) 35)); if (h >= 0) { v = v.substring(0, h).trim(); } if (v.length() > 1 && (v.charAt(0) == (char) 39 || v.charAt(0) == (char) 34)) { v = v.substring(1, v.length() - 1); } emit(v); return; } i = e; }"
+          }
+        }
       }
     }
   },

salt/elasticsearch/templates/component/so/detections-alerts-mappings.json

@@ -0,0 +1,149 @@
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "tags": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "sigma_level": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "rule": {
+          "properties": {
+            "name": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "uuid": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "category": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "product": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "service": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "correlation": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
+        },
+        "event": {
+          "properties": {
+            "severity": {
+              "type": "long"
+            },
+            "severity_label": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "module": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "dataset": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "kind": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "reason": {
+              "type": "match_only_text",
+              "fields": {
+                "keyword": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                }
+              }
+            },
+            "original": {
+              "type": "keyword",
+              "index": false,
+              "doc_values": false
+            }
+          }
+        },
+        "event_data": {
+          "properties": {
+            "@timestamp": {
+              "type": "date"
+            },
+            "window_start": {
+              "type": "date"
+            },
+            "event_count": {
+              "type": "long"
+            },
+            "value_count": {
+              "type": "long"
+            },
+            "event_type_count": {
+              "type": "long"
+            },
+            "value_sum": {
+              "type": "double"
+            },
+            "value_avg": {
+              "type": "double"
+            },
+            "value_percentile": {
+              "type": "double"
+            },
+            "value_median": {
+              "type": "double"
+            }
+          }
+        },
+        "labels": {
+          "properties": {
+            "correlation_group_by": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "correlation_group": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
+        },
+        "related": {
+          "properties": {
+            "ip": {
+              "type": "ip"
+            },
+            "user": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "hosts": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
+        },
+        "error": {
+          "properties": {
+            "message": {
+              "type": "match_only_text"
+            }
+          }
+        }
+      }
+    }
+  },
+  "_meta": {
+    "description": "Fields written by the ElastAlert SecurityOnionESAlerter to logs-detections.alerts-so"
+  }
+}

salt/elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch

@@ -0,0 +1,203 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+set -eo pipefail
+
+SETTINGS='{"index":{"auto_expand_replicas":"0-1"}}'
+KIBANA_PASSWORD=
+INDEX_PATTERNS=(
+  '.entity_analytics.risk_score.lookup-*'
+  '.entity_analytics.watchlists.*'
+  '.entity_analytics.monitoring.users-*'
+  '.entity_analytics.entity-leads-*'
+  '.asset-criticality.asset-criticality-*'
+  '.workflows-executions'
+  '.workflows-step-executions'
+  '.entities.v2.latest.security_*'
+  '.entities.v2.history.security_*'
+  'risk-score.risk-score-latest-*'
+  '.metrics-endpoint.metadata_united_*'
+  '.metrics-endpoint.metadata_current_*'
+)
+DATA_STREAM_PATTERNS=(
+  '.entities.v2.updates.security_*'
+  'risk-score.risk-score-*'
+  '.rule-events'
+  '.alert-actions'
+)
+TEMPLATE_PATTERNS=(
+  'entities_v2_latest_security_default_index_template'
+  'entities_v2_history_security_default_index_template'
+  '.entities_v2_updates_security_default_index_template'
+  '.risk-score.risk-score-default-index-template'
+  '.rule-events'
+  '.alert-actions'
+  '.metrics-endpoint.metadata_united_default-template'
+  '.metrics-endpoint.metadata_current_default-template'
+)
+
+query_es() {
+  if so-elasticsearch-query "$@" --fail --retry 3 --retry-delay 5; then
+    return 0
+  fi
+
+  # retry failed attempts with so_kibana user (system managed indices reject so_elastic user)
+  local query_path="$1"
+  shift
+
+  if [[ -z "$KIBANA_PASSWORD" ]]; then
+    KIBANA_PASSWORD=$(salt-call pillar.get elasticsearch:auth:users:so_kibana_user:pass --out=newline_values_only)
+  fi
+  [[ -n "$KIBANA_PASSWORD" ]] || return 1
+
+  echo "Retrying ${query_path} as so_kibana." >&2
+  curl -K /opt/so/conf/elasticsearch/curl.config --user "so_kibana:${KIBANA_PASSWORD}" \
+    -s -k -L --fail --retry 3 --retry-delay 5 -H 'Content-Type: application/json' "https://localhost:9200/${query_path}" "$@"
+}
+
+# add auto_expand_replicas=0-1 to given index
+set_auto_expand_replicas() {
+  local index="$1"
+
+  echo "Setting auto_expand_replicas to 0-1 on ${index}."
+  query_es "${index}/_settings" -XPUT -d "$SETTINGS" >/dev/null
+}
+
+# resolve index patterns and find each index with an unassigned replica
+unassigned_replicas() {
+  local pattern="$1"
+  local resolved_indices response index
+
+  if ! resolved_indices=$(query_es "_resolve/index/${pattern}?expand_wildcards=all" 2>/dev/null); then
+    return 0
+  fi
+
+  while read -r index; do
+    if ! response=$(query_es "_cat/shards/${index}?format=json&h=index,prirep,state" 2>/dev/null); then
+      continue
+    fi
+    jq -r '.[]? | objects | select(.prirep == "r" and .state == "UNASSIGNED") | .index' <<<"$response"
+  done < <(jq -r '.indices[]?.name' <<<"$resolved_indices")
+}
+
+data_stream_indices() {
+  local pattern="$1"
+  local response
+
+  if ! response=$(query_es "_data_stream/${pattern}?expand_wildcards=all" 2>/dev/null); then
+    return 0
+  fi
+  jq -r '.data_streams[]?.indices[]?.index_name' <<<"$response"
+}
+
+update_system_indices() {
+  local pattern="$1"
+  local index
+
+  while read -r index; do
+    [[ -n "$index" ]] && set_auto_expand_replicas "$index"
+  done < <(unassigned_replicas "$pattern")
+}
+
+# update data stream backing indices with unassigned replicas
+update_system_ds() {
+  local pattern="$1"
+  local index
+
+  while read -r index; do
+    while read -r unassigned_index; do
+      [[ -n "$unassigned_index" ]] && set_auto_expand_replicas "$unassigned_index"
+    done < <(unassigned_replicas "$index")
+  done < <(data_stream_indices "$pattern")
+}
+
+has_unassigned_replicas() {
+  local pattern="$1"
+  local index
+
+  index=$(unassigned_replicas "$pattern" | sed -n '1p')
+  [[ -n "$index" ]]
+}
+
+data_stream_has_unassigned_replicas() {
+  local pattern="$1"
+  local index
+  while read -r index; do
+    has_unassigned_replicas "$index" && return 0
+  done < <(data_stream_indices "$pattern")
+
+  return 1
+}
+
+needs_patch() {
+  local pattern
+  for pattern in "${INDEX_PATTERNS[@]}"; do
+    has_unassigned_replicas "$pattern" && return 0
+  done
+  for pattern in "${DATA_STREAM_PATTERNS[@]}"; do
+    data_stream_has_unassigned_replicas "$pattern" && return 0
+  done
+
+  return 1
+}
+
+# get index templates, update with auto_expand_replicas=0-1, and PUT back. Keeping mappings/settings/aliases in-place
+update_system_templates() {
+  local pattern="$1"
+  local templates name response template auto_expand_replicas
+
+  if ! templates=$(query_es "_index_template/${pattern}" 2>/dev/null); then
+    return 0
+  fi
+  while read -r name; do
+    response=$(query_es "_index_template/${name}")
+    template=$(jq -c '.index_templates[0].index_template' <<<"$response")
+    auto_expand_replicas=$(jq -r '.template.settings["index.auto_expand_replicas"] // .template.settings.index.auto_expand_replicas // empty' <<<"$template")
+    [[ "$auto_expand_replicas" == "0-1" ]] && continue
+
+    template=$(jq '
+            if (.template.settings.index | type) == "object" then
+                .template.settings.index.auto_expand_replicas = "0-1"
+            else
+                .template.settings["index.auto_expand_replicas"] = "0-1"
+            end
+            | del(.created_date_millis, .modified_date_millis)
+        ' <<<"$template")
+    echo "Setting auto_expand_replicas to 0-1 on index template ${name}."
+    query_es "_index_template/${name}" -XPUT -d "$template" >/dev/null
+  done < <(jq -r '.index_templates[]?.name' <<<"$templates")
+}
+
+if [[ "${1:-}" == "--check" ]]; then
+  needs_patch
+  exit $?
+fi
+
+if [[ $# -ne 0 ]]; then
+  echo "Usage: $0 [--check]" >&2
+  exit 1
+fi
+
+patched=false
+for pattern in "${INDEX_PATTERNS[@]}"; do
+  if has_unassigned_replicas "$pattern"; then
+    update_system_indices "$pattern"
+    patched=true
+  fi
+done
+
+for pattern in "${DATA_STREAM_PATTERNS[@]}"; do
+  if data_stream_has_unassigned_replicas "$pattern"; then
+    update_system_ds "$pattern"
+    patched=true
+  fi
+done
+
+if [[ "$patched" == true ]]; then
+  for pattern in "${TEMPLATE_PATTERNS[@]}"; do
+    update_system_templates "$pattern"
+  done
+fi

salt/elasticsearch/tools/sbin_jinja/so-catrust

@@ -1,26 +0,0 @@
-#!/bin/bash
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-get_elastic_agent_vars
-
-# Exit on errors, since all lines must succeed
-set -e
-
-# Check to see if we have extracted the ca cert.
-if [ ! -f /opt/so/saltstack/local/salt/elasticsearch/cacerts ]; then
-    docker run -v /etc/pki/ca.crt:/etc/ssl/ca.crt --name so-elasticsearchca --user root --entrypoint jdk/bin/keytool {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:$ELASTIC_AGENT_TARBALL_VERSION -keystore /usr/share/elasticsearch/jdk/lib/security/cacerts -alias SOSCA -import -file /etc/ssl/ca.crt -storepass changeit -noprompt
-    # Make sure symbolic links are followed when copying from container
-    docker cp -L so-elasticsearchca:/usr/share/elasticsearch/jdk/lib/security/cacerts /opt/so/saltstack/local/salt/elasticsearch/cacerts
-    docker cp -L so-elasticsearchca:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /opt/so/saltstack/local/salt/elasticsearch/tls-ca-bundle.pem
-    docker rm so-elasticsearchca
-    echo "" >> /opt/so/saltstack/local/salt/elasticsearch/tls-ca-bundle.pem
-    echo "sosca" >> /opt/so/saltstack/local/salt/elasticsearch/tls-ca-bundle.pem
-    cat /etc/pki/ca.crt >> /opt/so/saltstack/local/salt/elasticsearch/tls-ca-bundle.pem
-else
-    exit 0
-fi

salt/firewall/iptables.jinja

@@ -4,11 +4,19 @@
 {%- set role = GLOBALS.role.split('-')[1] %}
 {%- from 'firewall/containers.map.jinja' import NODE_CONTAINERS %}
 
+{%- set NODE_NETWORKS = [] %}
+{%- for NETNAME, NETWORK in DOCKERMERGED.networks.items() %}
+{%-   if not NETWORK.get('manager_only') or GLOBALS.get('is_manager', False) %}
+{%-     do NODE_NETWORKS.append(NETNAME) %}
+{%-   endif %}
+{%- endfor %}
+
 {%- set PR = [] %}
 {%- set D1 = [] %}
 {%- set D2 = [] %}
 {%- for container in NODE_CONTAINERS %}
 {%-   set IP = DOCKERMERGED.containers[container].ip %}
+{%-   set BRIDGE = DOCKERMERGED.containers[container].network %}
 {%-   if DOCKERMERGED.containers[container].port_bindings is defined %}
 {%-     for binding in DOCKERMERGED.containers[container].port_bindings %}
 {#-       cant split int so we convert to string #}
@@ -35,11 +43,11 @@
 {%-         endif %}
 {%-         do PR.append("-A POSTROUTING -s " ~ DOCKERMERGED.containers[container].ip ~ "/32 -d " ~ DOCKERMERGED.containers[container].ip ~ "/32 -p " ~  proto ~ " -m " ~ proto  ~ " --dport " ~ containerPort ~ " -j MASQUERADE") %}
 {%-         if bindip | length and bindip != '0.0.0.0' %}
-{%-           do D1.append("-A DOCKER -d " ~ bindip ~ "/32 ! -i sobridge -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
+{%-           do D1.append("-A DOCKER -d " ~ bindip ~ "/32 ! -i " ~ BRIDGE ~ " -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
 {%-         else %}
-{%-           do D1.append("-A DOCKER ! -i sobridge -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
+{%-           do D1.append("-A DOCKER ! -i " ~ BRIDGE ~ " -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
 {%-         endif %}
-{%-         do D2.append("-A DOCKER -d " ~ DOCKERMERGED.containers[container].ip ~ "/32 ! -i sobridge -o sobridge -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ containerPort ~ " -j ACCEPT") %}
+{%-         do D2.append("-A DOCKER -d " ~ DOCKERMERGED.containers[container].ip ~ "/32 ! -i " ~ BRIDGE ~ " -o " ~ BRIDGE ~ " -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ containerPort ~ " -j ACCEPT") %}
 {%-     endfor %}
 {%-   endif %}
 {%- endfor %}
@@ -52,11 +60,15 @@
 :DOCKER - [0:0]
 -A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER
 -A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER
--A POSTROUTING -s {{DOCKERMERGED.range}} ! -o sobridge -j MASQUERADE
+{%- for NETNAME in NODE_NETWORKS %}
+-A POSTROUTING -s {{ DOCKERMERGED.networks[NETNAME].range }} ! -o {{ NETNAME }} -j MASQUERADE
+{%- endfor %}
 {%- for rule in PR %}
 {{ rule }}
 {%- endfor %}
--A DOCKER -i sobridge -j RETURN
+{%- for NETNAME in NODE_NETWORKS %}
+-A DOCKER -i {{ NETNAME }} -j RETURN
+{%- endfor %}
 {%- for rule in D1 %}
 {{ rule }}
 {%- endfor %}
@@ -97,10 +109,12 @@ COMMIT
 {%- endif %}
 -A FORWARD -j DOCKER-USER
 -A FORWARD -j DOCKER-ISOLATION-STAGE-1
--A FORWARD -o sobridge -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
--A FORWARD -o sobridge -j DOCKER
--A FORWARD -i sobridge ! -o sobridge -j ACCEPT
--A FORWARD -i sobridge -o sobridge -j ACCEPT
+{%- for NETNAME in NODE_NETWORKS %}
+-A FORWARD -o {{ NETNAME }} -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A FORWARD -o {{ NETNAME }} -j DOCKER
+-A FORWARD -i {{ NETNAME }} ! -o {{ NETNAME }} -j ACCEPT
+-A FORWARD -i {{ NETNAME }} -o {{ NETNAME }} -j ACCEPT
+{%- endfor %}
 -A FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
 -A FORWARD -i lo -j ACCEPT
 -A FORWARD -m conntrack --ctstate INVALID -j DROP
@@ -112,13 +126,18 @@ COMMIT
 {%- for rule in D2 %}
 {{ rule }}
 {%- endfor %}
-
--A DOCKER-ISOLATION-STAGE-1 -i sobridge ! -o sobridge -j DOCKER-ISOLATION-STAGE-2
+{% for NETNAME in NODE_NETWORKS %}
+-A DOCKER-ISOLATION-STAGE-1 -i {{ NETNAME }} ! -o {{ NETNAME }} -j DOCKER-ISOLATION-STAGE-2
+{%- endfor %}
 -A DOCKER-ISOLATION-STAGE-1 -j RETURN
--A DOCKER-ISOLATION-STAGE-2 -o sobridge -j DROP
+{%- for NETNAME in NODE_NETWORKS %}
+-A DOCKER-ISOLATION-STAGE-2 -o {{ NETNAME }} -j DROP
+{%- endfor %}
 -A DOCKER-ISOLATION-STAGE-2 -j RETURN
--A DOCKER-USER ! -i sobridge -o sobridge -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
--A DOCKER-USER ! -i sobridge -o sobridge -j LOGGING
+{%- for NETNAME in NODE_NETWORKS %}
+-A DOCKER-USER ! -i {{ NETNAME }} -o {{ NETNAME }} -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-USER ! -i {{ NETNAME }} -o {{ NETNAME }} -j LOGGING
+{%- endfor %}
 -A DOCKER-USER -j RETURN
 -A LOGGING -m limit --limit 2/min -j LOG --log-prefix "IPTables-dropped: "
 -A LOGGING -j DROP

salt/firewall/map.jinja

@@ -4,8 +4,12 @@
 
 {# add our ip to self #}
 {% do FIREWALL_DEFAULT.firewall.hostgroups.self.append(GLOBALS.node_ip) %}
-{# add dockernet range #}
-{% do FIREWALL_DEFAULT.firewall.hostgroups.dockernet.append(DOCKERMERGED.range) %}
+{# add dockernet ranges #}
+{% for NETNAME, NETWORK in DOCKERMERGED.networks.items() %}
+{%   if not NETWORK.get('manager_only') or GLOBALS.get('is_manager', False) %}
+{%     do FIREWALL_DEFAULT.firewall.hostgroups.dockernet.append(NETWORK.range) %}
+{%   endif %}
+{% endfor %}
 
 {% if GLOBALS.role == 'so-idh' %}
 {%   from 'idh/opencanary_config.map.jinja' import IDH_PORTGROUPS %}

salt/hydra/enabled.sls

@@ -13,9 +13,11 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   if 'api' in salt['pillar.get']('features', []) %}
 
 include:
+  - ca.trust
   - hydra.config
   - hydra.sostatus
 
@@ -26,9 +28,10 @@ so-hydra:
     - hostname: hydra
     - name: so-hydra
     - networks:
-      - sobridge:
-        - ipv4_address: {{ DOCKERMERGED.containers['so-hydra'].ip }}
+      - soauth:
+        - ipv4_address: {{ DOCKERMERGED.containers['so-hydra'].ips['soauth'] }}
     - binds:
+      - {{ CATRUST.dir }}
       - /opt/so/conf/hydra/:/hydra-conf:ro
       - /opt/so/log/hydra/:/hydra-log:rw
       - /nsm/hydra/db:/hydra-data:rw
@@ -60,6 +63,7 @@ so-hydra:
     {%   endfor %}
     {% endif %}
     - watch:
+      - cmd: ca_trust_extract
       - file: hydraconfig
     - require:
       - file: hydraconfig
@@ -73,7 +77,7 @@ delete_so-hydra_so-status.disabled:
 
 wait_for_hydra:
   http.wait_for_successful_query:
-    - name: 'http://{{ GLOBALS.manager }}:4444/health/alive'
+    - name: 'http://{{ DOCKERMERGED.containers['so-hydra'].ips['soauth'] }}:4444/health/alive'
     - ssl: True
     - verify_ssl: False
     - status:

salt/hypervisor/init.sls

@@ -21,12 +21,16 @@ hypervisor_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://hypervisor/tools/sbin
+    - user: root
+    - group: root
     - file_mode: 744
 
 hypervisor_sbin_jinja:
   file.recurse:
     - name: /usr/sbin
     - source: salt://hypervisor/tools/sbin_jinja
+    - user: root
+    - group: root
     - template: jinja
     - file_mode: 744
 

salt/idh/config.sls

@@ -86,8 +86,8 @@ idh_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://idh/tools/sbin
-    - user: 939
-    - group: 939
+    - user: root
+    - group: root
     - file_mode: 755
 
 #idh_sbin_jinja:

salt/idh/enabled.sls

@@ -6,9 +6,11 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 
 include:
+  - ca.trust
   - idh.config
   - idh.sostatus
 
@@ -20,6 +22,7 @@ so-idh:
     - detach: True
     - network_mode: host
     - binds:
+      - {{ CATRUST.dir }}
       - /nsm/idh:/var/tmp:rw
       - /opt/so/conf/idh/http-skins:/opt/opencanary/http-skins:ro
       - /opt/so/conf/idh/opencanary.conf:/etc/opencanaryd/opencanary.conf:ro
@@ -34,12 +37,13 @@ so-idh:
       - {{ XTRAHOST }}
       {% endfor %}
     {% endif %}
-    {% if DOCKERMERGED.containers['so-idh'].extra_env %}
     - environment:
-      {% for XTRAENV in DOCKERMERGED.containers['so-idh'].extra_env %}
+      - REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt
+      {% if DOCKERMERGED.containers['so-idh'].extra_env %}
+        {% for XTRAENV in DOCKERMERGED.containers['so-idh'].extra_env %}
       - {{ XTRAENV }}
-      {% endfor %}
-    {% endif %}
+        {% endfor %}
+      {% endif %}
     {% if DOCKERMERGED.containers['so-idh'].ulimits %}
     - ulimits:
     {%   for ULIMIT in DOCKERMERGED.containers['so-idh'].ulimits %}
@@ -47,6 +51,7 @@ so-idh:
     {%   endfor %}
     {% endif %}
     - watch:
+      - cmd: ca_trust_extract
       - file: opencanary_config
     - require:
       - file: opencanary_config

salt/influxdb/config.sls

@@ -41,8 +41,8 @@ influxdb_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://influxdb/tools/sbin
-    - user: 939
-    - group: 939
+    - user: root
+    - group: root
     - file_mode: 755
 
 #influxdb_sbin_jinja:

salt/influxdb/enabled.sls

@@ -6,11 +6,13 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   set PASSWORD = salt['pillar.get']('secrets:influx_pass') %}
 {%   set TOKEN = salt['pillar.get']('influxdb:token') %}
 
 include:
+  - ca.trust
   - influxdb.ssl
   - influxdb.config
   - influxdb.sostatus
@@ -38,6 +40,7 @@ so-influxdb:
         {% endfor %}
       {% endif %}
     - binds:
+      - {{ CATRUST.pem }}:/etc/ssl/certs/ca-certificates.crt:ro
       - /opt/so/log/influxdb/:/log:rw
       - /opt/so/conf/influxdb/config.yaml:/conf/config.yaml:ro
       - /opt/so/conf/influxdb/etc:/etc/influxdb2:rw
@@ -66,6 +69,7 @@ so-influxdb:
     {%   endfor %}
     {% endif %}
     - watch:
+      - cmd: ca_trust_extract
       - file: influxdbconf
       - x509: influxdb_key
       - x509: influxdb_crt
@@ -94,9 +98,11 @@ metrics_link_file:
       - docker_container: so-influxdb
 
 # Install cron job to determine size of influxdb for telegraf
+# telegraf reads this while the cron rewrites it, so write aside and rename rather than
+# truncating in place. tgraflogdir recurses ownership, so the temp file is chowned to match
 get_influxdb_size:
   cron.present:
-    - name: 'du -s -k /nsm/influxdb | cut -f1 > /opt/so/log/telegraf/influxdb_size.log 2>&1'
+    - name: 'du -s -k /nsm/influxdb | cut -f1 > /opt/so/log/telegraf/influxdb_size.log.tmp 2>&1; chown 939:939 /opt/so/log/telegraf/influxdb_size.log.tmp; mv -f /opt/so/log/telegraf/influxdb_size.log.tmp /opt/so/log/telegraf/influxdb_size.log'
     - identifier: get_influxdb_size
     - user: root
     - minute: '*/1'

salt/kafka/config.sls

@@ -21,6 +21,7 @@ kafka_user:
     - gid: 960
     - home: /opt/so/conf/kafka
     - createhome: False
+    - shell: /sbin/nologin
 
 kafka_home_dir:
   file.absent:
@@ -30,16 +31,16 @@ kafka_sbin_tools:
   file.recurse:
     - name: /usr/sbin
     - source: salt://kafka/tools/sbin
-    - user: 960
-    - group: 960
+    - user: root
+    - group: root
     - file_mode: 755
 
 kafka_sbin_jinja_tools:
   file.recurse:
     - name: /usr/sbin
     - source: salt://kafka/tools/sbin_jinja
-    - user: 960
-    - group: 960
+    - user: root
+    - group: root
     - file_mode: 755
     - template: jinja
     - defaults:

salt/kafka/enabled.sls

@@ -12,12 +12,14 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   set KAFKANODES = salt['pillar.get']('kafka:nodes') %}
 {%   set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %}
 {%   if 'gmd' in salt['pillar.get']('features', []) %}
 
 include:
+  - ca.trust
   - kafka.ca
   - kafka.config
   - kafka.ssl
@@ -51,6 +53,7 @@ so-kafka:
       - {{ BINDING }}
       {% endfor %}
     - binds:
+      - {{ CATRUST.java }}:/opt/java/openjdk/lib/security/cacerts:ro
       - /etc/pki/kafka.p12:/etc/pki/kafka.p12:ro
       - /opt/so/conf/kafka/kafka-truststore.jks:/etc/pki/kafka-truststore.jks:ro
       - /nsm/kafka/data/:/nsm/kafka/data/:rw
@@ -68,6 +71,7 @@ so-kafka:
     {%   endfor %}
     {% endif %}
     - watch:
+      - cmd: ca_trust_extract
       {% for sc in ['server', 'client'] %}
       - file: kafka_kraft_{{sc}}_properties
       {% endfor %}

salt/kibana/config.sls

@@ -22,6 +22,7 @@ kibana:
     - gid: 932
     - home: /opt/so/conf/kibana
     - createhome: False
+    - shell: /sbin/nologin
 
 # Drop the correct nginx config based on role
 
@@ -36,16 +37,16 @@ kibana_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://kibana/tools/sbin
-    - user: 932
-    - group: 939
+    - user: root
+    - group: root
     - file_mode: 755
 
 kibana_sbin_jinja:
   file.recurse:
     - name: /usr/sbin
     - source: salt://kibana/tools/sbin_jinja
-    - user: 932
-    - group: 939 
+    - user: root
+    - group: root
     - file_mode: 755
     - template: jinja
     - defaults:

salt/kibana/defaults.yaml

@@ -22,7 +22,7 @@ kibana:
           - default
           - file
     migrations:
-      discardCorruptObjects: "9.3.7"
+      discardCorruptObjects: "9.4.8"
     telemetry:
       enabled: False
     xpack:

salt/kibana/enabled.sls

@@ -8,8 +8,10 @@
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 
 include:
+  - ca.trust
   - kibana.config
   - kibana.sostatus
 
@@ -27,6 +29,7 @@ so-kibana:
       - ELASTICSEARCH_HOST={{ GLOBALS.manager }}
       - ELASTICSEARCH_PORT=9200
       - MANAGER={{ GLOBALS.manager }}
+      - NODE_OPTIONS=--use-openssl-ca
       {% if DOCKERMERGED.containers['so-kibana'].extra_env %}
         {% for XTRAENV in DOCKERMERGED.containers['so-kibana'].extra_env %}
       - {{ XTRAENV }}
@@ -40,6 +43,7 @@ so-kibana:
       {% endfor %}
     {% endif %}
     - binds:
+      - {{ CATRUST.dir }}
       - /opt/so/conf/kibana/etc:/usr/share/kibana/config:rw
       - /opt/so/log/kibana:/var/log/kibana:rw
       - /opt/so/conf/kibana/customdashboards:/usr/share/kibana/custdashboards:ro
@@ -60,6 +64,7 @@ so-kibana:
     {%   endfor %}
     {% endif %}
     - watch:
+      - cmd: ca_trust_extract
       - file: kibanaconfig
 
 wait_for_so-kibana:

salt/kibana/tools/sbin_jinja/so-kibana-space-defaults

@@ -9,5 +9,5 @@ SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http:
 # Disable certain Features from showing up in the Kibana UI
 echo
 echo "Setting up default Kibana Space:"
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV1","entityManager","streams","cloudConnect","slo"]} ' >> /opt/so/log/kibana/misc.log
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","securitySolutionCasesV3","inventory","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV4","securitySolutionAlertsV1","entityManager","slo","streams","anonymization","searchInferenceEndpoints","cloudConnect","queryActivity","automatic_import","stackAlerts","monitoring","dataQuality","actions"]} ' >> /opt/so/log/kibana/misc.log
 echo

salt/kratos/config.sls

@@ -27,6 +27,7 @@ kratos:
     - uid: 928
     - gid: 928
     - home: /opt/so/conf/kratos
+    - shell: /sbin/nologin
 
 kratosdir:
   file.directory:

salt/kratos/enabled.sls

@@ -7,8 +7,10 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 
 include:
+  - ca.trust
   - kratos.config
   - kratos.sostatus
 
@@ -19,9 +21,10 @@ so-kratos:
     - hostname: kratos
     - name: so-kratos
     - networks:
-      - sobridge:
-        - ipv4_address: {{ DOCKERMERGED.containers['so-kratos'].ip }}
+      - soauth:
+        - ipv4_address: {{ DOCKERMERGED.containers['so-kratos'].ips['soauth'] }}
     - binds:
+      - {{ CATRUST.dir }}
       - /opt/so/conf/kratos/:/kratos-conf:ro
       - /opt/so/log/kratos/:/kratos-log:rw
       - /nsm/kratos/db:/kratos-data:rw
@@ -53,6 +56,7 @@ so-kratos:
     {%   endfor %}
     {% endif %}
     - watch:
+      - cmd: ca_trust_extract
       - file: kratosschema
       - file: kratosconfig
     - require:
@@ -71,7 +75,7 @@ delete_so-kratos_so-status.disabled:
 
 wait_for_kratos:
   http.wait_for_successful_query:
-    - name: 'http://{{ GLOBALS.manager }}:4434/'
+    - name: 'http://{{ DOCKERMERGED.containers['so-kratos'].ips['soauth'] }}:4434/'
     - ssl: True
     - verify_ssl: False
     - status:

salt/libvirt/64962/init.sls

@@ -6,6 +6,8 @@ so-fix-salt-ldap_script:
   file.managed:
     - name: /usr/sbin/so-fix-salt-ldap.py
     - source: salt://libvirt/64962/scripts/so-fix-salt-ldap.py
+    - user: root
+    - group: root
     - mode: 744
 
 fix-salt-ldap:

salt/logrotate/defaults.yaml

@@ -220,6 +220,26 @@ logrotate:
       - extension .log
       - dateext
       - dateyesterday
+    /opt/so/log/salt/virtual_node_manager:
+      - daily
+      - rotate 14
+      - missingok
+      - copytruncate
+      - compress
+      - create
+      - extension .log
+      - dateext
+      - dateyesterday
+    /opt/so/log/salt/so-salt-cloud:
+      - daily
+      - rotate 14
+      - missingok
+      - copytruncate
+      - compress
+      - create
+      - extension .log
+      - dateext
+      - dateyesterday
     /opt/so/log/salt/so-soup-grid-highstate:
       - daily
       - rotate 14

salt/logrotate/soc_logrotate.yaml

@@ -140,6 +140,20 @@ logrotate:
       multiline: True
       global: True
       forcedType: "[]string"
+    "/opt/so/log/salt/virtual_node_manager":
+      description: List of logrotate options for this file.
+      title: /opt/so/log/salt/virtual_node_manager
+      advanced: True
+      multiline: True
+      global: True
+      forcedType: "[]string"
+    "/opt/so/log/salt/so-salt-cloud":
+      description: List of logrotate options for this file.
+      title: /opt/so/log/salt/so-salt-cloud
+      advanced: True
+      multiline: True
+      global: True
+      forcedType: "[]string"
     "/opt/so/log/salt/so-soup-grid-highstate":
       description: List of logrotate options for this file.
       title: /opt/so/log/salt/so-soup-grid-highstate

salt/logstash/config.sls

@@ -35,13 +35,14 @@ logstash:
     - uid: 931
     - gid: 931
     - home: /opt/so/conf/logstash
+    - shell: /sbin/nologin
 
 logstash_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://logstash/tools/sbin
-    - user: 931
-    - group: 939
+    - user: root
+    - group: root
     - file_mode: 755
 
 #logstash_sbin_jinja:

salt/logstash/enabled.sls

@@ -6,6 +6,7 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'logstash/map.jinja' import LOGSTASH_MERGED %}
 {%   from 'logstash/map.jinja' import LOGSTASH_NODES %}
@@ -13,9 +14,6 @@
 
 include:
   - ca
-{%   if GLOBALS.role not in ['so-receiver','so-fleet'] %}
-  - elasticsearch.ca
-{%   endif %}
 {# Kafka ca runs on nodes that can run logstash for Kafka input / output. Only when Kafka is global pipeline #}
 {%   if GLOBALS.role in ['so-searchnode', 'so-manager', 'so-managerhype', 'so-managersearch', 'so-receiver', 'so-standalone'] and GLOBALS.pipeline == 'KAFKA' %}
   - kafka.ca
@@ -58,6 +56,8 @@ so-logstash:
       - {{ BINDING }}
       {% endfor %}
     - binds:
+      - {{ CATRUST.dir }}
+      - {{ CATRUST.java }}:/usr/share/logstash/jdk/lib/security/cacerts:ro
       - /opt/so/conf/elasticsearch/templates/:/templates/:ro
       - /opt/so/conf/logstash/etc/:/usr/share/logstash/config/:ro
       - /opt/so/conf/logstash/pipelines:/usr/share/logstash/pipelines:ro
@@ -78,10 +78,6 @@ so-logstash:
       - /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro
           {% endif %}
       {% endif %}
-      {% if GLOBALS.role not in ['so-receiver','so-fleet'] %}
-      - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro
-      - /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro
-      {% endif %}
       {% if GLOBALS.pipeline == "KAFKA" and GLOBALS.role in ['so-manager', 'so-managerhype', 'so-managersearch', 'so-standalone', 'so-searchnode'] %}
       - /etc/pki/kafka-logstash.p12:/usr/share/logstash/kafka-logstash.p12:ro
       - /opt/so/conf/kafka/kafka-truststore.jks:/etc/pki/kafka-truststore.jks:ro
@@ -104,14 +100,11 @@ so-logstash:
     {%   endfor %}
     {% endif %}
     - watch:
+      - cmd: ca_trust_extract
       - file: lsetcsync
       - file: lslog4j2
       - file: lspipelinesyml
       - file: trusttheca
-      {% if GLOBALS.is_manager %}
-      - file: elasticsearch_cacerts
-      - file: elasticsearch_capems
-      {% endif %}
       {% if GLOBALS.is_manager or GLOBALS.role in ['so-fleet', 'so-heavynode', 'so-receiver'] %}
       - x509: etc_elasticfleet_logstash_crt
       - x509: etc_elasticfleet_logstash_key
@@ -133,10 +126,6 @@ so-logstash:
       {% endif %}
     - require:
       - file: trusttheca
-      {% if GLOBALS.is_manager %}
-      - file: elasticsearch_cacerts
-      - file: elasticsearch_capems
-      {% endif %}
       {% if GLOBALS.is_manager or GLOBALS.role in ['so-fleet', 'so-heavynode', 'so-receiver'] %}
       - x509: etc_elasticfleet_logstash_crt
       - x509: etc_elasticfleet_logstash_key

salt/manager/files/beacons_pushstate.conf.jinja

@@ -3,9 +3,16 @@ beacons:
   postgres_pillar_beacon:
     - interval: {{ AUTOAPPLY.drain_interval }}
     - disable_during_state_run: False
-  rules_beacon:
+  local_files_beacon:
     - interval: {{ AUTOAPPLY.drain_interval }}
     - disable_during_state_run: False
+    # Tags are app names in salt/reactor/pillar_push_map.yaml.
+    # Allowlist on purpose: salt writes elsewhere under local/salt/ and would self-retrigger.
     - paths:
         /opt/so/saltstack/local/salt/suricata/rules: suricata
         /opt/so/saltstack/local/salt/strelka/rules/compiled: strelka
+        /opt/so/saltstack/local/salt/zeek/policy: zeek
+        /opt/so/saltstack/local/salt/zeek/zkg: zeek
+        /opt/so/saltstack/local/salt/elasticsearch/files/ingest: elasticsearch
+        /opt/so/saltstack/local/salt/elasticsearch/roles: elasticsearch
+        /opt/so/saltstack/local/salt/logstash/pipelines/config/custom: logstash

salt/manager/init.sls

@@ -113,8 +113,8 @@ manager_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://manager/tools/sbin
-    - user: 939
-    - group: 939
+    - user: root
+    - group: root
     - file_mode: 755
     - exclude_pat:
       - "*_test.py"
@@ -124,8 +124,8 @@ manager_sbin_jinja:
   file.recurse:
     - name: /usr/sbin/
     - source: salt://manager/tools/sbin_jinja/
-    - user: socore
-    - group: socore
+    - user: root
+    - group: root
     - file_mode: 755
     - template: jinja
     - show_changes: False
@@ -166,7 +166,7 @@ so-repo-sync:
 
 so_fleetagent_status:
   cron.present:
-    - name: /usr/sbin/so-elasticagent-status > /opt/so/log/agents/agentstatus.log 2>&1
+    - name: '/usr/sbin/so-elasticagent-status > /opt/so/log/agents/agentstatus.log.tmp 2>&1; mv -f /opt/so/log/agents/agentstatus.log.tmp /opt/so/log/agents/agentstatus.log'
     - identifier: so_fleetagent_status
     - user: root
     - minute: '*/5'
@@ -190,11 +190,15 @@ so_fleetagent_monitor:
   - month: '*'
   - dayweek: '*'
 
-socore_own_saltstack_default:
+# This tree is the source of every root-executed script (/usr/sbin, reactors, _runners,
+# engines, salt-relay.sh). SOC mounts /opt/so/saltstack rw as uid 939 but only writes
+# under local/. Do not add dir_mode/file_mode here -- SOC reads default/ and 750/640
+# would break its config load.
+root_own_saltstack_default:
   file.directory:
     - name: /opt/so/saltstack/default
-    - user: socore
-    - group: socore
+    - user: root
+    - group: root
     - recurse:
       - user
       - group

salt/manager/tools/sbin/so-client

@@ -106,7 +106,8 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
-hydraUrl=${HYDRA_URL:-http://127.0.0.1:4445}
+hydraContainer=${HYDRA_CONTAINER:-so-hydra}
+hydraUrl=${HYDRA_URL:-http://localhost:4445}
 socRolesFile=${SOC_ROLES_FILE:-/opt/so/conf/soc/soc_clients_roles}
 soUID=${SOCORE_UID:-939}
 soGID=${SOCORE_GID:-939}
@@ -124,6 +125,10 @@ function fail() {
   exit 1
 }
 
+function hydraCurl() {
+  docker exec "$hydraContainer" curl "$@"
+}
+
 function require() {
   cmd=$1
   which "$1" 2>&1 > /dev/null
@@ -133,8 +138,8 @@ function require() {
 # Verify this environment is capable of running this script
 function verifyEnvironment() {
   require "jq"
-  require "curl"
-  response=$(curl -Ss -L ${hydraUrl}/health/alive)
+  require "docker"
+  response=$(hydraCurl -Ss -L ${hydraUrl}/health/alive)
   [[ "$response" != '{"status":"ok"}' ]] && fail "Unable to communicate with Hydra; specify URL via HYDRA_URL environment variable"
 }
 
@@ -164,7 +169,7 @@ function ensureRoleFileExists() {
 }
 
 function listClients() {
-  response=$(curl -Ss -L -f ${hydraUrl}/admin/clients)
+  response=$(hydraCurl -Ss -L -f ${hydraUrl}/admin/clients)
   [[ $? != 0 ]] && fail "Unable to communicate with Hydra"
 
   clientIds=$(echo "${response}" | jq -r ".[] | .client_id" | sort)
@@ -251,7 +256,7 @@ function createClient() {
 EOF
   )
   
-  response=$(curl -Ss -L --fail-with-body -X POST ${hydraUrl}/admin/clients -d "$body")
+  response=$(hydraCurl -Ss -L --fail-with-body -X POST ${hydraUrl}/admin/clients -d "$body")
   if [[ $? != 0 ]]; then
     error=$(echo $response | jq .error)
     fail "Failed to submit request to Hydra: $error"
@@ -283,7 +288,7 @@ function update() {
 EOF
   )
   
-  response=$(curl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body")
+  response=$(hydraCurl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body")
   if [[ $? != 0 ]]; then
     error=$(echo $response | jq .error)
     fail "Failed to submit request to Hydra: $error"
@@ -305,7 +310,7 @@ function generateSecret() {
 EOF
   )
   
-  response=$(curl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body")
+  response=$(hydraCurl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body")
   if [[ $? != 0 ]]; then
     error=$(echo $response | jq .error)
     fail "Failed to submit request to Hydra: $error"
@@ -317,7 +322,7 @@ function deleteClient() {
 
   [[ ${identityId} == "" ]] && fail "Client not found"
 
-  response=$(curl -Ss -XDELETE -L --fail-with-body "${hydraUrl}/admin/clients/$identityId")
+  response=$(hydraCurl -Ss -XDELETE -L --fail-with-body "${hydraUrl}/admin/clients/$identityId")
   if [[ $? != 0 ]]; then
     error=$(echo $response | jq .error)
     fail "Failed to submit request to Hydra: $error"

salt/manager/tools/sbin/so-minion

@@ -121,8 +121,14 @@ for i in "$@"; do
 	esac
 done
 
-PILLARFILE=/opt/so/saltstack/local/pillar/minions/$MINION_ID.sls
-ADVPILLARFILE=/opt/so/saltstack/local/pillar/minions/adv_$MINION_ID.sls
+if [[ -n "$MINION_ID" && ! "$MINION_ID" =~ ^[A-Za-z0-9._-]{1,253}$ ]]; then
+	echo "Invalid minion id: $MINION_ID"
+	log "ERROR" "Invalid minion id: $MINION_ID"
+	exit 1
+fi
+
+readonly PILLARFILE=/opt/so/saltstack/local/pillar/minions/$MINION_ID.sls
+readonly ADVPILLARFILE=/opt/so/saltstack/local/pillar/minions/adv_$MINION_ID.sls
 
 function getinstallinfo() {
 	log "INFO" "Getting install info for minion $MINION_ID"
@@ -133,10 +139,23 @@ function getinstallinfo() {
 		return 1
 	fi
 
-	while read -r var; do export "$var"; done <<< "$INSTALLVARS"
-	if [ $? -ne 0 ]; then
-		log "ERROR" "Failed to source install variables"
-		return 1
+	# install.txt is controlled by the minion; only accept known keys and never eval or export them
+	local line key
+	while IFS= read -r line; do
+		[[ "$line" == *=* ]] || continue
+		key=${line%%=*}
+		case "$key" in
+			MAINIP|MNIC|NODE_DESCRIPTION|ES_HEAP_SIZE|PATCHSCHEDULENAME|INTERFACE|NODETYPE|CORECOUNT|LSHOSTNAME|LSHEAP|CPUCORES|IDH_MGTRESTRICT|IDH_SERVICES)
+				printf -v "$key" '%s' "${line#*=}"
+				;;
+			*)
+				log "WARN" "Ignoring unexpected install var from $MINION_ID: ${key:0:64}"
+				;;
+		esac
+	done <<< "$INSTALLVARS"
+
+	if [[ "$NODE_DESCRIPTION" == \'*\' ]]; then
+		NODE_DESCRIPTION=${NODE_DESCRIPTION:1:-1}
 	fi
 
 	log "INFO" "Fetched install info for $MINION_ID (node type: ${NODETYPE:-unset})"
@@ -176,6 +195,12 @@ function pcapspace() {
 		fi
 	fi
 
+	# Must be checked before arithmetic expansion, which evaluates array subscripts
+	if [[ ! "$SPACESIZE" =~ ^[0-9]+$ ]]; then
+		log "ERROR" "Invalid disk size for $MINION_ID: ${SPACESIZE:0:64}"
+		return 1
+	fi
+
 	local s=$(( $SPACESIZE / 1000000 ))
 	local s1=$(( $s / 4 * $PCAP_PERCENTAGE ))
 
@@ -1050,6 +1075,57 @@ function updateMineAndApplyStates() {
 	fi
 }
 
+# Values end up in a Jinja-rendered pillar and in bash, and may come from the minion
+function validate_minion_vars() {
+	local error_msg=""
+	# Inline rather than valid_ip4: so-common is not installed yet when setup runs -o=setup
+	local octet='(25[0-5]|2[0-4][0-9]|1?[0-9]?[0-9])'
+	local ip4_re="^($octet\.){3}$octet$"
+
+	case "$NODETYPE" in
+		EVAL|STANDALONE|MANAGER|MANAGERSEARCH|MANAGERHYPE|IMPORT)
+			# Manager pillars also rewrite the CA pillar, so never accept them from a remote node
+			[[ "$OPERATION" == "setup" ]] || error_msg="Node type $NODETYPE can only be configured during setup"
+			;;
+		FLEET|IDH|HEAVYNODE|SENSOR|SEARCHNODE|RECEIVER|HYPERVISOR|DESKTOP)
+			;;
+		*)
+			error_msg="Invalid node type: ${NODETYPE:0:64}"
+			;;
+	esac
+
+	if [[ -z "$error_msg" ]]; then
+		if [[ ! "$MAINIP" =~ $ip4_re ]]; then
+			error_msg="Invalid MAINIP: ${MAINIP:0:64}"
+		elif [[ ! "$MNIC" =~ ^[A-Za-z0-9._-]*$ ]]; then
+			error_msg="Invalid MNIC: ${MNIC:0:64}"
+		elif [[ ! "$INTERFACE" =~ ^[A-Za-z0-9._-]*$ ]]; then
+			error_msg="Invalid INTERFACE: ${INTERFACE:0:64}"
+		elif [[ ! "$LSHOSTNAME" =~ ^[A-Za-z0-9._-]*$ ]]; then
+			error_msg="Invalid LSHOSTNAME: ${LSHOSTNAME:0:64}"
+		elif [[ ! "$ES_HEAP_SIZE" =~ ^([0-9]+[kKmMgG]?)?$ ]]; then
+			error_msg="Invalid ES_HEAP_SIZE: ${ES_HEAP_SIZE:0:64}"
+		elif [[ ! "$LSHEAP" =~ ^([0-9]+[kKmMgG]?)?$ ]]; then
+			error_msg="Invalid LSHEAP: ${LSHEAP:0:64}"
+		elif [[ ! "$CORECOUNT" =~ ^[0-9]*$ ]]; then
+			error_msg="Invalid CORECOUNT: ${CORECOUNT:0:64}"
+		elif [[ ! "$CPUCORES" =~ ^[0-9]*$ ]]; then
+			error_msg="Invalid CPUCORES: ${CPUCORES:0:64}"
+		elif [[ ! "$IDH_MGTRESTRICT" =~ ^(True|False)?$ ]]; then
+			error_msg="Invalid IDH_MGTRESTRICT: ${IDH_MGTRESTRICT:0:64}"
+		fi
+	fi
+
+	if [[ -n "$error_msg" ]]; then
+		log "ERROR" "$error_msg"
+		echo "$error_msg"
+		return 1
+	fi
+
+	# Free text; removing braces is enough to prevent any Jinja delimiter
+	NODE_DESCRIPTION=${NODE_DESCRIPTION//[\{\}[:cntrl:]]/}
+}
+
 function setupMinionFiles() {
 	log "INFO" "Setting up minion files for $MINION_ID (pillar: $PILLARFILE)"
 	
@@ -1061,6 +1137,8 @@ function setupMinionFiles() {
 		return 1
 	fi
 
+	validate_minion_vars || return 1
+
 	# Create the base minion files
 	create_minion_files || return 1
 

salt/manager/tools/sbin/so-push-drainer

@@ -20,7 +20,7 @@ is older than debounce_seconds, this script:
     with the deduped actions list passed as pillar kwargs
   * deletes the contributed intent files on successful dispatch
 
-Reactor sls files (push_suricata, push_strelka, push_pillar) write intents
+Reactor sls files (push_files, push_pillar) write intents
 but never dispatch directly
 """
 

salt/manager/tools/sbin/so-saltstack-update

@@ -124,8 +124,8 @@ copy_new_files() {
   
   rsync -a salt $default_salt_dir/
   rsync -a pillar $default_salt_dir/
-  chown -R socore:socore $default_salt_dir/salt
-  chown -R socore:socore $default_salt_dir/pillar
+  chown -R root:root $default_salt_dir/salt
+  chown -R root:root $default_salt_dir/pillar
   chmod 755 $default_salt_dir/pillar/firewall/addfirewall.sh
 
   rm -rf /tmp/sogh

salt/manager/tools/sbin/so-user

@@ -129,7 +129,8 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
-kratosUrl=${KRATOS_URL:-http://127.0.0.1:4434/admin}
+kratosContainer=${KRATOS_CONTAINER:-so-kratos}
+kratosUrl=${KRATOS_URL:-http://localhost:4434/admin}
 databasePath=${KRATOS_DB_PATH:-/nsm/kratos/db/db.sqlite}
 databaseTimeout=${KRATOS_DB_TIMEOUT:-5000}
 bcryptRounds=${BCRYPT_ROUNDS:-12}
@@ -154,6 +155,10 @@ function fail() {
   exit 1
 }
 
+function kratosCurl() {
+  docker exec "$kratosContainer" curl "$@"
+}
+
 function require() {
   cmd=$1
   which "$1" 2>&1 > /dev/null
@@ -164,18 +169,18 @@ function require() {
 function verifyEnvironment() {
   require "htpasswd"
   require "jq"
-  require "curl"
+  require "docker"
   require "openssl"
   require "sqlite3"
   [[ ! -f $databasePath ]] && fail "Unable to find database file; specify path via KRATOS_DB_PATH environment variable"
-  response=$(curl -Ss -L ${kratosUrl}/)
+  response=$(kratosCurl -Ss -L ${kratosUrl}/)
   [[ "$response" != "404 page not found" ]] && fail "Unable to communicate with Kratos; specify URL via KRATOS_URL environment variable"
 }
 
 function findIdByEmail() {
   email=${1,,}
 
-  response=$(curl -Ss -L ${kratosUrl}/identities)
+  response=$(kratosCurl -Ss -L ${kratosUrl}/identities)
   identityId=$(echo "${response}" | jq -r ".[] | select(.verifiable_addresses[0].value == \"$email\") | .id")
   echo $identityId
 }
@@ -416,7 +421,7 @@ function syncAll() {
 }
 
 function listUsers() {
-  response=$(curl -Ss -L ${kratosUrl}/identities)
+  response=$(kratosCurl -Ss -L ${kratosUrl}/identities)
   [[ $? != 0 ]] && fail "Unable to communicate with Kratos"
 
   users=$(echo "${response}" | jq -r ".[] | .verifiable_addresses[0].value" | sort)
@@ -495,7 +500,7 @@ function createUser() {
 EOF
   )
   
-  response=$(curl -Ss -L ${kratosUrl}/identities -d "$addUserJson")
+  response=$(kratosCurl -Ss -L ${kratosUrl}/identities -d "$addUserJson")
   [[ $? != 0 ]] && fail "Unable to communicate with Kratos"
 
   identityId=$(echo "${response}" | jq -r ".id")
@@ -518,7 +523,7 @@ function updateStatus() {
   identityId=$(findIdByEmail "$email")
   [[ ${identityId} == "" ]] && fail "User not found"
 
-  response=$(curl -Ss -L "${kratosUrl}/identities/$identityId")
+  response=$(kratosCurl -Ss -L "${kratosUrl}/identities/$identityId")
   [[ $? != 0 ]] && fail "Unable to communicate with Kratos"
 
   schemaId=$(echo "$response" | jq -r .schema_id)
@@ -531,7 +536,7 @@ function updateStatus() {
     state="inactive"
   fi
   body="{ \"schema_id\": \"$schemaId\", \"state\": \"$state\", \"traits\": $traitBlock }"
-  response=$(curl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body")
+  response=$(kratosCurl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body")
   [[ $? != 0 ]] && fail "Unable to update user"
 }
 
@@ -550,7 +555,7 @@ function updateUserProfile() {
   identityId=$(findIdByEmail "$email")
   [[ ${identityId} == "" ]] && fail "User not found"
 
-  response=$(curl -Ss -L "${kratosUrl}/identities/$identityId")
+  response=$(kratosCurl -Ss -L "${kratosUrl}/identities/$identityId")
   [[ $? != 0 ]] && fail "Unable to communicate with Kratos"
 
   schemaId=$(echo "$response" | jq -r .schema_id)
@@ -559,7 +564,7 @@ function updateUserProfile() {
   traitBlock="{\"email\":\"$email\",\"firstName\":\"$firstName\",\"lastName\":\"$lastName\",\"note\":\"$note\"}"
 
   body="{ \"schema_id\": \"$schemaId\", \"state\": \"$state\", \"traits\": $traitBlock }"
-  response=$(curl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body")
+  response=$(kratosCurl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body")
   [[ $? != 0 ]] && fail "Unable to update user"
 }
 
@@ -569,7 +574,7 @@ function deleteUser() {
   identityId=$(findIdByEmail "$email")
   [[ ${identityId} == "" ]] && fail "User not found"
 
-  response=$(curl -Ss -XDELETE -L "${kratosUrl}/identities/$identityId")
+  response=$(kratosCurl -Ss -XDELETE -L "${kratosUrl}/identities/$identityId")
   [[ $? != 0 ]] && fail "Unable to communicate with Kratos"
 
   rolesTmpFile="${socRolesFile}.tmp"

salt/manager/tools/sbin/so-yaml.py

@@ -42,7 +42,8 @@ def loadYaml(filename):
     try:
         with open(filename, "r") as file:
             content = file.read()
-            return yaml.safe_load(content)
+            loaded = yaml.safe_load(content)
+            return loaded if loaded is not None else {}
     except FileNotFoundError:
         print(f"File not found: {filename}", file=sys.stderr)
         sys.exit(1)

salt/manager/tools/sbin/so-yaml_test.py

@@ -95,6 +95,20 @@ class TestRemove(unittest.TestCase):
         expected = "key1:\n  child1: 123\n  child2:\n    deep2: ab\nkey2: false\n"
         self.assertEqual(actual, expected)
 
+    def test_remove_empty_file(self):
+        filename = "/tmp/so-yaml_test-remove-empty.yaml"
+        file = open(filename, "w")
+        file.close()
+
+        code = soyaml.remove([filename, "key1"])
+        self.assertEqual(code, 0)
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        self.assertEqual(actual, "{}\n")
+
     def test_remove_missing_args(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
             with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -294,6 +308,36 @@ class TestRemove(unittest.TestCase):
         expected = "key1:\n  child1: 123\n  child2:\n    deep1: 45\n    deep2: d\nkey2: false\nkey3:\n- e\n- f\n- g\n"
         self.assertEqual(actual, expected)
 
+    def test_add_empty_file(self):
+        filename = "/tmp/so-yaml_test-add-empty.yaml"
+        file = open(filename, "w")
+        file.close()
+
+        code = soyaml.add([filename, "telegraf.output", "BOTH"])
+        self.assertEqual(code, 0)
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "telegraf:\n  output: BOTH\n"
+        self.assertEqual(actual, expected)
+
+    def test_add_empty_file_simple(self):
+        filename = "/tmp/so-yaml_test-add-empty-simple.yaml"
+        file = open(filename, "w")
+        file.close()
+
+        code = soyaml.add([filename, "telegraf", "BOTH"])
+        self.assertEqual(code, 0)
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "telegraf: BOTH\n"
+        self.assertEqual(actual, expected)
+
     def test_replace_missing_arg(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
             with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -346,6 +390,21 @@ class TestRemove(unittest.TestCase):
         expected = "key1:\n  child1: 123\n  child2:\n    deep1: 46\nkey2: false\nkey3:\n- e\n- f\n- g\n"
         self.assertEqual(actual, expected)
 
+    def test_replace_empty_file(self):
+        filename = "/tmp/so-yaml_test-replace-empty.yaml"
+        file = open(filename, "w")
+        file.close()
+
+        code = soyaml.replace([filename, "telegraf.output", "BOTH"])
+        self.assertEqual(code, 0)
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "telegraf:\n  output: BOTH\n"
+        self.assertEqual(actual, expected)
+
     def test_convert(self):
         self.assertEqual(soyaml.convertType("foo"), "foo")
         self.assertEqual(soyaml.convertType("foo.bar"), "foo.bar")
@@ -506,6 +565,18 @@ class TestRemove(unittest.TestCase):
             self.assertEqual(result, 2)
             self.assertEqual("", mock_stdout.getvalue())
 
+    def test_get_empty_file(self):
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                filename = "/tmp/so-yaml_test-get-empty.yaml"
+                file = open(filename, "w")
+                file.close()
+
+                result = soyaml.get([filename, "telegraf.output"])
+                self.assertEqual(result, 2)
+                self.assertEqual("", mock_stdout.getvalue())
+                self.assertIn("Key 'telegraf.output' not found by so-yaml.py", mock_stderr.getvalue())
+
     def test_get_usage(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
             with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -991,3 +1062,29 @@ class TestLoadYaml(unittest.TestCase):
                     soyaml.loadYaml("/tmp/so-yaml_test-unreadable.yaml")
                     sysmock.assert_called_with(1)
                     self.assertIn("Error reading file", mock_stderr.getvalue())
+
+    def test_load_yaml_empty_file(self):
+        filename = "/tmp/so-yaml_test-load-empty.yaml"
+        file = open(filename, "w")
+        file.close()
+
+        result = soyaml.loadYaml(filename)
+        self.assertEqual(result, {})
+
+    def test_load_yaml_whitespace_only(self):
+        filename = "/tmp/so-yaml_test-load-whitespace.yaml"
+        file = open(filename, "w")
+        file.write("   \n\n  \n")
+        file.close()
+
+        result = soyaml.loadYaml(filename)
+        self.assertEqual(result, {})
+
+    def test_load_yaml_comments_only(self):
+        filename = "/tmp/so-yaml_test-load-comments.yaml"
+        file = open(filename, "w")
+        file.write("# Just a comment\n# Another comment\n")
+        file.close()
+
+        result = soyaml.loadYaml(filename)
+        self.assertEqual(result, {})

salt/manager/tools/sbin/soup

@@ -28,6 +28,7 @@ INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
 # percentage like "25%"). Empty means so-soup-grid-highstate uses the salt:auto_apply:batch
 # pillar default.
 BATCHSIZE=
+DEFAULT_DOCKER_RANGE='172.17.1.0/24'
 SOUP_LOG=/root/soup.log
 SOUP_DEBUG_LOG=/root/soup-debug.log
 WHATWOULDYOUSAYYAHDOHERE=soup
@@ -120,6 +121,9 @@ check_err() {
       161)
         echo 'Required intermediate Elasticsearch upgrade not complete'
       ;;
+      162)
+        echo 'One or more Elastic Agent nodes do not support the x86-64-v3 CPU instruction set'
+      ;;
       170)
         echo "Intermediate upgrade completed successfully to $next_step_so_version, but next soup to Security Onion $originally_requested_so_version could not be started automatically."
         echo "Start soup again manually to continue the upgrade to Security Onion $originally_requested_so_version."
@@ -331,29 +335,101 @@ check_pillar_items() {
 }
 
 check_cluster_health() {
+  # Require a 'green' cluster before upgrading
   echo "Checking Elasticsearch cluster health."
-  # Require a 'green' cluster before upgrading; anything less (yellow, red, or
-  # unreachable) blocks. Modeled on the wait used in so-elasticsearch-roles-load.
   if so-elasticsearch-query "_cluster/health?wait_for_status=green&timeout=120s" --fail > /dev/null 2>&1; then
     printf "\nThe Elasticsearch cluster is healthy (green). We can proceed with SOUP.\n\n"
+    return
+  fi
+
+  if command -v so-elasticsearch-troubleshoot > /dev/null 2>&1; then
+    printf "\nRunning so-elasticsearch-troubleshoot for additional detail.\n"
+    so-elasticsearch-troubleshoot || true
+  fi
+
+  printf "\nThe Elasticsearch cluster is not green. Resolve the cluster health issue so the cluster is green before running SOUP again.\n\n"
+  exit 0
+}
+
+no_soup_for_you() {
+  echo ""
+  echo "No soup for you!"
+  exit 162
+}
+
+check_cpu_compatibility() {
+  # Roles running a container built from the so-elastic-agent image; mirrors the
+  # elasticagent and elasticfleet entries in salt/reactor/pillar_push_map.yaml.
+  local cpu_target='G@role:so-heavynode or G@role:so-eval or G@role:so-fleet or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone'
+  local expected_nodes cpu_results node result confirm
+  local -a unsupported=() offline=()
+
+  echo "Checking that Elastic Agent nodes support the x86-64-v3 CPU instruction set now required by Elastic."
+
+  if [[ "$SKIP_CPU_CHECK" == "true" ]]; then
+    printf "\nSkipping the x86-64-v3 CPU check because --skip-cpu-check was specified.\n\n"
+    return
+  fi
+
+  # Nodes that never answer are absent from the results, so diff against who should have.
+  expected_nodes=$(salt -C "$cpu_target" --preview-target --out=json 2>/dev/null | jq -r '.[]?') || true
+  if [[ -z "$expected_nodes" ]]; then
+    printf "\nCould not determine which nodes run the Elastic Agent, so the x86-64-v3 CPU check cannot run.\n"
+    no_soup_for_you
+  fi
+
+  cpu_results=$(salt -t 30 -C "$cpu_target" cmd.run "/lib64/ld-linux-x86-64.so.2 --help | grep x86-64-v3" --out=json 2>/dev/null) || true
+
+  while IFS= read -r node; do
+    [[ -z "$node" ]] && continue
+    result=$(jq -r --arg node "$node" '.[$node] // empty' <<< "$cpu_results" 2>/dev/null)
+    if [[ -z "$result" || "$result" == *"did not return"* ]]; then
+      offline+=("$node")
+    elif [[ "$result" != *"x86-64-v3 (supported"* ]]; then
+      # glibc appends "(supported, searched)" only when supported; the open paren keeps
+      # this from matching a future "(unsupported".
+      unsupported+=("$node")
+    fi
+  done <<< "$expected_nodes"
+
+  if [[ ${#unsupported[@]} -eq 0 && ${#offline[@]} -eq 0 ]]; then
+    printf "\nAll Elastic Agent nodes support x86-64-v3. We can proceed with SOUP.\n\n"
+    return
+  fi
+
+  echo ""
+  if [[ ${#unsupported[@]} -gt 0 ]]; then
+    echo "The following node(s) do NOT support the x86-64-v3 CPU instruction set:"
+    printf '  %s\n' "${unsupported[@]}"
+    echo ""
+    echo "Upstream Elastic now builds its binaries for x86-64-v3, so these nodes can no"
+    echo "longer run Elastic. Upgrading them WILL BREAK them."
+    echo ""
+  fi
+  if [[ ${#offline[@]} -gt 0 ]]; then
+    echo "The following node(s) did not respond and could not be checked:"
+    printf '  %s\n' "${offline[@]}"
+    echo ""
+    echo "These nodes are offline, so we cannot confirm they support x86-64-v3, which"
+    echo "upstream Elastic now requires."
+    echo ""
+  fi
+
+  if [[ -n $UNATTENDED ]]; then
+    echo "Unattended mode cannot prompt for an override. Re-run soup interactively, or pass --skip-cpu-check to bypass this check."
+    no_soup_for_you
+  fi
+
+  read -rp "Type 'override' to continue anyway, or press Enter to exit: " confirm
+  if [[ "${confirm,,}" == "override" ]]; then
+    printf "\nOverride accepted. Continuing at your own risk.\n\n"
   else
-    printf "\nThe Elasticsearch cluster is not green. Please resolve the cluster health issue so the cluster is green before running SOUP again.\n\n"
-    exit 0
+    no_soup_for_you
   fi
 }
 
 check_fleet_server() {
   echo "Checking that Elastic Fleet Server is responding."
-  # Before checking fleet health, check for and fix known issue with elastic-agent container and fs.protected_symlinks
-  local protected_symlinks=$(sysctl -b fs.protected_symlinks)
-  if [[ "$protected_symlinks" == "1" ]]; then
-    # disable fs.protected_symlinks and restart elasticfleet
-    sysctl -w fs.protected_symlinks=0
-    docker stop so-elastic-fleet; docker rm -f so-elastic-fleet
-    printf "\nUpdated sysctl fs.protected_symlinks. Restarting fleet before running health check and continuing with soup.\n"
-    salt-call state.apply elasticfleet queue=True
-  fi
-  
   # Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls,
   # which waits for HTTP 200 from the Fleet Server status API.
   if curl -sk --fail --retry 3 --retry-delay 10 --max-time 30 "https://localhost:8220/api/status" > /dev/null 2>&1; then
@@ -465,6 +541,19 @@ highstate() {
   salt-call state.highstate -l info queue=True
 }
 
+upgrade_searchnode_elasticsearch() {
+  # Run the elasticsearch state across the true elastic cluster (non-heavy) with a retry attempt
+  #   Excludes the manager, so that kibana & elasticfleet are not upgraded until searchnodes are upgraded.
+  echo "Getting ready to upgrade Elasticsearch across the grid. This may take a while..."
+  if salt -b 10% -C "I@elasticsearch:enabled and not G@role:so-heavynode and not G@id:${MINIONID}" state.apply elasticsearch queue=True; then
+    return 0
+  fi
+  
+  echo "Initial elasticsearch state attempt had a problem; retrying in 30 seconds."
+  sleep 30
+  salt -b 10% -C "I@elasticsearch:enabled and not G@role:so-heavynode and not G@id:${MINIONID}" state.apply elasticsearch queue=True
+}
+
 push_grid_highstate() {
   # Drive a batched, role-tiered highstate across the rest of the grid so remote minions
   # pick up this upgrade now instead of waiting up to ~2.5 hours for their own scheduled
@@ -493,11 +582,10 @@ push_grid_highstate() {
 masterlock() {
   echo "Locking Salt Master"
   mv -v $TOPFILE $BACKUPTOPFILE
-  # Render the real top file only for the host running soup; every other
-  # minion gets an empty top (no states) while the master is upgrading.
-  echo "{% if grains['id'] == '$MINIONID' %}" > $TOPFILE
-  cat $BACKUPTOPFILE >> $TOPFILE
-  echo "{% endif %}" >> $TOPFILE
+  echo "base:" > $TOPFILE
+  echo "  $MINIONID:" >> $TOPFILE
+  echo "    - ca" >> $TOPFILE
+  echo "    - elasticsearch" >> $TOPFILE
 }
 
 masterunlock() {
@@ -518,6 +606,7 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == "3.0.0" ]] && up_to_3.1.0
     [[ "$INSTALLEDVERSION" == "3.1.0" ]] && up_to_3.2.0
     [[ "$INSTALLEDVERSION" == "3.2.0" ]] && up_to_3.3.0
+    [[ "$INSTALLEDVERSION" == "3.3.0" ]] && up_to_3.4.0
     true
 }
 
@@ -536,6 +625,7 @@ postupgrade_changes() {
     [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0
     [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0
     [[ "$POSTVERSION" == "3.2.0" ]] && post_to_3.3.0
+    [[ "$POSTVERSION" == "3.3.0" ]] && post_to_3.4.0
     # All applicable post-upgrade steps completed; clear the resume marker.
     rm -f "$POSTVERSION_FILE"
     true
@@ -1013,9 +1103,6 @@ update_kafka_metadata() {
 up_to_3.2.0() {
   ensure_salt_local_pillar
 
-  # download 9.3.7 elastic agent packages
-  determine_elastic_agent_upgrade
-
   fix_logstash_0013_lumberjack_pipeline_name
 
   pin_elasticsearch_data_retention_method
@@ -1034,10 +1121,6 @@ post_to_3.2.0() {
   # SOC database bootstrap is handled by the postgres.enabled highstate.
   scrub_postgres_log_passwords
 
-  # Generate 9.3.7 elastic agent installers
-  echo "Regenerating Elastic Agent Installers"
-  /sbin/so-elastic-agent-gen-installers
-
   kibana_backport_streams_index_template
 
   update_kafka_metadata "4.3"
@@ -1047,19 +1130,13 @@ post_to_3.2.0() {
 ### 3.2.0 End ###
 
 ### 3.3.0 Scripts ###
-
- # Sets fs.protected_symlinks=0
- #
- # Elastic Agent docker image chowns its directory to the running UID 
- # but does not chown the elastic-agent launcher symlink. 
- # Preventing non-root users from following that launcher symlink.
-disable_sysctl_fs_protected_symlink() {
-  salt -C 'I@stig:enabled' state.single sysctl.present name=fs.protected_symlinks value=0 config=/etc/sysctl.conf || true
-}
-    
 up_to_3.3.0() {
-  disable_sysctl_fs_protected_symlink
-  
+  # remove existing (patched) elasticsearch index template to match integration naming change
+  if ! remove_elasticsearch_index_template "so-logs-sentinel_one_cloud_funnel.login" "sentinel_one_cloud_funnel.login changed to sentinel_one_cloud_funnel.logins"; then
+    FINAL_MESSAGE_QUEUE+=("WARNING: Unable to automatically remove the so-logs-sentinel_one_cloud_funnel.login index template. This step can be performed manually using the following command:")
+    FINAL_MESSAGE_QUEUE+=("    - sudo so-elasticsearch-query _index_template/so-logs-sentinel_one_cloud_funnel.login -XDELETE && so-checkin")
+  fi
+
   INSTALLEDVERSION=3.3.0
 }
 
@@ -1085,11 +1162,126 @@ telegraf_repair() {
 post_to_3.3.0() {
   # Recollate again since some internal DBs were excluded during 3.2.0 soup
   recollate_postgres
-
+  
   telegraf_repair
+
+  set_postversion 3.3.0
 }
 ### 3.3.0 End ###
 
+### 3.4.0 Scripts ###
+up_to_3.4.0() {
+  # download 9.4.8 elastic agent packages
+  determine_elastic_agent_upgrade
+  
+  set_soauth_range
+
+  echo "Removing so-kratos, so-hydra and so-soc so they are recreated on the soauth network."
+  docker rm -f so-kratos so-hydra so-soc >> $SOUP_LOG 2>&1
+
+  for template in so-metrics-logstash.node so-metrics-logstash.stack_monitoring.node; do
+    if ! remove_elasticsearch_index_template "$template" "logstash node and node_cel index patterns reversed"; then
+      FINAL_MESSAGE_QUEUE+=("WARNING: Unable to automatically remove the $template index template. Addon integration templates may fail to load until it is removed:")
+      FINAL_MESSAGE_QUEUE+=("    - sudo so-elasticsearch-query _index_template/$template -XDELETE && so-checkin")
+    fi
+  done
+
+  INSTALLEDVERSION=3.4.0
+}
+
+set_soauth_range() {
+  local pillar_file=/opt/so/saltstack/local/pillar/docker/soc_docker.sls
+  local current_range suggested authnet authgw input
+
+  [[ -f "$pillar_file" ]] || return 0
+
+  current_range=$(so-yaml.py get -r "$pillar_file" docker.range 2>/dev/null) || return 0
+
+  # A default range gets the 172.17.2.0/24 from docker/defaults.yaml, same as a fresh
+  # install, so there is nothing to ask about.
+  [[ -n "$current_range" && "$current_range" != "$DEFAULT_DOCKER_RANGE" ]] || return 0
+
+  if so-yaml.py get -r "$pillar_file" docker.networks.soauth.range >/dev/null 2>&1; then
+    return 0
+  fi
+
+  suggested=$(echo "${current_range%%/*}" | awk -F'.' '{ printf "%s.%s.%s.%s", $1, $2, ($3 + 1) % 256, $4 }')
+
+  if [[ -z $UNATTENDED ]]; then
+    echo ""
+    echo "This grid uses a custom Docker range ($current_range). The authentication"
+    echo "services are moving to their own isolated network, which needs a second /24"
+    echo "that does not overlap it."
+    echo ""
+    while :; do
+      read -rp "Enter the network without the /24 suffix, or press Enter for ${suggested}: " input
+      [[ -z "$input" ]] && input="$suggested"
+      if valid_soauth_range "$input" "$current_range"; then
+        authnet="$input"
+        break
+      fi
+      echo "That range must be a valid IPv4 network, must not be within 172.17.0.0/24, and must not overlap ${current_range}."
+    done
+  else
+    if ! valid_soauth_range "$suggested" "$current_range"; then
+      FINAL_MESSAGE_QUEUE+=("WARNING: Unable to pick a range for the authentication network alongside $current_range. Set it manually before the next highstate:")
+      FINAL_MESSAGE_QUEUE+=("    - so-yaml.py add $pillar_file docker.networks.soauth.range <network>/24")
+      FINAL_MESSAGE_QUEUE+=("    - so-yaml.py add $pillar_file docker.networks.soauth.gateway <gateway>")
+      return 0
+    fi
+    authnet="$suggested"
+    FINAL_MESSAGE_QUEUE+=("NOTE: The authentication services moved to an isolated Docker network and were assigned ${authnet}/24.")
+    FINAL_MESSAGE_QUEUE+=("    - If that conflicts with your environment, update docker.networks.soauth in $pillar_file and run so-checkin.")
+  fi
+
+  authgw=$(echo "$authnet" | awk -F'.' '{print $1,$2,$3,1}' OFS='.')
+
+  echo "Assigning the authentication network the range ${authnet}/24."
+  so-yaml.py add "$pillar_file" docker.networks.soauth.range "${authnet}/24" >> $SOUP_LOG 2>&1
+  so-yaml.py add "$pillar_file" docker.networks.soauth.gateway "$authgw" >> $SOUP_LOG 2>&1
+}
+
+valid_soauth_range() {
+  local candidate=$1 docker_range=$2
+
+  valid_ip4 "$candidate" || return 1
+  [[ $candidate =~ ^172\.17\.0\. ]] && return 1
+  [[ "${candidate}/24" == "$docker_range" ]] && return 1
+  return 0
+}
+
+post_to_3.4.0() {
+  # Generate 9.4.8 elastic agent installers
+  echo "Regenerating Elastic Agent Installers"
+  /sbin/so-elastic-agent-gen-installers
+  
+  for idx in "metrics-logstash.node-default" "metrics-logstash.stack_monitoring.node-default"; do
+    rollover_index "$idx"
+  done
+
+  add_detection_runtime_fields
+
+  remove_frozen_ca_bundles
+
+  set_postversion 3.4.0
+}
+
+remove_frozen_ca_bundles() {
+  rm -f /opt/so/saltstack/local/salt/elasticsearch/cacerts /opt/so/saltstack/local/salt/elasticsearch/tls-ca-bundle.pem >> $SOUP_LOG 2>&1
+}
+
+add_detection_runtime_fields() {
+  local template=$DEFAULT_SALT_DIR/salt/elasticsearch/templates/component/so/detection-mappings.json
+  local runtime
+
+  runtime=$(jq -c '{runtime: .template.mappings.runtime}' "$template")
+  if ! so-elasticsearch-query so-detection/_mapping -XPUT -d "$runtime" --retry 3 --retry-delay 5 --fail >> $SOUP_LOG 2>&1; then
+    FINAL_MESSAGE_QUEUE+=("WARNING: Unable to add the Sigma search fields to the so-detection index. Searches on so_detection.sigmaKind will match nothing until they are added:")
+    FINAL_MESSAGE_QUEUE+=("    - sudo so-elasticsearch-query so-detection/_mapping -XPUT -d \"\$(jq -c '{runtime: .template.mappings.runtime}' $template)\"")
+  fi
+}
+### 3.4.0 End ###
+
 
 repo_sync() {
   echo "Sync the local repo."
@@ -1359,11 +1551,13 @@ verify_es_version_compatibility() {
     local is_active_intermediate_upgrade=1
     # supported upgrade paths for SO-ES versions
     declare -A es_upgrade_map=(
-        ["8.18.4"]="8.18.6 8.18.8 9.0.8"
+      ["8.18.4"]="8.18.6 8.18.8 9.0.8"
 	    ["8.18.6"]="8.18.8 9.0.8"
 	    ["8.18.8"]="9.0.8"
-        ["9.0.8"]="9.3.3 9.3.7"
-        ["9.3.3"]="9.3.7"
+      ["9.0.8"]="9.3.3 9.3.7 9.4.5 9.4.8"
+      ["9.3.3"]="9.3.7 9.4.5 9.4.8"
+      ["9.3.7"]="9.4.5 9.4.8"
+      ["9.4.5"]="9.4.8"
     )
 
     # Elasticsearch MUST upgrade through these versions
@@ -1973,6 +2167,9 @@ main() {
 
   echo "Let's see if we need to update Security Onion."
   upgrade_check
+
+  check_cpu_compatibility
+
   upgrade_space
 
   echo "Verifying Elasticsearch version compatibility across the grid before upgrading."
@@ -2139,12 +2336,28 @@ main() {
 
     # ensure the mine is updated and populated before highstates run, following the salt-master restart
     update_salt_mine
-
+    
+    # kick off a searchnode elasticsearch upgrade
+    set +e
+    if [[ "$es_version" != "$target_es_version" ]]; then
+      if salt-key -L accepted | grep -q "_searchnode$" 2>/dev/null; then
+        # only run if there is atleast 1 searchnode
+        upgrade_searchnode_elasticsearch
+      fi
+    fi
+    set -e
+    
     highstate
     check_saltmaster_status
     postupgrade_changes
     [[ $is_airgap -eq 0 ]] && unmount_update
-
+    
+    if [[ "$es_version" != "$target_es_version" ]]; then
+      # Run final elasticsearch / fleet state on manager to ensure addon index templates are created/regenerated and loaded
+      echo "Running final Elastic states at $(date +"%T.%6N"), after upgrade to $NEWVERSION"
+      salt-call state.apply elasticsearch,elasticfleet queue=True
+    fi
+    
     echo ""
     echo "Upgrade to $NEWVERSION complete."
 
@@ -2235,6 +2448,17 @@ fi
   echo "### soup has been served at $(date) ###"
 }
 
+SKIP_CPU_CHECK=false
+declare -a SOUP_ARGS=()
+for arg in "$@"; do
+  if [[ "$arg" == "--skip-cpu-check" ]]; then
+    SKIP_CPU_CHECK=true
+  else
+    SOUP_ARGS+=("$arg")
+  fi
+done
+set -- "${SOUP_ARGS[@]}"
+
 while getopts ":b:f:y" opt; do
   case ${opt} in
     b )
@@ -2258,7 +2482,7 @@ while getopts ":b:f:y" opt; do
       ISOLOC="$OPTARG"
     ;;
     \? )
-      echo "Usage: soup [-b] [-y] [-f <iso location>]"
+      echo "Usage: soup [-b] [-y] [-f <iso location>] [--skip-cpu-check]"
       exit 1
     ;;
     : )

salt/nginx/config.sls

@@ -57,8 +57,8 @@ nginx_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://nginx/tools/sbin
-    - user: 939
-    - group: 939
+    - user: root
+    - group: root
     - file_mode: 755
 
 #nginx_sbin_jinja: