Merge remote-tracking branch 'origin/3/dev' into rotatehype

# Conflicts:
#	salt/logrotate/defaults.yaml
#	salt/logrotate/soc_logrotate.yaml

salt/_beacons/local_files_beacon.py

@@ -3,13 +3,14 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-# Custom salt beacon that watches the suricata/strelka rule directories for changes
-# and emits a beacon event per changed directory. This replaces the stock salt
-# `inotify` beacon, which leaks a kernel inotify instance every time the minion
-# rebuilds the beacon loader's __context__ (orphaning the old pyinotify.Notifier
-# without closing it) until fs.inotify.max_user_instances is exhausted and the
-# beacon dies with EMFILE. Polling holds zero inotify instances, so the leak is
-# impossible, and it keeps firing during state runs (no blackout).
+# Custom salt beacon that watches hand-edited directories under
+# /opt/so/saltstack/local/salt/ for changes and emits a beacon event per changed
+# directory. This replaces the stock salt `inotify` beacon, which leaks a kernel
+# inotify instance every time the minion rebuilds the beacon loader's __context__
+# (orphaning the old pyinotify.Notifier without closing it) until
+# fs.inotify.max_user_instances is exhausted and the beacon dies with EMFILE.
+# Polling holds zero inotify instances, so the leak is impossible, and it keeps
+# firing during state runs (no blackout).
 #
 # Detection is poll-based with a per-directory fingerprint persisted to
 # WATERMARK_DIR: each pass walks the directory and hashes every file's
@@ -19,9 +20,10 @@
 # up on the next one).
 #
 # Each emitted event carries the watched directory path under the configured tag
-# (e.g. salt/beacon/<minion>/rules_beacon/suricata); the push_suricata / push_strelka
-# reactors write a push intent, after which the existing so-push-drainer /
-# orch.push_batch pipeline takes over unchanged.
+# (e.g. salt/beacon/<minion>/local_files_beacon/zeek); the push_files reactor
+# looks the tag up in salt/reactor/pillar_push_map.yaml and writes a push intent,
+# after which the existing so-push-drainer / orch.push_batch pipeline takes over
+# unchanged.
 
 import hashlib
 import logging
@@ -77,7 +79,9 @@ def _fingerprint(directory):
     h = hashlib.sha1()
     if os.path.isdir(directory):
         entries = []
-        for root, _dirs, files in os.walk(directory):
+        for root, dirs, files in os.walk(directory):
+            # zkg packages are git clones; .git churn would fire a state apply on its own.
+            dirs[:] = [d for d in dirs if d != '.git']
             for name in files:
                 full = os.path.join(root, name)
                 if _excluded(full):
@@ -94,20 +98,22 @@ def _fingerprint(directory):
     return h.hexdigest()
 
 
-def _watermark_file(tag):
-    return os.path.join(WATERMARK_DIR, 'rules_beacon_%s.hash' % tag)
+def _watermark_file(tag, directory):
+    # Keyed by directory: zeek/policy and zeek/zkg share the tag `zeek`.
+    scope = hashlib.sha1(directory.encode('utf-8', 'surrogateescape')).hexdigest()[:12]
+    return os.path.join(WATERMARK_DIR, 'local_files_beacon_%s_%s.hash' % (tag, scope))
 
 
-def _read_watermark(tag):
+def _read_watermark(tag, directory):
     try:
-        with open(_watermark_file(tag), 'r') as f:
+        with open(_watermark_file(tag, directory), 'r') as f:
             return (f.read() or '').strip() or None
     except IOError:
         return None
 
 
-def _write_watermark(tag, digest):
-    path = _watermark_file(tag)
+def _write_watermark(tag, directory, digest):
+    path = _watermark_file(tag, directory)
     try:
         os.makedirs(WATERMARK_DIR, exist_ok=True)
         tmp = path + '.tmp'
@@ -115,7 +121,7 @@ def _write_watermark(tag, digest):
             f.write(digest)
         os.rename(tmp, path)
     except OSError:
-        log.exception('rules_beacon: failed to persist watermark to %s', path)
+        log.exception('local_files_beacon: failed to persist watermark to %s', path)
 
 
 def beacon(config):
@@ -123,17 +129,17 @@ def beacon(config):
 
     for directory, tag in _paths_from_config(config).items():
         digest = _fingerprint(directory)
-        previous = _read_watermark(tag)
+        previous = _read_watermark(tag, directory)
 
         # First run / missing watermark: seed the digest and emit nothing so a
         # fresh host does not fire a spurious fleetwide push.
         if previous is None:
-            _write_watermark(tag, digest)
+            _write_watermark(tag, directory, digest)
             continue
 
         if digest != previous:
-            _write_watermark(tag, digest)
+            _write_watermark(tag, directory, digest)
             retval.append({'tag': tag, 'path': directory})
-            log.info('rules_beacon: change detected in %s, emitting %s', directory, tag)
+            log.info('local_files_beacon: change detected in %s, emitting %s', directory, tag)
 
     return retval

salt/_beacons/local_files_beacon_test.py

@@ -0,0 +1,221 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+import hashlib
+import os
+import shutil
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import local_files_beacon
+
+
+class TestRulesBeacon(unittest.TestCase):
+
+    def setUp(self):
+        # Isolate all on-disk state (watermarks and the dirs we fingerprint) in a
+        # throwaway tree, and point WATERMARK_DIR at it so the real read/write
+        # helpers run against actual files.
+        self.tmpdir = tempfile.mkdtemp()
+        self.state = os.path.join(self.tmpdir, 'state')
+        patcher = patch.object(local_files_beacon, 'WATERMARK_DIR', self.state)
+        patcher.start()
+        self.addCleanup(patcher.stop)
+
+    def tearDown(self):
+        shutil.rmtree(self.tmpdir, ignore_errors=True)
+
+    def _make_dir(self, name, files=None):
+        path = os.path.join(self.tmpdir, name)
+        os.makedirs(path, exist_ok=True)
+        for fname, content in (files or {}).items():
+            with open(os.path.join(path, fname), 'w') as f:
+                f.write(content)
+        return path
+
+    # -- trivial contract -------------------------------------------------
+
+    def test_virtual_returns_true(self):
+        self.assertTrue(local_files_beacon.__virtual__())
+
+    def test_validate_returns_valid(self):
+        self.assertEqual(local_files_beacon.validate({}), (True, 'valid'))
+
+    # -- _paths_from_config -----------------------------------------------
+
+    def test_paths_from_config_list_of_dicts(self):
+        config = [{'interval': 10}, {'paths': {'/a': 'suricata', '/b': 'strelka'}}]
+        self.assertEqual(
+            local_files_beacon._paths_from_config(config),
+            {'/a': 'suricata', '/b': 'strelka'},
+        )
+
+    def test_paths_from_config_plain_dict(self):
+        self.assertEqual(
+            local_files_beacon._paths_from_config({'paths': {'/a': 'suricata'}}),
+            {'/a': 'suricata'},
+        )
+
+    def test_paths_from_config_skips_non_dict_items(self):
+        self.assertEqual(local_files_beacon._paths_from_config(['bogus', 42]), {})
+
+    def test_paths_from_config_paths_not_a_dict(self):
+        self.assertEqual(local_files_beacon._paths_from_config({'paths': 'nope'}), {})
+
+    def test_paths_from_config_unexpected_type(self):
+        self.assertEqual(local_files_beacon._paths_from_config('nonsense'), {})
+
+    # -- _excluded --------------------------------------------------------
+
+    def test_excluded_matches_temp_and_editor_files(self):
+        for pathname in ('/rules/foo.swp', '/rules/foo~', '/rules/4913', '/rules/.#foo'):
+            self.assertTrue(local_files_beacon._excluded(pathname), pathname)
+
+    def test_excluded_allows_real_rule_files(self):
+        self.assertFalse(local_files_beacon._excluded('/rules/suricata.rules'))
+
+    # -- _fingerprint -----------------------------------------------------
+
+    def test_fingerprint_missing_dir_is_empty_tree_digest(self):
+        missing = os.path.join(self.tmpdir, 'does-not-exist')
+        self.assertEqual(local_files_beacon._fingerprint(missing), hashlib.sha1().hexdigest())
+
+    def test_fingerprint_changes_when_content_changes(self):
+        d = self._make_dir('rules', {'a.rules': 'alert'})
+        before = local_files_beacon._fingerprint(d)
+        with open(os.path.join(d, 'a.rules'), 'w') as f:
+            f.write('alert tcp any any -> any any')  # different size
+        self.assertNotEqual(local_files_beacon._fingerprint(d), before)
+
+    def test_fingerprint_ignores_excluded_files(self):
+        d = self._make_dir('rules', {'a.rules': 'alert'})
+        before = local_files_beacon._fingerprint(d)
+        with open(os.path.join(d, 'a.rules.swp'), 'w') as f:
+            f.write('editor swap')
+        self.assertEqual(local_files_beacon._fingerprint(d), before)
+
+    def test_fingerprint_skips_unstatable_entries(self):
+        # A dangling symlink appears in os.walk's file list but os.stat raises
+        # OSError, exercising the except-continue path.
+        d = self._make_dir('rules', {'a.rules': 'alert'})
+        good = local_files_beacon._fingerprint(d)
+        os.symlink(os.path.join(d, 'missing-target'), os.path.join(d, 'broken.link'))
+        self.assertEqual(local_files_beacon._fingerprint(d), good)
+
+    def test_fingerprint_prunes_git_metadata(self):
+        # zkg packages are git clones, so the watched tree carries .git.
+        d = self._make_dir('zkg', {'pkg.zeek': 'print 1;'})
+        before = local_files_beacon._fingerprint(d)
+        git_dir = os.path.join(d, 'pkg', '.git', 'refs', 'heads')
+        os.makedirs(git_dir)
+        with open(os.path.join(git_dir, 'main'), 'w') as f:
+            f.write('0' * 40)
+        self.assertEqual(local_files_beacon._fingerprint(d), before)
+
+    def test_fingerprint_still_sees_worktree_next_to_git(self):
+        d = self._make_dir('zkg', {'pkg.zeek': 'print 1;'})
+        os.makedirs(os.path.join(d, 'pkg', '.git'))
+        before = local_files_beacon._fingerprint(d)
+        with open(os.path.join(d, 'pkg', 'scripts.zeek'), 'w') as f:
+            f.write('print 2;')
+        self.assertNotEqual(local_files_beacon._fingerprint(d), before)
+
+    # -- _read_watermark / _write_watermark -------------------------------
+
+    def test_watermark_round_trip(self):
+        local_files_beacon._write_watermark('suricata', '/rules/suricata', 'deadbeef')
+        self.assertEqual(
+            local_files_beacon._read_watermark('suricata', '/rules/suricata'), 'deadbeef')
+
+    def test_read_watermark_missing_returns_none(self):
+        self.assertIsNone(local_files_beacon._read_watermark('suricata', '/rules/suricata'))
+
+    def test_read_watermark_empty_file_returns_none(self):
+        os.makedirs(self.state, exist_ok=True)
+        with open(local_files_beacon._watermark_file('suricata', '/rules/suricata'), 'w') as f:
+            f.write('')
+        self.assertIsNone(local_files_beacon._read_watermark('suricata', '/rules/suricata'))
+
+    def test_write_watermark_swallows_oserror(self):
+        with patch.object(local_files_beacon.os, 'makedirs', side_effect=OSError):
+            local_files_beacon._write_watermark('suricata', '/rules/suricata', 'deadbeef')
+        self.assertIsNone(local_files_beacon._read_watermark('suricata', '/rules/suricata'))
+
+    def test_watermark_file_differs_per_directory_within_one_tag(self):
+        # zeek/policy and zeek/zkg share the tag 'zeek'.
+        self.assertNotEqual(
+            local_files_beacon._watermark_file('zeek', '/local/zeek/policy'),
+            local_files_beacon._watermark_file('zeek', '/local/zeek/zkg'),
+        )
+
+    def test_watermarks_are_independent_within_one_tag(self):
+        local_files_beacon._write_watermark('zeek', '/local/zeek/policy', 'policyhash')
+        local_files_beacon._write_watermark('zeek', '/local/zeek/zkg', 'zkghash')
+        self.assertEqual(
+            local_files_beacon._read_watermark('zeek', '/local/zeek/policy'), 'policyhash')
+        self.assertEqual(
+            local_files_beacon._read_watermark('zeek', '/local/zeek/zkg'), 'zkghash')
+
+    # -- beacon -----------------------------------------------------------
+
+    def _config(self, mapping):
+        return [{'paths': mapping}]
+
+    def test_beacon_seeds_first_run_and_emits_nothing(self):
+        with patch.object(local_files_beacon, '_fingerprint', return_value='hash1'), \
+             patch.object(local_files_beacon, '_read_watermark', return_value=None), \
+             patch.object(local_files_beacon, '_write_watermark') as mock_write:
+            result = local_files_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
+        self.assertEqual(result, [])
+        mock_write.assert_called_once_with('suricata', '/rules/suricata', 'hash1')
+
+    def test_beacon_emits_on_change(self):
+        with patch.object(local_files_beacon, '_fingerprint', return_value='newhash'), \
+             patch.object(local_files_beacon, '_read_watermark', return_value='oldhash'), \
+             patch.object(local_files_beacon, '_write_watermark') as mock_write:
+            result = local_files_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
+        self.assertEqual(result, [{'tag': 'suricata', 'path': '/rules/suricata'}])
+        mock_write.assert_called_once_with('suricata', '/rules/suricata', 'newhash')
+
+    def test_beacon_no_change_emits_nothing(self):
+        with patch.object(local_files_beacon, '_fingerprint', return_value='samehash'), \
+             patch.object(local_files_beacon, '_read_watermark', return_value='samehash'), \
+             patch.object(local_files_beacon, '_write_watermark') as mock_write:
+            result = local_files_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
+        self.assertEqual(result, [])
+        mock_write.assert_not_called()
+
+    def test_beacon_end_to_end_with_real_files(self):
+        # Exercise the full stack (real fingerprint + real watermark files) across
+        # two poll passes: first seeds silently, second fires after a write.
+        d = self._make_dir('rules', {'a.rules': 'alert'})
+        config = self._config({d: 'suricata'})
+
+        self.assertEqual(local_files_beacon.beacon(config), [])  # seed pass
+        self.assertEqual(local_files_beacon.beacon(config), [])  # unchanged pass
+
+        with open(os.path.join(d, 'b.rules'), 'w') as f:
+            f.write('alert tcp any any -> any any')
+        self.assertEqual(local_files_beacon.beacon(config), [{'tag': 'suricata', 'path': d}])
+
+    def test_beacon_two_dirs_one_tag_do_not_flap(self):
+        # Tag-keyed watermarks would clobber each other and emit on every pass.
+        policy = self._make_dir('zeek/policy', {'intel.dat': '#fields\tindicator'})
+        zkg = self._make_dir('zeek/zkg', {'README': 'place packages here'})
+        config = self._config({policy: 'zeek', zkg: 'zeek'})
+
+        self.assertEqual(local_files_beacon.beacon(config), [])  # seed pass
+        self.assertEqual(local_files_beacon.beacon(config), [])  # idle
+        self.assertEqual(local_files_beacon.beacon(config), [])  # still idle
+
+        with open(os.path.join(policy, 'intel.dat'), 'a') as f:
+            f.write('\nevil.com\tIntel::DOMAIN\tsource\n')
+        self.assertEqual(local_files_beacon.beacon(config), [{'tag': 'zeek', 'path': policy}])
+        self.assertEqual(local_files_beacon.beacon(config), [])  # quiet again
+
+
+if __name__ == '__main__':
+    unittest.main()

salt/_beacons/rules_beacon_test.py

@@ -1,172 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-import hashlib
-import os
-import shutil
-import tempfile
-import unittest
-from unittest.mock import patch
-
-import rules_beacon
-
-
-class TestRulesBeacon(unittest.TestCase):
-
-    def setUp(self):
-        # Isolate all on-disk state (watermarks and the dirs we fingerprint) in a
-        # throwaway tree, and point WATERMARK_DIR at it so the real read/write
-        # helpers run against actual files.
-        self.tmpdir = tempfile.mkdtemp()
-        self.state = os.path.join(self.tmpdir, 'state')
-        patcher = patch.object(rules_beacon, 'WATERMARK_DIR', self.state)
-        patcher.start()
-        self.addCleanup(patcher.stop)
-
-    def tearDown(self):
-        shutil.rmtree(self.tmpdir, ignore_errors=True)
-
-    def _make_dir(self, name, files=None):
-        path = os.path.join(self.tmpdir, name)
-        os.makedirs(path, exist_ok=True)
-        for fname, content in (files or {}).items():
-            with open(os.path.join(path, fname), 'w') as f:
-                f.write(content)
-        return path
-
-    # -- trivial contract -------------------------------------------------
-
-    def test_virtual_returns_true(self):
-        self.assertTrue(rules_beacon.__virtual__())
-
-    def test_validate_returns_valid(self):
-        self.assertEqual(rules_beacon.validate({}), (True, 'valid'))
-
-    # -- _paths_from_config -----------------------------------------------
-
-    def test_paths_from_config_list_of_dicts(self):
-        config = [{'interval': 10}, {'paths': {'/a': 'suricata', '/b': 'strelka'}}]
-        self.assertEqual(
-            rules_beacon._paths_from_config(config),
-            {'/a': 'suricata', '/b': 'strelka'},
-        )
-
-    def test_paths_from_config_plain_dict(self):
-        self.assertEqual(
-            rules_beacon._paths_from_config({'paths': {'/a': 'suricata'}}),
-            {'/a': 'suricata'},
-        )
-
-    def test_paths_from_config_skips_non_dict_items(self):
-        self.assertEqual(rules_beacon._paths_from_config(['bogus', 42]), {})
-
-    def test_paths_from_config_paths_not_a_dict(self):
-        self.assertEqual(rules_beacon._paths_from_config({'paths': 'nope'}), {})
-
-    def test_paths_from_config_unexpected_type(self):
-        self.assertEqual(rules_beacon._paths_from_config('nonsense'), {})
-
-    # -- _excluded --------------------------------------------------------
-
-    def test_excluded_matches_temp_and_editor_files(self):
-        for pathname in ('/rules/foo.swp', '/rules/foo~', '/rules/4913', '/rules/.#foo'):
-            self.assertTrue(rules_beacon._excluded(pathname), pathname)
-
-    def test_excluded_allows_real_rule_files(self):
-        self.assertFalse(rules_beacon._excluded('/rules/suricata.rules'))
-
-    # -- _fingerprint -----------------------------------------------------
-
-    def test_fingerprint_missing_dir_is_empty_tree_digest(self):
-        missing = os.path.join(self.tmpdir, 'does-not-exist')
-        self.assertEqual(rules_beacon._fingerprint(missing), hashlib.sha1().hexdigest())
-
-    def test_fingerprint_changes_when_content_changes(self):
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        before = rules_beacon._fingerprint(d)
-        with open(os.path.join(d, 'a.rules'), 'w') as f:
-            f.write('alert tcp any any -> any any')  # different size
-        self.assertNotEqual(rules_beacon._fingerprint(d), before)
-
-    def test_fingerprint_ignores_excluded_files(self):
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        before = rules_beacon._fingerprint(d)
-        with open(os.path.join(d, 'a.rules.swp'), 'w') as f:
-            f.write('editor swap')
-        self.assertEqual(rules_beacon._fingerprint(d), before)
-
-    def test_fingerprint_skips_unstatable_entries(self):
-        # A dangling symlink appears in os.walk's file list but os.stat raises
-        # OSError, exercising the except-continue path.
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        good = rules_beacon._fingerprint(d)
-        os.symlink(os.path.join(d, 'missing-target'), os.path.join(d, 'broken.link'))
-        self.assertEqual(rules_beacon._fingerprint(d), good)
-
-    # -- _read_watermark / _write_watermark -------------------------------
-
-    def test_watermark_round_trip(self):
-        rules_beacon._write_watermark('suricata', 'deadbeef')
-        self.assertEqual(rules_beacon._read_watermark('suricata'), 'deadbeef')
-
-    def test_read_watermark_missing_returns_none(self):
-        self.assertIsNone(rules_beacon._read_watermark('suricata'))
-
-    def test_read_watermark_empty_file_returns_none(self):
-        os.makedirs(self.state, exist_ok=True)
-        with open(rules_beacon._watermark_file('suricata'), 'w') as f:
-            f.write('')
-        self.assertIsNone(rules_beacon._read_watermark('suricata'))
-
-    def test_write_watermark_swallows_oserror(self):
-        with patch.object(rules_beacon.os, 'makedirs', side_effect=OSError):
-            rules_beacon._write_watermark('suricata', 'deadbeef')
-        self.assertIsNone(rules_beacon._read_watermark('suricata'))
-
-    # -- beacon -----------------------------------------------------------
-
-    def _config(self, mapping):
-        return [{'paths': mapping}]
-
-    def test_beacon_seeds_first_run_and_emits_nothing(self):
-        with patch.object(rules_beacon, '_fingerprint', return_value='hash1'), \
-             patch.object(rules_beacon, '_read_watermark', return_value=None), \
-             patch.object(rules_beacon, '_write_watermark') as mock_write:
-            result = rules_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
-        self.assertEqual(result, [])
-        mock_write.assert_called_once_with('suricata', 'hash1')
-
-    def test_beacon_emits_on_change(self):
-        with patch.object(rules_beacon, '_fingerprint', return_value='newhash'), \
-             patch.object(rules_beacon, '_read_watermark', return_value='oldhash'), \
-             patch.object(rules_beacon, '_write_watermark') as mock_write:
-            result = rules_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
-        self.assertEqual(result, [{'tag': 'suricata', 'path': '/rules/suricata'}])
-        mock_write.assert_called_once_with('suricata', 'newhash')
-
-    def test_beacon_no_change_emits_nothing(self):
-        with patch.object(rules_beacon, '_fingerprint', return_value='samehash'), \
-             patch.object(rules_beacon, '_read_watermark', return_value='samehash'), \
-             patch.object(rules_beacon, '_write_watermark') as mock_write:
-            result = rules_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
-        self.assertEqual(result, [])
-        mock_write.assert_not_called()
-
-    def test_beacon_end_to_end_with_real_files(self):
-        # Exercise the full stack (real fingerprint + real watermark files) across
-        # two poll passes: first seeds silently, second fires after a write.
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        config = self._config({d: 'suricata'})
-
-        self.assertEqual(rules_beacon.beacon(config), [])  # seed pass
-        self.assertEqual(rules_beacon.beacon(config), [])  # unchanged pass
-
-        with open(os.path.join(d, 'b.rules'), 'w') as f:
-            f.write('alert tcp any any -> any any')
-        self.assertEqual(rules_beacon.beacon(config), [{'tag': 'suricata', 'path': d}])
-
-
-if __name__ == '__main__':
-    unittest.main()

salt/common/init.sls

@@ -141,6 +141,20 @@ pin_nic_names:
       - file: common_sbin
       - file: statedir
 
+# Once a node is actually running UEK8, the stock EL9 (RHCK) kernel packages are dead weight.
+# They can't be removed any earlier -- dnf protects the running kernel -- so the cleanup waits
+# for the reboot, which makes the highstate the natural place to catch it: fresh installs
+# reboot at the end of setup, and upgraded nodes reboot whenever the admin schedules it.
+# so-kernel-upgrade --cleanup checks rpm before touching dnf, so this costs an rpm query on
+# every highstate after the first pass. The package list lives in the script only, so there
+# is nothing here to drift out of sync with it.
+remove_stock_kernel:
+  cmd.run:
+    - name: /usr/sbin/so-kernel-upgrade --cleanup
+    - onlyif: 'uname -r | grep -qE "^6\.[0-9]+.*uek"'
+    - require:
+      - file: common_sbin
+
 common_sbin_jinja:
   file.recurse:
     - name: /usr/sbin

salt/common/tools/sbin/so-kernel-upgrade

@@ -5,10 +5,11 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 #
-# so-kernel-upgrade — install the UEK8 (6.x) kernel and make it the boot default.
+# so-kernel-upgrade — install the UEK8 (6.x) kernel, make it the boot default, and once the
+# node is running it, remove the stock EL9 kernel.
 #
 # Security Onion is moving off the EL9 stock kernel (RHCK, 5.14) and UEK7 (5.15) onto UEK8
-# (6.x). Three things have to happen, and the tool has to drive each one:
+# (6.x). Four things have to happen, and the tool has to drive each one:
 #
 #   1. Populate. The manager mirrors the UEK8 packages into /nsm/kernelrepo via so-repo-sync,
 #      and serves them to the grid over https://<manager>/kernelrepo. Until that sync runs the
@@ -26,10 +27,21 @@
 #        - From the stock EL9 kernel (RHCK, 5.14, no UEK) it is a flavor CROSS that is NOT
 #          auto-promoted, so the box keeps booting RHCK until grubby is told otherwise.
 #      This tool inspects the running kernel and only runs 'grubby --set-default' for RHCK.
+#   4. Clean up. Once the node is actually RUNNING UEK8 the stock kernel packages are dead
+#      weight -- disk in /boot and a stale GRUB entry. They cannot come off any earlier:
+#      dnf's protect_running_kernel refuses to erase the booted kernel-core, so the removal
+#      has to wait for the reboot. Waiting is also the safer sequencing on its own terms --
+#      the node has proven it comes up on UEK8 before its fallback is deleted. That is why
+#      the removal does not happen in the uek7 branch either, where dnf would allow it.
 #
 # Every one of those failure modes is silent by default. This tool handles each case and fails
 # loudly when it cannot, rather than reporting success while changing nothing.
 #
+# Invocation: with no arguments it drives the whole sequence for whatever kernel the node is
+# on. With --cleanup it does the step 4 removal ONLY, and no-ops on a node that isn't running
+# UEK8 yet -- that is the form the common highstate calls (remove_stock_kernel in
+# salt/common/init.sls) so the cleanup lands grid-wide after each node reboots.
+#
 # Manager vs minion: only the manager owns /nsm/kernelrepo, so only the manager can populate
 # it. If the repo is empty here, a manager runs so-repo-sync itself; a minion has no way to
 # fix it and exits non-zero telling the admin to sync the manager first.
@@ -49,6 +61,11 @@ KERNEL_REPO_DIR="/nsm/kernelrepo"
 REPOSYNC_CONF="/opt/so/conf/reposync/repodownload.conf"
 GLOBAL_PILLAR="/opt/so/saltstack/local/pillar/global/soc_global.sls"
 
+# Stock EL9 (RHCK) kernel packages, removed only once the node is running UEK8 (see step 4
+# in the header). Left deliberately narrow: UEK7 kernel-uek builds age out on their own via
+# installonly_limit=3, and kernel-devel/kernel-headers are not touched.
+RHCK_PKGS="kernel kernel-core kernel-modules kernel-modules-core kernel-tools kernel-tools-libs"
+
 log() { echo "[so-kernel-upgrade] $*"; }
 die() { echo "[so-kernel-upgrade] ERROR: $*" >&2; exit 1; }
 
@@ -149,8 +166,13 @@ ensure_kernel_repo() {
 }
 
 reboot_notice() {
-    [ "$(uname -r)" = "$(basename "$1" | sed 's/^vmlinuz-//')" ] \
-        || log "REBOOT REQUIRED to start using the UEK8 kernel (currently running $(uname -r))."
+    [ "$(uname -r)" = "$(basename "$1" | sed 's/^vmlinuz-//')" ] && return 0
+    log "REBOOT REQUIRED to start using the UEK8 kernel (currently running $(uname -r))."
+    # The stock kernel can't be removed until it stops being the running one, so say when
+    # that will happen rather than leaving the admin to wonder if it was missed.
+    [ -n "$(rhck_installed)" ] \
+        && log "The stock EL9 kernel is left in place until then; it is removed by the next highstate after the reboot."
+    return 0
 }
 
 # Keep future kernel updates on the UEK line rather than falling back to RHCK. Oracle ships
@@ -162,6 +184,32 @@ set_default_kernel_conf() {
     fi
 }
 
+# Which of RHCK_PKGS are actually installed, one per line. rpm -qa treats each argument as a
+# name glob and prints only what it finds, so a package that was never installed (or is
+# already gone) simply doesn't appear -- no "not installed" noise and no non-zero exit.
+rhck_installed() {
+    rpm -qa $RHCK_PKGS 2>/dev/null
+}
+
+# Remove the stock EL9 kernel. Only ever called once the running kernel is UEK8. The rpm
+# check above is the idempotency guard, so this is a cheap no-op on every highstate after
+# the first one -- it costs an rpm query, not a dnf transaction.
+remove_rhck() {
+    local installed; installed="$(rhck_installed)"
+    if [ -z "$installed" ]; then
+        log "no stock EL9 (RHCK) kernel packages installed; nothing to remove."
+        return 0
+    fi
+
+    log "running UEK8; removing the stock EL9 (RHCK) kernel packages:"
+    echo "$installed" | sed 's/^/[so-kernel-upgrade]   /'
+    dnf -y remove $RHCK_PKGS || die "failed to remove the stock EL9 kernel packages"
+
+    installed="$(rhck_installed)"
+    [ -z "$installed" ] || die "dnf reported success but these remain: $(echo $installed)"
+    log "stock EL9 kernel packages removed."
+}
+
 # Make sure a UEK8 kernel is installed, leaving its boot entry in INSTALLED_UEK8. If one is
 # already present we leave the repo alone -- it may be disabled or empty and we don't need it
 # just to flip the boot default. Otherwise install the explicit NEVRA, not the bare package
@@ -184,12 +232,38 @@ ensure_uek8_installed() {
     log "installed UEK8 kernel: $INSTALLED_UEK8"
 }
 
+# --cleanup does step 4 and nothing else. It exits 0 rather than failing on a node that
+# isn't on UEK8 yet: the highstate gates on 'uname -r' before calling this, and a state that
+# fails whenever that gate races would be worse than one that says what it's waiting for.
+case "$1" in
+"")
+    ;;
+--cleanup)
+    if [ "$(running_flavor)" != uek8 ]; then
+        log "not running a UEK8 kernel yet (currently $(uname -r)); leaving the stock EL9 kernel in place."
+        log "Run so-kernel-upgrade with no arguments to install UEK8, then reboot."
+        exit 0
+    fi
+    set_default_kernel_conf
+    remove_rhck
+    exit 0
+    ;;
+*)
+    echo "Usage: so-kernel-upgrade [--cleanup]" >&2
+    echo "  (no arguments)  install UEK8, make it the boot default, clean up once it's running" >&2
+    echo "  --cleanup       remove the stock EL9 kernel; no-op unless already running UEK8" >&2
+    exit 1
+    ;;
+esac
+
 case "$(running_flavor)" in
 uek8)
     # Already on the 6.x UEK line. A plain 'dnf update' keeps this node current within the
-    # lineage and auto-promotes newer builds, so there is nothing for this tool to do.
-    log "already running a UEK8 kernel ($(uname -r)); nothing to do."
-    exit 0
+    # lineage and auto-promotes newer builds, so there is no install or grubby work left --
+    # only the step 4 cleanup, which this is the first point in the sequence that can run it.
+    log "already running a UEK8 kernel ($(uname -r)); no kernel install needed."
+    set_default_kernel_conf
+    remove_rhck
     ;;
 
 uek7)

salt/common/tools/sbin/so-log-check

@@ -154,6 +154,8 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|id.orig_h"                    # false positive (zeek test data)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|emerging-all.rules"           # false positive (error in rulename)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|invalid query input"          # false positive (Invalid user input in hunt query)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no data available for the requested dates" # false positive (pcap cypress test submits a job with an empty time frame)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no job processor"             # false positive (same empty-time-frame job on import nodes, where no pcap processor runs)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|example"                      # false positive (example test data)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200"                   # false positive (request successful, contained error string in content)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|app_layer.error"              # false positive (suricata 7) in stats.log e.g. app_layer.error.imap.parser | Total | 0
@@ -169,6 +171,11 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error while parsing document for index \[.ds-logs-kratos-so-.*object mapping for \[file\]" # false positive (mapping error occuring BEFORE kratos index has rolled over in 2.4.210)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|No such container"            # false positive (telegraf trying to run stats on an old container)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|passwords do not match"       # false positive (automated hydra test)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Request did not pass preprocessing" # expected WARN log lines indicating invalid auth header
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Missing or invalid authorization header for bearer token" # expected WARN log lines indicating invalid auth header
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unexpected authorization header" # expected WARN log lines indicating invalid auth header
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Missing ory_kratos_session cookie" # expected WARN log lines indicating invalid auth header
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Static assets preprocessor only supports GET and HEAD requests" # expected WARN log lines indicating invalid auth header
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then

salt/logrotate/defaults.yaml

@@ -220,6 +220,36 @@ logrotate:
       - extension .log
       - dateext
       - dateyesterday
+    /opt/so/log/salt/virtual_node_manager:
+      - daily
+      - rotate 14
+      - missingok
+      - copytruncate
+      - compress
+      - create
+      - extension .log
+      - dateext
+      - dateyesterday
+    /opt/so/log/salt/so-salt-cloud:
+      - daily
+      - rotate 14
+      - missingok
+      - copytruncate
+      - compress
+      - create
+      - extension .log
+      - dateext
+      - dateyesterday
+    /opt/so/log/salt/so-soup-grid-highstate:
+      - daily
+      - rotate 14
+      - missingok
+      - copytruncate
+      - compress
+      - create
+      - extension .log
+      - dateext
+      - dateyesterday
     /nsm/idh/*_x_log:
       - daily
       - rotate 14

salt/logrotate/soc_logrotate.yaml

@@ -140,6 +140,27 @@ logrotate:
       multiline: True
       global: True
       forcedType: "[]string"
+    "/opt/so/log/salt/virtual_node_manager":
+      description: List of logrotate options for this file.
+      title: /opt/so/log/salt/virtual_node_manager
+      advanced: True
+      multiline: True
+      global: True
+      forcedType: "[]string"
+    "/opt/so/log/salt/so-salt-cloud":
+      description: List of logrotate options for this file.
+      title: /opt/so/log/salt/so-salt-cloud
+      advanced: True
+      multiline: True
+      global: True
+      forcedType: "[]string"
+    "/opt/so/log/salt/so-soup-grid-highstate":
+      description: List of logrotate options for this file.
+      title: /opt/so/log/salt/so-soup-grid-highstate
+      advanced: True
+      multiline: True
+      global: True
+      forcedType: "[]string"
     "/nsm/idh/*_x_log":
       description: List of logrotate options for this file.
       title: /nsm/idh/*.log

salt/logstash/config.sls

@@ -81,6 +81,14 @@ ls_custom_pipeline_conf_{{assigned_pipeline}}_{{pipeline}}:
 
 
 {% for assigned_pipeline in ASSIGNED_PIPELINES %}
+{# a blank per-pipeline setting falls back to the global logstash.yml value #}
+{% set PARSED_OVERRIDES = LOGSTASH_MERGED.get('pipeline_settings', {}).get(assigned_pipeline, {}) %}
+{% if PARSED_OVERRIDES is not mapping %}
+{%   do salt.log.warning('logstash: ignoring malformed pipeline_settings for pipeline ' ~ assigned_pipeline ~ '; expected a set of settings') %}
+{% endif %}
+{% set PIPELINE_OVERRIDES = PARSED_OVERRIDES if PARSED_OVERRIDES is mapping else {} %}
+{% set THREADS = PIPELINE_OVERRIDES.get('pipeline_x_workers') or LOGSTASH_MERGED.config.pipeline_x_workers %}
+{% set BATCH = PIPELINE_OVERRIDES.get('pipeline_x_batch_x_size') or LOGSTASH_MERGED.config.pipeline_x_batch_x_size %}
     {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %}
 ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}:
   file.managed:
@@ -92,8 +100,8 @@ ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_")
         GLOBALS: {{ GLOBALS }}
         ES_USER: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') }}"
         ES_PASS: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') }}"
-        THREADS: {{ LOGSTASH_MERGED.config.pipeline_x_workers }}
-        BATCH: {{ LOGSTASH_MERGED.config.pipeline_x_batch_x_size }}
+        THREADS: {{ THREADS }}
+        BATCH: {{ BATCH }}
       {% else %}
     - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1]}}
       {% endif %}
@@ -125,6 +133,14 @@ lspipelinesyml:
     - defaults:
         ASSIGNED_PIPELINES: {{ ASSIGNED_PIPELINES }}
 
+lslog4j2:
+  file.managed:
+    - name: /opt/so/conf/logstash/etc/log4j2.properties
+    - source: salt://logstash/etc/log4j2.properties.jinja
+    - template: jinja
+    - user: 931
+    - group: 939
+
 lsetcsync:
   file.recurse:
     - name: /opt/so/conf/logstash/etc
@@ -133,7 +149,11 @@ lsetcsync:
     - group: 939
     - template: jinja
     - clean: True
-    - exclude_pat: pipelines*
+{#- both names are matched: the .jinja source so the recurse does not copy it verbatim,
+    and the rendered file so clean: True does not delete what lslog4j2 wrote #}
+    - exclude_pat:
+      - pipelines*
+      - log4j2.properties*
     - defaults:
         LOGSTASH_MERGED: {{ LOGSTASH_MERGED }}
 

salt/logstash/defaults.yaml

@@ -42,6 +42,11 @@ logstash:
     custom2: []
     custom3: []
     custom4: []
+    custom5: []
+    custom6: []
+    custom7: []
+    custom8: []
+    custom9: []
   pipeline_config:
     custom001: |-
       filter {
@@ -60,10 +65,405 @@ logstash:
     custom008: PLACEHOLDER
     custom009: PLACEHOLDER
     custom010: PLACEHOLDER
+  pipeline_settings:
+    fleet:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    manager:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    receiver:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    search:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom0:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom1:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom2:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom3:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom4:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom5:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom6:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom7:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom8:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom9:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
   settings:
     lsheap: 500m
   config:
     api_x_http_x_host: 0.0.0.0
+    log_x_level: info
+    log_x_format: plain
     path_x_logs: /var/log/logstash
     pipeline_x_workers: 1
     pipeline_x_batch_x_size: 125

salt/logstash/enabled.sls

@@ -105,6 +105,8 @@ so-logstash:
     {% endif %}
     - watch:
       - file: lsetcsync
+      - file: lslog4j2
+      - file: lspipelinesyml
       - file: trusttheca
       {% if GLOBALS.is_manager %}
       - file: elasticsearch_cacerts

salt/logstash/etc/log4j2.properties.jinja

@@ -1,3 +1,4 @@
+{%- from 'logstash/map.jinja' import LOGSTASH_MERGED -%}
 status = error
 name = LogstashPropertiesConfig
 
@@ -16,8 +17,14 @@ name = LogstashPropertiesConfig
 appender.rolling.type = RollingFile
 appender.rolling.name = rolling
 appender.rolling.fileName = /var/log/logstash/logstash.log
+{%- if LOGSTASH_MERGED.config.get('log_x_format', 'plain') == 'json' %}
+appender.rolling.layout.type = JSONLayout
+appender.rolling.layout.compact = true
+appender.rolling.layout.eventEol = true
+{%- else %}
 appender.rolling.layout.type = PatternLayout
 appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %.10000m%n
+{%- endif %}
 appender.rolling.filePattern = /var/log/logstash/logstash-%d{yyyy-MM-dd}.log.gz
 appender.rolling.policies.type = Policies
 appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
@@ -32,7 +39,5 @@ appender.rolling.strategy.action.condition.type = IfFileName
 appender.rolling.strategy.action.condition.glob = *.gz
 appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified
 appender.rolling.strategy.action.condition.nested_condition.age = 7D
-rootLogger.level = info
+rootLogger.level = ${sys:ls.log.level}
 rootLogger.appenderRef.rolling.ref = rolling
-#rootLogger.level = ${sys:ls.log.level}
-#rootLogger.appenderRef.console.ref = ${sys:ls.log.format}_console

salt/logstash/etc/pipelines.yml.jinja

@@ -1,4 +1,17 @@
+{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
+{%- set PIPELINE_SETTINGS = LOGSTASH_MERGED.get('pipeline_settings', {}) %}
 {%- for assigned_pipeline in ASSIGNED_PIPELINES %}
 - pipeline.id: {{ assigned_pipeline }}
   path.config: "/usr/share/logstash/pipelines/{{ assigned_pipeline }}/"
+{%-   set extra = PIPELINE_SETTINGS.get(assigned_pipeline, {}) %}
+{%-   if extra is mapping %}
+{#-     values are emitted unquoted so yaml re-infers the type logstash expects:
+        4 as an integer, false as a boolean, 1024mb and auto as strings #}
+{%-     for key, value in extra | dictsort %}
+{%-       set rendered = key | replace('_x_', '.') %}
+{%-       if value not in ['', None] and rendered not in ['pipeline.id', 'path.config'] %}
+  {{ rendered }}: {{ value }}
+{%-       endif %}
+{%-     endfor %}
+{%-   endif %}
 {% endfor -%}

salt/logstash/soc_logstash.yaml

@@ -16,6 +16,7 @@ logstash:
       heavynode: *assigned_pipelines
       searchnode: *assigned_pipelines
       manager: *assigned_pipelines
+      managerhype: *assigned_pipelines
       managersearch: *assigned_pipelines
       fleet: *assigned_pipelines
   defined_pipelines:
@@ -34,6 +35,11 @@ logstash:
     custom2: *defined_pipelines
     custom3: *defined_pipelines
     custom4: *defined_pipelines
+    custom5: *defined_pipelines
+    custom6: *defined_pipelines
+    custom7: *defined_pipelines
+    custom8: *defined_pipelines
+    custom9: *defined_pipelines
   pipeline_config:
     custom001: &pipeline_config
       description: Pipeline configuration for Logstash
@@ -51,6 +57,351 @@ logstash:
     custom008: *pipeline_config
     custom009: *pipeline_config
     custom010: *pipeline_config
+  pipeline_settings:
+    manager: &pipeline_settings
+      pipeline_x_workers:
+        description: >-
+          Number of worker threads that run filters and outputs for this pipeline. May be set higher
+          than the CPU core count when outputs spend time waiting on I/O. Leave blank to use the value
+          from logstash.yml.
+        title: pipeline.workers
+        regex: '^$|^[1-9][0-9]*$'
+        regexFailureMessage: Must be blank, or a positive whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_batch_x_size:
+        description: >-
+          Maximum number of events an individual worker thread collects before running filters and
+          outputs. Larger batches are more efficient but increase heap use; total in-flight events is
+          workers multiplied by batch size. Leave blank to use the value from logstash.yml.
+        title: pipeline.batch.size
+        regex: '^$|^[1-9][0-9]*$'
+        regexFailureMessage: Must be blank, or a positive whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_batch_x_delay:
+        description: >-
+          Milliseconds a worker waits for the next event before running a batch that is not yet full.
+          Leave blank to use the value from logstash.yml.
+        title: pipeline.batch.delay
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_batch_x_metrics_x_sampling_mode:
+        description: >-
+          Controls how often batch size metrics are collected for this pipeline, which helps tune
+          pipeline.batch.size to the batch sizes actually being processed. Fuller sampling consumes
+          additional heap. Elastic marks this setting as a technical preview that may change in a
+          future release. Leave blank to use the value from logstash.yml.
+        title: pipeline.batch.metrics.sampling_mode
+        options:
+        - ''
+        - 'disabled'
+        - 'minimal'
+        - 'full'
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_ordered:
+        description: >-
+          Whether event order is preserved through this pipeline. auto enables ordering only when
+          pipeline.workers is explicitly set to 1, and does nothing otherwise. Setting this to true
+          requires pipeline.workers to be 1 as well; with more workers this pipeline fails to start.
+          Leave blank to use the value from logstash.yml.
+        title: pipeline.ordered
+        options:
+        - ''
+        - 'auto'
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_ecs_compatibility:
+        description: >-
+          Elastic Common Schema compatibility mode for plugins in this pipeline. Security Onion sets
+          this globally and it should rarely be changed per pipeline. Elastic considers values other
+          than disabled to be BETA, and they may produce unintended consequences when upgrading
+          Logstash. Leave blank to use the value from logstash.yml.
+        title: pipeline.ecs_compatibility
+        options:
+        - ''
+        - 'disabled'
+        - 'v1'
+        - 'v8'
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_reloadable:
+        description: >-
+          Whether this pipeline may be reloaded when its configuration changes. Leave blank to use the
+          value from logstash.yml.
+        title: pipeline.reloadable
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_type:
+        description: >-
+          Queue backing this pipeline. persisted buffers events to disk under /nsm/logstash so they
+          survive a restart, at some throughput cost; memory does not. Leave blank to use the value
+          from logstash.yml.
+        title: queue.type
+        options:
+        - ''
+        - 'memory'
+        - 'persisted'
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_max_bytes:
+        description: >-
+          Total capacity of the persistent queue for this pipeline, in bytes. Only applies when
+          queue.type is persisted. The disk backing /nsm/logstash must have room for this much data or
+          the pipeline fails to start, reporting that it was unable to allocate the space. If both
+          queue.max_events and queue.max_bytes are set, whichever is reached first applies. Leave
+          blank to use the value from logstash.yml.
+        title: queue.max_bytes
+        regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$'
+        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb, or 64k. Units are lowercase.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_page_capacity:
+        description: >-
+          Size of the individual append-only page data files that make up the persistent queue for
+          this pipeline. Only applies when queue.type is persisted. Leave blank to use the value from
+          logstash.yml.
+        title: queue.page_capacity
+        regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$'
+        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb, or 64k. Units are lowercase.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_max_events:
+        description: >-
+          Maximum number of unread events in the persistent queue for this pipeline. 0 means
+          unlimited. Only applies when queue.type is persisted. Leave blank to use the value from
+          logstash.yml.
+        title: queue.max_events
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_checkpoint_x_acks:
+        description: >-
+          Maximum number of acknowledged events before a checkpoint is forced. 0 means unlimited. Only
+          applies when queue.type is persisted. Leave blank to use the value from logstash.yml.
+        title: queue.checkpoint.acks
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_checkpoint_x_writes:
+        description: >-
+          Maximum number of written events before a checkpoint is forced. Setting this to 1 gives
+          maximum durability at a severe performance cost. 0 means unlimited. Only applies when
+          queue.type is persisted. Leave blank to use the value from logstash.yml.
+        title: queue.checkpoint.writes
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_checkpoint_x_interval:
+        description: >-
+          Milliseconds between forced checkpoints on the persistent queue head page. 0 eliminates
+          periodic checkpoints. Deprecated by Elastic as of Logstash 9.1. Only applies when queue.type
+          is persisted. Leave blank to use the value from logstash.yml.
+        title: queue.checkpoint.interval
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_checkpoint_x_retry:
+        description: >-
+          When enabled, Logstash retries four times per attempted checkpoint write that fails; later
+          errors are not retried. Elastic describes this as a workaround for failed checkpoint writes
+          seen only on Windows and on filesystems with non-standard behaviour such as SANs, and does
+          not recommend enabling it otherwise. Only applies when queue.type is persisted. Leave blank
+          to use the value from logstash.yml.
+        title: queue.checkpoint.retry
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_compression:
+        description: >-
+          Compression applied to persistent queue pages for this pipeline, trading CPU for disk: speed
+          favours the fastest operation, size the smallest files, and balanced sits between them. Once
+          compressed events have been written, that queue cannot be read by Logstash releases earlier
+          than 9.2. Only applies when queue.type is persisted. Leave blank to use the value from
+          logstash.yml.
+        title: queue.compression
+        options:
+        - ''
+        - 'none'
+        - 'speed'
+        - 'balanced'
+        - 'size'
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_drain:
+        description: >-
+          When enabled, Logstash waits for the persistent queue to drain before shutting down this
+          pipeline. Draining a large queue makes shutdown take considerably longer. Only applies when
+          queue.type is persisted. Leave blank to use the value from logstash.yml.
+        title: queue.drain
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_enable:
+        description: >-
+          Whether events this pipeline cannot process are written to a dead letter queue instead of
+          being dropped. Leave blank to use the value from logstash.yml.
+        title: dead_letter_queue.enable
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_max_bytes:
+        description: >-
+          Total capacity of the dead letter queue for this pipeline, in bytes. Only applies when
+          dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
+        title: dead_letter_queue.max_bytes
+        regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$'
+        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb, or 64k. Units are lowercase.
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_flush_interval:
+        description: >-
+          Milliseconds before an incomplete dead letter queue segment is flushed and made available to
+          the dead_letter_queue input. Lower values write more, smaller segment files; higher values
+          add latency before events can be read. Only applies when dead_letter_queue.enable is true.
+          Leave blank to use the value from logstash.yml.
+        title: dead_letter_queue.flush_interval
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_flush_check_interval:
+        description: >-
+          Milliseconds between checks for a stale dead letter queue segment needing a flush. Cannot be
+          set lower than 1000. Smaller values rotate segments sooner at the cost of CPU. Only applies
+          when dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
+        title: dead_letter_queue.flush_check_interval
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_storage_policy:
+        description: >-
+          Action taken when dead_letter_queue.max_bytes is reached: drop_newer stops accepting new
+          events, drop_older removes the oldest events to make room. Only applies when
+          dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
+        title: dead_letter_queue.storage_policy
+        options:
+        - ''
+        - 'drop_newer'
+        - 'drop_older'
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_retain_x_age:
+        description: >-
+          How long an event is kept in the dead letter queue before Logstash removes it, such as 5d.
+          Units are d, h, m and s; there is no default unit, so one must be given. Only applies when
+          dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
+        title: dead_letter_queue.retain.age
+        regex: '^$|^[0-9]+\s*[dhms]$'
+        regexFailureMessage: Must be blank, or a number followed by d, h, m, or s, such as 5d.
+        advanced: True
+        global: False
+        helpLink: logstash
+      path_x_queue:
+        description: >-
+          Directory inside the Logstash container holding the persistent queue for this pipeline. The
+          default lives under the /nsm/logstash bind mount; a path outside it will not survive a
+          container restart. Logstash creates the directory if it is missing, requires it to be
+          writable, and refuses to start if the path is a symlink. Only applies when queue.type is
+          persisted. Leave blank to use the value from logstash.yml.
+        title: path.queue
+        advanced: True
+        global: False
+        helpLink: logstash
+      path_x_dead_letter_queue:
+        description: >-
+          Directory inside the Logstash container holding the dead letter queue for this pipeline. The
+          default lives under the /nsm/logstash bind mount; a path outside it will not survive a
+          container restart. Logstash creates the directory if it is missing, requires it to be
+          writable, and refuses to start if the path is a symlink. Only applies when
+          dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
+        title: path.dead_letter_queue
+        advanced: True
+        global: False
+        helpLink: logstash
+      config_x_debug:
+        description: >-
+          Whether the fully compiled configuration for this pipeline is written to the log. The output
+          may contain sensitive values from the pipeline configuration. Leave blank to use the value
+          from logstash.yml.
+        title: config.debug
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      config_x_support_escapes:
+        description: >-
+          Whether escape sequences such as \n and \t in this pipeline's quoted strings are
+          interpreted. Leave blank to use the value from logstash.yml.
+        title: config.support_escapes
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+    fleet: *pipeline_settings
+    receiver: *pipeline_settings
+    search: *pipeline_settings
+    custom0: *pipeline_settings
+    custom1: *pipeline_settings
+    custom2: *pipeline_settings
+    custom3: *pipeline_settings
+    custom4: *pipeline_settings
+    custom5: *pipeline_settings
+    custom6: *pipeline_settings
+    custom7: *pipeline_settings
+    custom8: *pipeline_settings
+    custom9: *pipeline_settings
   settings:
     lsheap: 
       description: Heap size to use for logstash
@@ -62,6 +413,35 @@ logstash:
       helpLink: logstash
       readonly: True
       advanced: True
+    log_x_level:
+      description: >-
+        Verbosity of the Logstash log at /opt/so/log/logstash/logstash.log. debug and trace produce
+        a very large volume of log data on a busy node and should be used only while troubleshooting;
+        the log rotates at 1GB and rotated files are deleted after 7 days. Setting this to debug is
+        also what makes the per-pipeline config.debug setting emit anything.
+      title: log.level
+      options:
+      - 'fatal'
+      - 'error'
+      - 'warn'
+      - 'info'
+      - 'debug'
+      - 'trace'
+      advanced: True
+      global: False
+      helpLink: logstash
+    log_x_format:
+      description: >-
+        Layout of the Logstash log. plain writes human readable lines; json writes one JSON object
+        per line, which is easier to parse but harder to read directly. The file name and location
+        do not change.
+      title: log.format
+      options:
+      - 'plain'
+      - 'json'
+      advanced: True
+      global: False
+      helpLink: logstash
     path_x_logs: 
       description: Path inside the container to wrote logs.
       helpLink: logstash

salt/manager/files/beacons_pushstate.conf.jinja

@@ -3,9 +3,16 @@ beacons:
   postgres_pillar_beacon:
     - interval: {{ AUTOAPPLY.drain_interval }}
     - disable_during_state_run: False
-  rules_beacon:
+  local_files_beacon:
     - interval: {{ AUTOAPPLY.drain_interval }}
     - disable_during_state_run: False
+    # Tags are app names in salt/reactor/pillar_push_map.yaml.
+    # Allowlist on purpose: salt writes elsewhere under local/salt/ and would self-retrigger.
     - paths:
         /opt/so/saltstack/local/salt/suricata/rules: suricata
         /opt/so/saltstack/local/salt/strelka/rules/compiled: strelka
+        /opt/so/saltstack/local/salt/zeek/policy: zeek
+        /opt/so/saltstack/local/salt/zeek/zkg: zeek
+        /opt/so/saltstack/local/salt/elasticsearch/files/ingest: elasticsearch
+        /opt/so/saltstack/local/salt/elasticsearch/roles: elasticsearch
+        /opt/so/saltstack/local/salt/logstash/pipelines/config/custom: logstash

salt/manager/tools/sbin/so-minion

@@ -138,6 +138,8 @@ function getinstallinfo() {
 		log "ERROR" "Failed to source install variables"
 		return 1
 	fi
+
+	log "INFO" "Fetched install info for $MINION_ID (node type: ${NODETYPE:-unset})"
 }
 
 function pcapspace() {
@@ -483,6 +485,7 @@ function add_sensoroni_with_analyze_to_minion() {
 
 # Sensor settings for the minion pillar
 function add_sensor_to_minion() {
+	log "INFO" "Writing sensor configuration for $MINION_ID (interface: ${INTERFACE:-unset})"
     {
         echo "sensor:"
         echo "  interface: '$INTERFACE'"
@@ -509,6 +512,8 @@ function add_sensor_to_minion() {
         log "ERROR" "Failed to add sensor configuration to $PILLARFILE"
         return 1
     fi
+
+	log "INFO" "Wrote sensor configuration for $MINION_ID"
 }
 
 function add_elastalert_to_minion() {
@@ -581,11 +586,14 @@ function add_telegraf_to_minion() {
     # generates a password on first add and is a no-op on re-add so the cred
     # is stable across repeated so-minion runs. postgres.telegraf_users on the
     # manager creates/updates the DB role from the same pillar.
-    so-telegraf-cred add "$MINION_ID"
-    if [ $? -ne 0 ]; then
-        log "ERROR" "Failed to provision postgres telegraf cred for $MINION_ID"
-        return 1
-    fi
+	log "INFO" "Provisioning postgres telegraf credential for $MINION_ID"
+	so-telegraf-cred add "$MINION_ID"
+	local result=$?
+	if [ $result -ne 0 ]; then
+		log "ERROR" "Failed to provision postgres telegraf cred for $MINION_ID (exit code: $result)"
+		return 1
+	fi
+	log "INFO" "Provisioned postgres telegraf credential for $MINION_ID"
 }
 
 function add_influxdb_to_minion() {
@@ -1043,7 +1051,7 @@ function updateMineAndApplyStates() {
 }
 
 function setupMinionFiles() {
-	log "INFO" "Setting up minion files for $MINION_ID"
+	log "INFO" "Setting up minion files for $MINION_ID (pillar: $PILLARFILE)"
 	
 	# Check to see if nodetype is set
 	if [ -z $NODETYPE ]; then
@@ -1069,7 +1077,10 @@ function setupMinionFiles() {
 	fi
 
 	# Create node-specific configuration
-	create$NODETYPE || return 1
+	create$NODETYPE || {
+		log "ERROR" "Failed to create $NODETYPE configuration for $MINION_ID"
+		return 1
+	}
 	
 	# Ensure proper ownership after all content is written
 	ensure_socore_ownership || return 1

salt/manager/tools/sbin/so-push-drainer

@@ -20,7 +20,7 @@ is older than debounce_seconds, this script:
     with the deduped actions list passed as pillar kwargs
   * deletes the contributed intent files on successful dispatch
 
-Reactor sls files (push_suricata, push_strelka, push_pillar) write intents
+Reactor sls files (push_files, push_pillar) write intents
 but never dispatch directly
 """
 

salt/manager/tools/sbin/so-soup-grid-highstate

@@ -0,0 +1,219 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+# so-soup-grid-highstate
+# ======================
+# Drives a batched, role-tiered highstate across every non-manager minion in the
+# grid. soup fires this (detached) after it finishes upgrading the manager so the
+# rest of the grid converges immediately instead of waiting for its own scheduled
+# highstate -- which, since the schedule moved from 15 minutes to 120 minutes
+# (salt:schedule:highstate_interval_minutes), could otherwise leave nodes on the
+# old version for up to ~2.5 hours (interval + splay) while the manager runs new code.
+#
+# Work is done by the existing orch.push_batch orchestration (salt/orch/push_batch.sls),
+# the same runner the active-push drainer uses, so batching/queueing behavior matches.
+# Tiers are dispatched in declaration order: searchnodes/heavynodes (Elasticsearch data
+# nodes) first, then receivers, then everything else -- so the data tier converges before
+# the ingest tier before sensors/fleet/idh/etc.
+#
+# When soup also upgraded Salt itself, remote minions must first highstate onto the new
+# salt-minion package (top.sls gates every real state on G@saltversion, so a stale-version
+# minion only gets salt.minion until it upgrades and reconnects). --salt-upgraded runs that
+# preliminary pass and waits for the fleet to settle before the tiered pass.
+#
+# This is best-effort: soup has already completed by the time this runs, and the 120-minute
+# scheduled highstate remains the backstop for any node that is offline or missed a batch.
+
+LOG_FILE=/opt/so/log/salt/so-soup-grid-highstate
+LOCK_FILE=/opt/so/state/so-soup-grid-highstate.lock
+SETTLE_MAX_WAIT=${GRID_HIGHSTATE_SETTLE_WAIT:-900}   # backstop for the post-salt-upgrade settle loop
+SETTLE_INTERVAL=15
+SETTLE_STABLE_CHECKS=3
+# salt-minion on an upgraded node restarts ~30s after the upgrade state runs
+# (salt/salt/minion/init.sls start_minion_post_upgrade); wait past that before sampling
+# so the settle loop sees the drop-off instead of settling on the pre-restart set.
+SETTLE_INITIAL_WAIT=${GRID_HIGHSTATE_SETTLE_INITIAL_WAIT:-45}
+
+BATCH=""
+BATCH_WAIT=""
+SALT_UPGRADED=false
+REASON="manual"
+
+log() {
+  echo "$(date '+%Y-%m-%d %H:%M:%S') | $*" | tee -a "$LOG_FILE"
+}
+
+usage() {
+  echo "Usage: so-soup-grid-highstate [--batch <spec>] [--batch-wait <sec>] [--salt-upgraded] [--reason <text>]"
+  exit 1
+}
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --batch)        BATCH="$2"; shift 2 ;;
+    --batch-wait)   BATCH_WAIT="$2"; shift 2 ;;
+    --salt-upgraded) SALT_UPGRADED=true; shift ;;
+    --reason)       REASON="$2"; shift 2 ;;
+    -h|--help)      usage ;;
+    *)              echo "Unknown option: $1"; usage ;;
+  esac
+done
+
+mkdir -p "$(dirname "$LOG_FILE")" "$(dirname "$LOCK_FILE")"
+
+# Serialize: a second invocation (e.g. two soups, or a manual run overlapping soup's)
+# should not dispatch a competing set of batches.
+exec 9>"$LOCK_FILE"
+if ! flock -n 9; then
+  log "another so-soup-grid-highstate is already running (lock $LOCK_FILE held); exiting"
+  exit 0
+fi
+
+# Resolve batch settings from the salt:auto_apply pillar when not overridden on the
+# command line, falling back to the same defaults orch.push_batch/salt.defaults use.
+if [ -z "$BATCH" ]; then
+  BATCH=$(salt-call --out=newline_values_only pillar.get salt:auto_apply:batch 2>/dev/null)
+  [ -z "$BATCH" ] && BATCH='10%'
+fi
+if [ -z "$BATCH_WAIT" ]; then
+  BATCH_WAIT=$(salt-call --out=newline_values_only pillar.get salt:auto_apply:batch_wait 2>/dev/null)
+  [ -z "$BATCH_WAIT" ] && BATCH_WAIT=15
+fi
+
+MINIONID=$(salt-call --local --out=newline_values_only grains.get id 2>/dev/null)
+[ -z "$MINIONID" ] && MINIONID=$(cat /etc/salt/minion_id 2>/dev/null)
+if [ -z "$MINIONID" ]; then
+  log "could not determine this minion's id; aborting"
+  exit 1
+fi
+
+# Single-node grids (eval/standalone/import with no other accepted keys) have nothing
+# remote to push -- the manager already highstated during soup.
+NUM_ACCEPTED=$(salt-key --out=json --list=accepted 2>/dev/null | jq -r '.minions | length' 2>/dev/null)
+NUM_ACCEPTED=${NUM_ACCEPTED:-0}
+if [ "$NUM_ACCEPTED" -le 1 ]; then
+  log "single node grid ($NUM_ACCEPTED accepted minion(s)); nothing to push (reason=$REASON)"
+  exit 0
+fi
+
+log "starting grid highstate: reason=$REASON minion=$MINIONID accepted=$NUM_ACCEPTED batch=$BATCH batch_wait=$BATCH_WAIT salt_upgraded=$SALT_UPGRADED"
+
+# Count minions currently responsive on the bus (includes this manager).
+count_up() {
+  salt-run manage.up --out=json 2>/dev/null \
+    | python3 -c 'import sys,json; print(len(json.load(sys.stdin)))' 2>/dev/null
+}
+
+# Dispatch a single synchronous orch.push_batch run for the given actions JSON.
+# Synchronous is fine: soup launched us detached, so blocking here does not hold soup up.
+# expect_restart=true marks a dispatch (the salt-upgrade pass) where a non-zero rc is normal
+# because targets restart salt-minion mid-run -- so we don't log a misleading failure warning.
+dispatch() {
+  local desc="$1"
+  local actions="$2"
+  local expect_restart="${3:-false}"
+  local rc
+  log "dispatching $desc"
+  salt-run state.orchestrate orch.push_batch pillar="{\"actions\": $actions}" >>"$LOG_FILE" 2>&1
+  rc=$?
+  if [ "$rc" -eq 0 ]; then
+    log "$desc dispatch completed (rc=0)"
+  elif [ "$expect_restart" = "true" ]; then
+    log "$desc returned rc=$rc; this is expected during a salt upgrade (targets restart salt-minion mid-run). Waiting for them to reconnect before the tiered pass."
+  else
+    log "WARNING: $desc dispatch returned rc=$rc; nodes it missed will converge on the scheduled highstate"
+  fi
+}
+
+# Wait for the reachable minion set to recover to its pre-upgrade size and hold steady.
+# Used after the salt-upgrade pass, where targets restart salt-minion (~30s delayed, see
+# salt/salt/minion/init.sls) and drop off the bus before reconnecting on the new version.
+# target = how many minions were reachable just before the pass; requiring up >= target keeps
+# us from releasing the tiered pass while nodes are still down for their restart (settling on
+# the not-yet-restarted subset). We deliberately compare against the pre-upgrade reachable
+# count, not accepted keys, so a node an operator intentionally powered off never stalls us.
+# Bounded by SETTLE_MAX_WAIT.
+wait_for_settle() {
+  local target="$1"
+  local elapsed=0 prev=-1 stable=0 up=0
+  # Let the delayed salt-minion restart begin before we start counting stability, otherwise
+  # we could see the pre-restart set as "stable" and settle before the drop-off even happens.
+  sleep "$SETTLE_INITIAL_WAIT"
+  elapsed=$SETTLE_INITIAL_WAIT
+  while [ "$elapsed" -lt "$SETTLE_MAX_WAIT" ]; do
+    up=$(count_up); up=${up:-0}
+    if [ "$up" -ge "$target" ] && [ "$up" -eq "$prev" ]; then
+      stable=$((stable + 1))
+      [ "$stable" -ge "$SETTLE_STABLE_CHECKS" ] && break
+    else
+      stable=0
+    fi
+    prev=$up
+    sleep "$SETTLE_INTERVAL"
+    elapsed=$((elapsed + SETTLE_INTERVAL))
+  done
+  if [ "$up" -ge "$target" ]; then
+    log "fleet recovered to ${up} minions up (>= pre-upgrade ${target}) after ${elapsed}s"
+  else
+    log "WARNING: ${SETTLE_MAX_WAIT}s settle backstop hit; only ${up}/${target} pre-upgrade minions back up; proceeding (stragglers converge on the scheduled highstate)"
+  fi
+}
+
+# Pass 0: when Salt itself was upgraded, remote minions still on the old version only match
+# top.sls's 'not G@saltversion' block (salt.minion, which performs the package upgrade). Push
+# an untiered highstate so they upgrade+reconnect, then wait for them to come back before the
+# real tiered pass applies the new version's states.
+if [ "$SALT_UPGRADED" = "true" ]; then
+  PRE_UP=$(count_up); PRE_UP=${PRE_UP:-1}
+  log "pre-upgrade reachable minions (incl. this manager): $PRE_UP"
+  dispatch "salt-upgrade pass (all remote minions)" \
+    "[{\"highstate\": true, \"tgt\": \"not $MINIONID\", \"tgt_type\": \"compound\", \"batch\": \"$BATCH\", \"batch_wait\": $BATCH_WAIT}]" \
+    true
+  log "waiting for minions to reconnect on the new salt version"
+  wait_for_settle "$PRE_UP"
+fi
+
+# Tiered pass: Elasticsearch data nodes first, then receivers, then the remainder. The last
+# tier is defined as the complement of the earlier tiers (and of this manager) so coverage is
+# exhaustive -- sensors, fleet, idh, desktop, hypervisor, and any future role are all included.
+TIER_TGTS=(
+  "( *_searchnode or *_heavynode ) and not $MINIONID"
+  "*_receiver and not $MINIONID"
+  "not $MINIONID and not *_searchnode and not *_heavynode and not *_receiver"
+)
+
+# Count minions a compound target matches, using the master's key/cache data (no execution).
+tier_count() {
+  salt --out=json -C "$1" --preview-target 2>/dev/null | jq 'length' 2>/dev/null
+}
+
+# Build the actions JSON, including only tiers that actually match minions. An empty target
+# would make orch.push_batch's salt.state step return "No minions returned" -- a failure --
+# even though nothing needed to run, and grids commonly lack a tier (no receiver, etc.).
+# Keep the JSON on a single line: salt parses `pillar=<value>` kwargs with a non-DOTALL
+# regex, so an embedded newline makes it treat the whole token as a positional saltenv
+# instead ("No matching salt environment for environment 'pillar=...'").
+actions=""
+for tgt in "${TIER_TGTS[@]}"; do
+  n=$(tier_count "$tgt"); n=${n:-0}
+  if [ "$n" -ge 1 ]; then
+    [ -n "$actions" ] && actions="$actions, "
+    actions="$actions{\"highstate\": true, \"tgt\": \"$tgt\", \"tgt_type\": \"compound\", \"batch\": \"$BATCH\", \"batch_wait\": $BATCH_WAIT}"
+    log "tier matched $n minion(s): $tgt"
+  else
+    log "tier matched 0 minions, skipping: $tgt"
+  fi
+done
+
+if [ -z "$actions" ]; then
+  log "no remote minions matched any tier; nothing to push (reason=$REASON)"
+  exit 0
+fi
+dispatch "tiered pass (searchnodes/heavynodes -> receivers -> remainder)" "[$actions]"
+
+log "grid highstate complete (reason=$REASON)"
+exit 0

salt/manager/tools/sbin/soup

@@ -24,7 +24,10 @@ else
   POSTVERSION=$INSTALLEDVERSION
 fi
 INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
-BATCHSIZE=5
+# Optional -b override for the grid highstate batch size (a count like "5" or a
+# percentage like "25%"). Empty means so-soup-grid-highstate uses the salt:auto_apply:batch
+# pillar default.
+BATCHSIZE=
 SOUP_LOG=/root/soup.log
 SOUP_DEBUG_LOG=/root/soup-debug.log
 WHATWOULDYOUSAYYAHDOHERE=soup
@@ -452,6 +455,31 @@ highstate() {
   salt-call state.highstate -l info queue=True
 }
 
+push_grid_highstate() {
+  # Drive a batched, role-tiered highstate across the rest of the grid so remote minions
+  # pick up this upgrade now instead of waiting up to ~2.5 hours for their own scheduled
+  # highstate (the schedule moved from 15 to 120 minutes). so-soup-grid-highstate does the work
+  # via orch.push_batch; it only exists once the manager highstate has deployed this
+  # version's sbin files, so guard on it. Launch fully detached (setsid) so it survives an
+  # SSH drop, and never let it affect soup's exit status -- it is best-effort with the
+  # scheduled highstate as backstop.
+  if [[ ! -x /usr/sbin/so-soup-grid-highstate ]]; then
+    echo "so-soup-grid-highstate not present; remote nodes will converge on their scheduled highstate."
+    return 0
+  fi
+
+  local extra_args=()
+  if [[ $SALTUPGRADED == true || $UPGRADESALT -eq 1 ]]; then
+    extra_args+=(--salt-upgraded)
+  fi
+  if [[ -n "$BATCHSIZE" ]]; then
+    extra_args+=(--batch "$BATCHSIZE")
+  fi
+
+  echo "Dispatching a grid-wide highstate to remote nodes. Progress: /opt/so/log/salt/so-soup-grid-highstate"
+  setsid nohup /usr/sbin/so-soup-grid-highstate --reason soup "${extra_args[@]}" >/dev/null 2>&1 &
+}
+
 masterlock() {
   echo "Locking Salt Master"
   mv -v $TOPFILE $BACKUPTOPFILE
@@ -1969,6 +1997,9 @@ main() {
     # rather than reporting "already latest". The soversion/pillar writes in
     # update_version are no-ops here since the version is unchanged for a hotfix.
     update_version
+    # Push the hotfix out to the rest of the grid rather than waiting for the scheduled
+    # highstate. Hotfixes never upgrade Salt, so no --salt-upgraded pass is needed.
+    push_grid_highstate
   else
     SOUP_UPGRADE_STARTED=true
     echo ""
@@ -2135,13 +2166,18 @@ main() {
 
     if [[ $NUM_MINIONS -gt 1 ]]; then
 
+      # Actively drive the rest of the grid to this version now. The scheduled highstate
+      # runs only every 120 minutes (salt:schedule:highstate_interval_minutes), so without
+      # this remote nodes could sit on the old version for a couple of hours after soup finishes.
+      push_grid_highstate
+
       cat << EOF
 
 
 
-This appears to be a distributed deployment. Other nodes should update themselves at the next Salt highstate (typically within 15 minutes). Do not manually restart anything until you know that all the search/heavy nodes in your deployment are updated. This is especially important if you are using true clustering for Elasticsearch.
+This appears to be a distributed deployment. soup has dispatched a batched, grid-wide highstate to update the other nodes now: Elasticsearch data nodes (search/heavynodes) first, then receivers, then sensors and the remaining nodes. Progress is logged to /opt/so/log/salt/so-soup-grid-highstate, and you can watch nodes update from the Grid section of SOC. Do not manually restart anything until you know that all the search/heavynodes in your deployment are updated. This is especially important if you are using true clustering for Elasticsearch.
 
-Each minion is on a random 15 minute check-in period and things like network bandwidth can be a factor in how long the actual upgrade takes. If you have a heavy node on a slow link, it is going to take a while to get the containers to it. Depending on what changes happened between the versions, Elasticsearch might not be able to talk to said heavy node until the update is complete.
+Nodes are updated in batches, and things like network bandwidth can be a factor in how long the actual upgrade takes. If you have a heavy node on a slow link, it is going to take a while to get the containers to it. Depending on what changes happened between the versions, Elasticsearch might not be able to talk to said heavy node until the update is complete. Any node that is offline or missed a batch will converge on its own scheduled highstate (every 120 minutes by default).
 
 If it looks like you’re missing data after the upgrade, please avoid restarting services and instead make sure at least one search node has completed its upgrade. The best way to do this is to run 'sudo salt-call state.highstate' from a search node and make sure there are no errors. Typically if it works on one node it will work on the rest. Sensor nodes are less complex and will update as they check in so you can monitor those from the Grid section of SOC.
 
@@ -2181,8 +2217,10 @@ while getopts ":b:f:y" opt; do
   case ${opt} in
     b )
       BATCHSIZE="$OPTARG"
-      if ! [[ "$BATCHSIZE" =~ ^[1-9][0-9]*$ ]]; then
-        echo "Batch size must be a number greater than 0."
+      # Accept either a plain count (e.g. 5) or a percentage (e.g. 25%); passed through
+      # to so-soup-grid-highstate --batch, which salt's batch/batch_wait accepts in both forms.
+      if ! [[ "$BATCHSIZE" =~ ^[1-9][0-9]*%?$ ]]; then
+        echo "Batch size must be a number greater than 0, optionally with a trailing % (e.g. 5 or 25%)."
         exit 1
       fi
     ;;

salt/nginx/etc/nginx.conf

@@ -260,7 +260,7 @@ http {
 		}
 
 {%  if 'api' in salt['pillar.get']('features', []) %}
-		location ~* (^/oauth2/token.*|^.well-known/jwks.json|^.well-known/openid-configuration) {
+		location ~* (^/oauth2/token.*|^/\.well-known/jwks.json|^/\.well-known/openid-configuration) {
 			limit_req             zone=auth_throttle burst={{ NGINXMERGED.config.throttle_login_burst }} nodelay;
 			limit_req_status      429;
 			proxy_pass            http://{{ GLOBALS.manager }}:4444;

salt/postgres/files/init-db.sh

@@ -29,6 +29,8 @@ psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-E
     -- revoking CONNECT closes the soft edge entirely.
     REVOKE CONNECT ON DATABASE "$POSTGRES_DB" FROM PUBLIC;
     GRANT CONNECT ON DATABASE "$POSTGRES_DB" TO "$SO_POSTGRES_USER";
+
+    CREATE EXTENSION IF NOT EXISTS vector;
 EOSQL
 
 # Bootstrap the Telegraf metrics database. Per-minion roles + schemas are

salt/reactor/pillar_push_map.yaml

@@ -1,3 +1,6 @@
+# Read by push_pillar.sls (SOC config saves) and push_files.sls (local/salt file edits);
+# both key on the app name. An app missing here waits for the next scheduled highstate.
+#
 # One pillar directory can map to multiple (state, tgt) actions.
 # tgt is a raw salt compound expression. tgt_type is always "compound".
 # Per-action `batch` / `batch_wait` override the orch defaults (25% / 15s).

salt/reactor/push_files.sls

@@ -0,0 +1,131 @@
+#!py
+
+# Reactor invoked by local_files_beacon when a watched directory under
+# /opt/so/saltstack/local/salt/ changes. The beacon tag is an app name in
+# pillar_push_map.yaml, so file changes and pillar changes route through the same
+# table -- see salt/reactor/push_pillar.sls.
+#
+# The app comes from the event tag, not the payload: salt's beacon loop pops the
+# beacon's 'tag' key off the data and appends it to the event tag instead (see
+# salt/beacons/__init__.py). The reactor renderer sets both `tag` and `data` as
+# module globals.
+#
+# Reactors never dispatch directly. The so-push-drainer schedule picks up ready
+# intents, dedupes across pending files, and dispatches orch.push_batch.
+
+import fcntl
+import json
+import logging
+import os
+import time
+
+from salt.client import Caller
+import yaml
+
+LOG = logging.getLogger(__name__)
+
+PENDING_DIR = '/opt/so/state/push_pending'
+LOCK_FILE = os.path.join(PENDING_DIR, '.lock')
+MAX_PATHS = 20
+
+# The pillar_push_map.yaml is shipped via salt:// but the reactor runs on the
+# master, which mounts the default saltstack tree at this path.
+PUSH_MAP_PATH = '/opt/so/saltstack/default/salt/reactor/pillar_push_map.yaml'
+
+_PUSH_MAP_CACHE = {'mtime': 0, 'data': None}
+
+
+def _load_push_map():
+    try:
+        st = os.stat(PUSH_MAP_PATH)
+    except OSError:
+        LOG.warning('push_files: %s not found', PUSH_MAP_PATH)
+        return {}
+    if _PUSH_MAP_CACHE['mtime'] != st.st_mtime:
+        try:
+            with open(PUSH_MAP_PATH, 'r') as f:
+                _PUSH_MAP_CACHE['data'] = yaml.safe_load(f) or {}
+        except Exception:
+            LOG.exception('push_files: failed to load %s', PUSH_MAP_PATH)
+            _PUSH_MAP_CACHE['data'] = {}
+        _PUSH_MAP_CACHE['mtime'] = st.st_mtime
+    return _PUSH_MAP_CACHE['data'] or {}
+
+
+def _push_enabled():
+    try:
+        caller = Caller()
+        return bool(caller.cmd('pillar.get', 'salt:auto_apply:enabled', True))
+    except Exception:
+        LOG.exception('push_files: pillar.get salt:auto_apply:enabled failed, assuming enabled')
+        return True
+
+
+def _write_intent(key, actions, path):
+    now = time.time()
+    try:
+        os.makedirs(PENDING_DIR, exist_ok=True)
+    except OSError:
+        LOG.exception('push_files: cannot create %s', PENDING_DIR)
+        return
+
+    intent_path = os.path.join(PENDING_DIR, '{}.json'.format(key))
+    lock_fd = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR, 0o644)
+    try:
+        fcntl.flock(lock_fd, fcntl.LOCK_EX)
+
+        intent = {}
+        if os.path.exists(intent_path):
+            try:
+                with open(intent_path, 'r') as f:
+                    intent = json.load(f)
+            except (IOError, ValueError):
+                intent = {}
+
+        intent.setdefault('first_touch', now)
+        intent['last_touch'] = now
+        intent['actions'] = actions
+        paths = intent.get('paths', [])
+        if path and path not in paths:
+            paths.append(path)
+            paths = paths[-MAX_PATHS:]
+        intent['paths'] = paths
+
+        tmp_path = intent_path + '.tmp'
+        with open(tmp_path, 'w') as f:
+            json.dump(intent, f)
+        os.rename(tmp_path, intent_path)
+    except Exception:
+        LOG.exception('push_files: failed to write intent %s', intent_path)
+    finally:
+        try:
+            fcntl.flock(lock_fd, fcntl.LOCK_UN)
+        finally:
+            os.close(lock_fd)
+
+
+def run():
+    if not _push_enabled():
+        LOG.info('push_files: push disabled, skipping')
+        return {}
+
+    event = data.get('data', data)  # noqa: F821 -- data provided by reactor
+    path = event.get('path', '')
+    app = tag.rsplit('/', 1)[-1].strip()  # noqa: F821 -- tag provided by reactor
+
+    if not app:
+        LOG.debug('push_files: ignoring event with no app segment: tag=%s', tag)  # noqa: F821
+        return {}
+
+    entry = _load_push_map().get(app)
+    if not entry:
+        LOG.warning(
+            'push_files: app "%s" is not in pillar_push_map.yaml; change will be '
+            'picked up at the next scheduled highstate (path=%s)',
+            app, path,
+        )
+        return {}
+
+    _write_intent('files_{}'.format(app), list(entry), path)
+    LOG.info('push_files: intent updated for %s (path=%s)', app, path)
+    return {}

salt/reactor/push_strelka.sls

@@ -1,96 +0,0 @@
-#!py
-
-# Reactor invoked by the rules_beacon poll beacon (salt/_beacons/rules_beacon.py) on rule
-# file changes under /opt/so/saltstack/local/salt/strelka/rules/compiled/.
-#
-# Writes (or updates) a push intent at /opt/so/state/push_pending/rules_strelka.json
-# and returns {}. The so-push-drainer schedule picks up ready intents, dedupes
-# across pending files, and dispatches orch.push_batch. Reactors never dispatch
-# directly
-
-import fcntl
-import json
-import logging
-import os
-import time
-
-from salt.client import Caller
-
-LOG = logging.getLogger(__name__)
-
-PENDING_DIR = '/opt/so/state/push_pending'
-LOCK_FILE = os.path.join(PENDING_DIR, '.lock')
-MAX_PATHS = 20
-
-# Mirrors GLOBALS.sensor_roles in salt/vars/globals.map.jinja. Sensor-side
-# strelka runs on exactly these four roles; so-import gets strelka.manager
-# instead, which is not fired on pillar changes.
-SENSOR_ROLES = ['so-eval', 'so-heavynode', 'so-sensor', 'so-standalone']
-
-
-def _sensor_compound():
-    return ' or '.join('G@role:{}'.format(r) for r in SENSOR_ROLES)
-
-
-def _push_enabled():
-    try:
-        caller = Caller()
-        return bool(caller.cmd('pillar.get', 'salt:auto_apply:enabled', True))
-    except Exception:
-        LOG.exception('push_strelka: pillar.get salt:auto_apply:enabled failed, assuming enabled')
-        return True
-
-
-def _write_intent(key, actions, path):
-    now = time.time()
-    try:
-        os.makedirs(PENDING_DIR, exist_ok=True)
-    except OSError:
-        LOG.exception('push_strelka: cannot create %s', PENDING_DIR)
-        return
-
-    intent_path = os.path.join(PENDING_DIR, '{}.json'.format(key))
-    lock_fd = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR, 0o644)
-    try:
-        fcntl.flock(lock_fd, fcntl.LOCK_EX)
-
-        intent = {}
-        if os.path.exists(intent_path):
-            try:
-                with open(intent_path, 'r') as f:
-                    intent = json.load(f)
-            except (IOError, ValueError):
-                intent = {}
-
-        intent.setdefault('first_touch', now)
-        intent['last_touch'] = now
-        intent['actions'] = actions
-        paths = intent.get('paths', [])
-        if path and path not in paths:
-            paths.append(path)
-            paths = paths[-MAX_PATHS:]
-        intent['paths'] = paths
-
-        tmp_path = intent_path + '.tmp'
-        with open(tmp_path, 'w') as f:
-            json.dump(intent, f)
-        os.rename(tmp_path, intent_path)
-    except Exception:
-        LOG.exception('push_strelka: failed to write intent %s', intent_path)
-    finally:
-        try:
-            fcntl.flock(lock_fd, fcntl.LOCK_UN)
-        finally:
-            os.close(lock_fd)
-
-
-def run():
-    if not _push_enabled():
-        LOG.info('push_strelka: push disabled, skipping')
-        return {}
-
-    path = data.get('path', '')  # noqa: F821 -- data provided by reactor
-    actions = [{'state': 'strelka', 'tgt': _sensor_compound()}]
-    _write_intent('rules_strelka', actions, path)
-    LOG.info('push_strelka: intent updated for path=%s', path)
-    return {}

salt/reactor/push_suricata.sls

@@ -1,95 +0,0 @@
-#!py
-
-# Reactor invoked by the rules_beacon poll beacon (salt/_beacons/rules_beacon.py) on rule
-# file changes under /opt/so/saltstack/local/salt/suricata/rules/.
-#
-# Writes (or updates) a push intent at /opt/so/state/push_pending/rules_suricata.json
-# and returns {}. The so-push-drainer schedule picks up ready intents, dedupes
-# across pending files, and dispatches orch.push_batch. Reactors never dispatch
-# directly
-
-import fcntl
-import json
-import logging
-import os
-import time
-
-from salt.client import Caller
-
-LOG = logging.getLogger(__name__)
-
-PENDING_DIR = '/opt/so/state/push_pending'
-LOCK_FILE = os.path.join(PENDING_DIR, '.lock')
-MAX_PATHS = 20
-
-# Mirrors GLOBALS.sensor_roles in salt/vars/globals.map.jinja. Suricata also
-# runs on so-import per salt/top.sls, so that role is appended below.
-SENSOR_ROLES = ['so-eval', 'so-heavynode', 'so-sensor', 'so-standalone']
-
-
-def _sensor_compound_plus_import():
-    return ' or '.join('G@role:{}'.format(r) for r in SENSOR_ROLES) + ' or G@role:so-import'
-
-
-def _push_enabled():
-    try:
-        caller = Caller()
-        return bool(caller.cmd('pillar.get', 'salt:auto_apply:enabled', True))
-    except Exception:
-        LOG.exception('push_suricata: pillar.get salt:auto_apply:enabled failed, assuming enabled')
-        return True
-
-
-def _write_intent(key, actions, path):
-    now = time.time()
-    try:
-        os.makedirs(PENDING_DIR, exist_ok=True)
-    except OSError:
-        LOG.exception('push_suricata: cannot create %s', PENDING_DIR)
-        return
-
-    intent_path = os.path.join(PENDING_DIR, '{}.json'.format(key))
-    lock_fd = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR, 0o644)
-    try:
-        fcntl.flock(lock_fd, fcntl.LOCK_EX)
-
-        intent = {}
-        if os.path.exists(intent_path):
-            try:
-                with open(intent_path, 'r') as f:
-                    intent = json.load(f)
-            except (IOError, ValueError):
-                intent = {}
-
-        intent.setdefault('first_touch', now)
-        intent['last_touch'] = now
-        intent['actions'] = actions
-        paths = intent.get('paths', [])
-        if path and path not in paths:
-            paths.append(path)
-            paths = paths[-MAX_PATHS:]
-        intent['paths'] = paths
-
-        tmp_path = intent_path + '.tmp'
-        with open(tmp_path, 'w') as f:
-            json.dump(intent, f)
-        os.rename(tmp_path, intent_path)
-    except Exception:
-        LOG.exception('push_suricata: failed to write intent %s', intent_path)
-    finally:
-        try:
-            fcntl.flock(lock_fd, fcntl.LOCK_UN)
-        finally:
-            os.close(lock_fd)
-
-
-def run():
-    if not _push_enabled():
-        LOG.info('push_suricata: push disabled, skipping')
-        return {}
-
-    path = data.get('path', '')  # noqa: F821 -- data provided by reactor
-    actions = [{'state': 'suricata', 'tgt': _sensor_compound_plus_import()}]
-    _write_intent('rules_suricata', actions, path)
-    LOG.info('push_suricata: intent updated for path=%s', path)
-    return {}

salt/salt/defaults.yaml

@@ -3,7 +3,7 @@ salt:
     enabled: true
     debounce_seconds: 30
     drain_interval: 15
-    batch: '25%'
+    batch: '10%'
     batch_wait: 15
   schedule:
     highstate_interval_minutes: 120

salt/salt/files/reactor_pushstate.conf

@@ -1,7 +1,5 @@
 reactor:
-  - 'salt/beacon/*/rules_beacon/suricata':
-    - salt://reactor/push_suricata.sls
-  - 'salt/beacon/*/rules_beacon/strelka':
-    - salt://reactor/push_strelka.sls
+  - 'salt/beacon/*/local_files_beacon/*':
+    - salt://reactor/push_files.sls
   - 'salt/beacon/*/postgres_pillar_beacon/audit_settings':
     - salt://reactor/push_pillar.sls

salt/salt/minion/boot_highstate.sls

@@ -3,6 +3,8 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
 # Manages /etc/systemd/system/so-boot-highstate.service, a Type=oneshot
 # RemainAfterExit=yes unit that runs `salt-call state.highstate` exactly once
 # per system boot. Replaces the legacy `startup_states: highstate` minion
@@ -19,9 +21,25 @@ so_boot_highstate_unit_file:
     - onchanges_in:
       - module: systemd_reload
 
+# Non-managers never apply salt.minion during setup, so reaching this state means
+# setup is finished and the marker is safe to write unconditionally. This also
+# heals nodes installed before this fix, which have no marker and no legacy
+# startup_states line to grep for. Managers do highstate mid-setup, so they only
+# get the marker from the legacy upgrade signal; fresh installs get it from
+# mark_setup_complete in setup/so-functions.
+mark_setup_complete:
+  file.managed:
+    - name: /opt/so/state/setup-complete
+    - replace: false
+    - makedirs: True
+{% if GLOBALS.is_manager %}
+    - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion"
+{% endif %}
+    - require_in:
+      - service: so_boot_highstate_service
+
 # Only enable once setup is complete. Until then the gate file is missing and
-# the unit's own ConditionPathExists would no-op it anyway -- this just keeps
-# `systemctl is-enabled` honest for the sync_es_users gate.
+# the unit's own ConditionPathExists would no-op it anyway.
 so_boot_highstate_service:
   service.enabled:
     - name: so-boot-highstate.service

salt/salt/minion/init.sls

@@ -87,27 +87,15 @@ set_log_levels:
 # so-boot-highstate.service (managed in salt.minion.boot_highstate), which
 # runs once per system boot only. Strip the line from /etc/salt/minion on
 # upgrade; both the commented and uncommented forms historically existed.
+# Ordered after mark_setup_complete (salt.minion.boot_highstate); the manager
+# gate there greps for this line, so it must run before we delete it.
 remove_startup_states:
   file.line:
     - name: /etc/salt/minion
     - match: 'startup_states: highstate'
     - mode: delete
-
-# Upgrade-path bridge: systems that already passed setup under the old gate
-# (`grep -x 'startup_states: highstate' /etc/salt/minion`) get a /opt/so/state/setup-complete
-# marker so so-boot-highstate.service can be enabled and the so-user_sync cron
-# in sync_es_users.sls keeps installing. Setup-in-progress systems instead get
-# the marker from `mark_setup_complete` in setup/so-functions at the right
-# moment. `replace: false` means we never overwrite a marker once written.
-mark_setup_complete_for_upgrades:
-  file.managed:
-    - name: /opt/so/state/setup-complete
-    - replace: false
-    - makedirs: True
-    - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion"
-    - require_in:
-      - file: remove_startup_states
-      - service: so_boot_highstate_service
+    - require:
+      - file: mark_setup_complete
 
 {% endif %}
 

salt/setup/virt/setSalt.sls

@@ -8,6 +8,15 @@ set_role_grain:
     - name: role
     - value: so-{{ grains.id.split("_") | last }}
 
+# salt-cloud guests never run so-setup, so nothing else marks them setup-complete.
+# Replaces the 'startup_states: highstate' line this state used to append. No
+# GLOBALS import -- this runs before the guest's pillars exist.
+mark_setup_complete_vm_guest:
+  file.managed:
+    - name: /opt/so/state/setup-complete
+    - replace: false
+    - makedirs: True
+
 enable_salt_minion:
   service.enabled:
     - name: salt-minion

salt/soc/defaults.yaml

@@ -36,8 +36,8 @@ soc:
         icon: fa-stream
         target: ''
         links:
-          - '/joblookup?esid={:soc_id}&time={:@timestamp}&gridId={gridId}'
-          - '/joblookup?ncid={:network.community_id}&time={:@timestamp}&gridId={gridId}'
+          - '/api/joblookup?esid={:soc_id}&time={:@timestamp}&gridId={gridId}'
+          - '/api/joblookup?ncid={:network.community_id}&time={:@timestamp}&gridId={gridId}'
         categories:
           - hunt
           - alerts
@@ -1537,6 +1537,20 @@ soc:
             Orchestrator: sonnet@SOAI
             Investigator: gemma@SOAI
             DetectionEngineer: gemma@SOAI
+          useMemory: true
+          useMemoryScanner: false
+          memoryScanIntervalSeconds: 300
+          memoryProximityThreshold: 0.8
+          messageProximityThreshold: 0.5
+          maxUserMemoriesToInclude: 5
+          maxGlobalMemoriesToInclude: 5
+          maxUserMemoriesToReconcile: 20
+          maxGlobalMemoriesToReconcile: 20
+          memoryModel: gemma@SOAI
+          embedModel: amazon.titan-embed-text-v2@SOAI
+          reconcileModel: gemma@SOAI
+          memoryPersona: ""
+          reconcilePersona: ""
         onionconfig:
           saltstackDir: /opt/so/saltstack
           bypassEnabled: false
@@ -2671,7 +2685,7 @@ soc:
               # The id (UUIDv4) is pregenerated and can safely be used.
               # Click "Convert" to convert the Sigma rule to use Security Onion field mappings within an EQL query
               #
-              # Rule Creation Guide: https://github.com/SigmaHQ/sigma/wiki/Rule-Creation-Guide
+              # Rule Creation Guide: https://github.com/SigmaHQ/sigma/wiki/Rule-Creation-High%E2%80%90Level-Guide
               # Logsources: https://sigmahq.io/docs/basics/log-sources.html
 
               title: 'A Short Capitalized Title With Less Than 50 Characters'
@@ -2683,7 +2697,7 @@ soc:
               references:
                 - 'https://local.invalid'
               author: '@SecurityOnion'
-              date: 'YYYY/MM/DD'
+              date: '[today]'
               tags:
                 - detection.threat_hunting
                 - attack.technique_id
@@ -2727,5 +2741,14 @@ soc:
               enabled: true
               adapter: SOAI
               charsPerTokenEstimate: 4
+            - id: amazon.titan-embed-text-v2
+              displayName: amazon.titan-embed-text-v2
+              origin: USA
+              contextLimitSmall: 8192
+              contextLimitLarge: 8192
+              lowBalanceColorAlert: 500000
+              enabled: true
+              adapter: SOAI
+              charsPerTokenEstimate: 4
               
             

salt/soc/soc_soc.yaml

@@ -732,11 +732,13 @@ soc:
             global: True
             advanced: True
             forcedType: int
+            readonlyUi: True
           maxDelegationDepth:
             description: Maximum delegation nesting depth for sub-agents. For example, a value of 2 lets the main agent delegate to a sub-agent that may itself delegate one level deeper. Any deeper delegation is refused and the requesting agent continues without it. Set to 0 to disable the limit.
             global: True
             advanced: True
             forcedType: int
+            readonlyUi: True
           adapters:
             description: Configuration for AI adapters used by the Onion AI assistant. Please see documentation for help on which fields are required for which protocols.
             global: True
@@ -779,6 +781,60 @@ soc:
             description: Indicates if the Assistant Module should operate in agentic mode or not. If true, agents can work together to solve tasks.
             global: True
             forcedType: bool
+          agents:
+            description: Agent definitions for the Onion AI assistant, managed from the Agent Studio. An entry naming a system agent overrides only the fields an admin may change; everything else comes from the built-in definition.
+            global: True
+            advanced: False
+            readonlyUi: True
+            storage: db
+            forcedType: "[]{}"
+            helpLink: onion-ai
+            syntax: json
+            uiElements:
+            - field: name
+              label: Name
+              required: True
+            - field: enabled
+              label: Enabled
+              forcedType: bool
+            - field: isOrchestrator
+              label: Orchestrator
+              forcedType: bool
+            - field: model
+              label: Model
+            - field: allowedSkills
+              label: Skills
+              forcedType: "[]string"
+            - field: canDelegateTo
+              label: Delegates To
+              forcedType: "[]string"
+            - field: description
+              label: Description
+            - field: persona
+              label: Persona
+              multiline: True
+          skills:
+            description: Skill definitions for the Onion AI assistant, managed from the Agent Studio. An entry naming a system skill overrides only its enabled state and persona addendum; its tool set comes from the built-in definition.
+            global: True
+            advanced: False
+            readonlyUi: True
+            storage: db
+            forcedType: "[]{}"
+            helpLink: onion-ai
+            syntax: json
+            uiElements:
+            - field: name
+              label: Name
+              required: True
+            - field: enabled
+              label: Enabled
+              forcedType: bool
+            - field: tools
+              label: Tools
+              forcedType: "[]string"
+            - field: persona
+              label: Persona
+              multiline: True
           agentMapping:
             Orchestrator:
               description: The initial agent in most agentic conversations. This agent will delegate requests to specialized agents.
@@ -789,6 +845,55 @@ soc:
             DetectionEngineer:
               description: This agent manages detections and their overrides, including tuning noisy rules and authoring rule content.
               global: True
+          useMemory:
+            description: Enables the Memory system for OnionAI
+            global: True
+            forcedType: bool
+          useMemoryScanner:
+            description: Enables the memory scanner for automatic memory extraction from historical sessions.
+            global: True
+            forcedType: bool
+          memoryScanIntervalSeconds:
+            description: How long to wait in seconds between attempts to scan sessions for new memories.
+            global: True
+          memoryProximityThreshold:
+            description: Describes how close memories need to be on a floating point scale from 0.0 to 1.0 to be considered when reconciling new memories with old ones. This value is usually higher than messageProximityThreshold.
+            global: True
+          messageProximityThreshold:
+            description: Describes how close a memory needs to be to a user's message on a floating point scale from 0.0 to 1.0 to be included in the context. This value is usually lower than memoryProximityThreshold.
+            global: True
+          maxUserMemoriesToInclude:
+            description: Specify the max number of user-specific memories to include in the prompt when a user sends a message.
+            global: True
+          maxGlobalMemoriesToInclude:
+            description: Specify the max number of global memories to include in the prompt when a user sends a message.
+            global: True
+          maxUserMemoriesToReconcile:
+            description: When reconciling new user-specific memories with existing user-specific memories, this determines how many old memories may be considered.
+            global: True
+          maxGlobalMemoriesToReconcile:
+            description: When reconciling new global memories with existing global memories, this determines how many old memories may be considered.
+            global: True
+          memoryModel:
+            description: The model to use when extracting memories from sessions.
+            global: True
+          embedModel: 
+            description: The model to use when embedding a memory as a vector. Note that only memories embedded using the same model may be compared and only memories created with the model specified here will be considered when informing an agent of existing memories.
+            global: True
+            advanced: True
+          reconcileModel:
+            description: The model to use when reconciling memories that contain nearly the same content.
+            global: True
+          memoryPersona:
+            description: Text appended to the built-in prompt of the memory extraction agent, managed from the Agent Studio. Use it to steer what is worth remembering.
+            global: True
+            readonlyUi: True
+            multiline: True
+          reconcilePersona:
+            description: Text appended to the built-in prompt of the memory reconciliation agent, managed from the Agent Studio. Use it to steer how new memories are merged with existing ones.
+            global: True
+            readonlyUi: True
+            multiline: True
       client:
         assistant:
           enabled:

salt/telegraf/etc/telegraf.conf

@@ -335,7 +335,7 @@
 {%-   do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('sostatus.sh') %}
 [[inputs.exec]]
    commands = [
-      "/scripts/sostatus.sh"
+      ["/scripts/sostatus.sh"]
    ]
    data_format = "influx"
    timeout = "15s"
@@ -346,7 +346,7 @@
 [[inputs.exec]]
    commands = [
 {%-   for script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %}
-      "/scripts/{{script}}"{% if not loop.last %},{% endif %}
+      ["/scripts/{{script}}"]{% if not loop.last %},{% endif %}
 {%-   endfor %}
    ]
   data_format = "influx"
@@ -375,7 +375,7 @@
 {%- if GLOBALS.is_manager or GLOBALS.role == 'so-heavynode' %}
 [[ inputs.exec ]]
   commands = [
-    "/scripts/esindexsize.sh"
+    ["/scripts/esindexsize.sh"]
   ]
   data_format = "influx"
   interval = "1h"

salt/zeek/defaults.yaml

@@ -15,7 +15,7 @@ zeek:
       MailHostUpDown: 0
       LogRotationInterval: 3600
       LogExpireInterval: 0
-      StatsLogEnable: 1
+      StatsLogEnable: 0
       StatsLogExpireInterval: 0
       StatusCmdShowAll: 0
       CrashExpireInterval: 0

salt/zeek/disabled.sls

@@ -23,6 +23,11 @@ zeekpacketlosscron:
     - identifier: zeekpacketlosscron
     - user: root
 
+zeekctlcron:
+  cron.absent:
+    - identifier: zeekctlcron
+    - user: root
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/zeek/enabled.sls

@@ -87,6 +87,21 @@ zeekpacketlosscron:
     - month: '*'
     - dayweek: '*'
 
+# LogExpireInterval, StatsLogExpireInterval and CrashExpireInterval are only acted on by
+# 'zeekctl cron', so run it on the interval upstream recommends. This also restarts any
+# node that died unexpectedly. Runs as root because the script needs the docker socket;
+# it drops to the zeek user inside the container.
+zeekctlcron:
+  cron.present:
+    - name: /usr/sbin/so-zeek-cron > /dev/null 2>&1
+    - identifier: zeekctlcron
+    - user: root
+    - minute: '*/5'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/zeek/soc_zeek.yaml

@@ -58,6 +58,86 @@ zeek:
       CompressLogs:
         description: This setting enables compression of Zeek logs. If you are seeing packet loss at the top of the hour in Zeek or PCAP you might need to disable this by seting it to 0. This will use more disk space but save IO and CPU.
         helpLink: zeek
+      LogExpireInterval:
+        description: >-
+          How long to keep rotated Zeek logs in /nsm/zeek/logs. A bare number means DAYS, so 7 means 7 days.
+          You may also give an explicit unit, such as "7 days" or "12 hr". Use 0 to keep logs forever.
+          This value must not be shorter than LogRotationInterval (3600 seconds by default), so the smallest
+          usable value is 1 hr - Zeek will fail to start if it is shorter. Expiry is applied by "zeekctl cron",
+          which runs every 5 minutes, and removes log files older than this based on their modification time.
+        regex: ^(0|[1-9][0-9]*( ?(day|hr)s?)?)$
+        regexFailureMessage: Enter 0, or a positive number optionally followed by "day" or "hr" (for example 7, "7 days", or "12 hr"). Minutes are not accepted because a log expire interval shorter than the log rotation interval prevents Zeek from starting.
+        helpLink: zeek
+        advanced: True
+      StatsLogEnable:
+        description: >-
+          Set to 1 to have "zeekctl cron" write node statistics to /nsm/zeek/logs/stats. This is
+          disabled because the CPU and memory portion depends on the "top" command, which the Zeek
+          container does not include, so every run records an error for each node instead. The
+          interface packet counters it also collects are not used anywhere in Security Onion, which
+          tracks Zeek packet loss separately through packetloss.log and Telegraf. It is read only
+          for that reason.
+        regex: ^[01]$
+        regexFailureMessage: You must enter 0 or 1.
+        helpLink: zeek
+        advanced: True
+        readonly: True
+      StatsLogExpireInterval:
+        description: >-
+          Number of days to keep entries in the Zeek stats log, or 0 to keep them forever.
+          Applied by "zeekctl cron", which runs every 5 minutes. This has no effect unless
+          StatsLogEnable is turned on, which it is not by default.
+        regex: ^[0-9]+$
+        regexFailureMessage: You must enter a whole number of days, or 0 to keep entries forever.
+        helpLink: zeek
+        advanced: True
+      CrashExpireInterval:
+        description: >-
+          Number of days to keep Zeek crash directories, or 0 to keep them forever.
+          Applied by "zeekctl cron", which runs every 5 minutes.
+        regex: ^[0-9]+$
+        regexFailureMessage: You must enter a whole number of days, or 0 to keep crash directories forever.
+        helpLink: zeek
+        advanced: True
+      MinDiskSpace:
+        description: >-
+          Percentage of free disk space below which ZeekControl reports a warning, or 0 to disable the check
+          entirely. The Zeek container does not include a mail program, so the warning is not emailed - it
+          appears in the output of "zeekctl cron" instead. This setting never deletes anything - cleanup based
+          on disk usage is handled separately by so-sensor-clean.
+        regex: ^([0-9]|[1-9][0-9]|100)$
+        regexFailureMessage: You must enter a percentage between 0 and 100.
+        helpLink: zeek
+        advanced: True
+      MailTo:
+        description: >-
+          Address that ZeekControl would send mail to, covering cron output and crash reports, and the address
+          Zeek's notice framework would use. The Zeek container does not include a mail program, and Security
+          Onion never enables the notice email action, so no mail is sent and this address is unused. It is
+          read only for that reason.
+        helpLink: zeek
+        advanced: True
+        readonly: True
+      MailConnectionSummary:
+        description: >-
+          Set to 1 to email the hourly connection summary. This only controls the emailed copy - the summary is
+          generated and archived with the other Zeek logs either way. The Zeek container does not include a mail
+          program, so no mail is sent and this setting has no effect. It is read only for that reason.
+        regex: ^[01]$
+        regexFailureMessage: You must enter 0 or 1.
+        helpLink: zeek
+        advanced: True
+        readonly: True
+      MailHostUpDown:
+        description: >-
+          Set to 1 to report when a Zeek node changes between the up and down states. The Zeek container does
+          not include a mail program, so this notification cannot be emailed. It is read only for that reason.
+          Host status detection still runs regardless of this setting - only the notification is affected.
+        regex: ^[01]$
+        regexFailureMessage: You must enter 0 or 1.
+        helpLink: zeek
+        advanced: True
+        readonly: True
   policy:
     custom:
       filters: 

salt/zeek/tools/sbin/so-zeek-cron

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+# Run zeekctl's periodic maintenance tasks. This is what actually enforces
+# LogExpireInterval, StatsLogExpireInterval and CrashExpireInterval - without a periodic
+# 'zeekctl cron' those settings are inert no matter what they are set to.
+
+# This also restarts any node that died unexpectedly, and marks it crashed so a crash report
+# is produced. That is upstream's default cron behavior and it recovers a single node in
+# place. The beacon in salt/_beacons/zeek.py is the only other recovery path, it is disabled
+# by default (healthcheck:enabled), and it removes and recreates the whole container, so
+# letting zeekctl handle a single dead worker avoids the heavier restart.
+
+if ! docker ps --filter name=so-zeek --format '{{.Names}}' | grep -q '^so-zeek$'; then
+  exit 0
+fi
+
+# Run as the zeek user so the stats logs and zeekctl-config.sh this writes stay owned by
+# uid 937 rather than root.
+docker exec so-zeek runuser -l zeek -c '/opt/zeek/bin/zeekctl cron'

setup/so-functions

@@ -971,8 +971,8 @@ docker_seed_registry() {
 	if [ -f /nsm/docker-registry/docker/registry.tar ]; then
 		logCmd "tar xvf /nsm/docker-registry/docker/registry.tar -C /nsm/docker-registry/docker"
 		logCmd "rm /nsm/docker-registry/docker/registry.tar"
-	elif [ -d /nsm/docker-registry/docker/registry ] && [ -f /etc/SOCLOUD ]; then
-		echo "Using existing docker registry content for cloud install"
+	elif [[ -d /nsm/docker-registry/docker/registry && ( -f /etc/SOCLOUD || "$is_airgap" == true ) ]]; then
+		echo "Using existing docker registry content"
 	else
 		if [ "$install_type" == 'IMPORT' ]; then
 			container_list 'so-import'	

setup/so-setup

@@ -833,6 +833,7 @@ if ! [[ -f $install_opt_file ]]; then
 		check_sos_appliance
 		drop_install_options
 		hypervisor_local_states
+		mark_setup_complete
 		verify_setup
 	fi