Merge remote-tracking branch 'origin/3/dev' into soupmod

.github/workflows/pythontest.yml

@@ -5,7 +5,6 @@ on:
     paths:
       - "salt/sensoroni/files/analyzers/**"
       - "salt/manager/tools/sbin/**"
-      - "salt/_beacons/**"
 
 jobs:
   build:
@@ -15,7 +14,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.14"]
-        python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin", "salt/_beacons"]
+        python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin"]
 
     steps:
     - uses: actions/checkout@v3

pillar/data/addtotab.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+# This script adds sensors/nodes/etc to the nodes tab
+default_salt_dir=/opt/so/saltstack/default
+local_salt_dir=/opt/so/saltstack/local
+TYPE=$1
+NAME=$2
+IPADDRESS=$3
+CPUS=$4
+GUID=$5
+MANINT=$6
+ROOTFS=$7
+NSM=$8
+MONINT=$9
+#NODETYPE=$10
+#HOTNAME=$11
+
+echo "Seeing if this host is already in here. If so delete it"
+if grep -q $NAME "$local_salt_dir/pillar/data/$TYPE.sls"; then
+  echo "Node Already Present - Let's re-add it"
+  awk -v blah="  $NAME:" 'BEGIN{ print_flag=1 }
+{
+    if( $0 ~ blah )
+    {
+       print_flag=0;
+       next
+    }
+    if( $0 ~ /^  [a-zA-Z0-9]+:$/ )
+    {
+        print_flag=1;
+    }
+    if ( print_flag == 1 )
+        print $0
+
+} ' $local_salt_dir/pillar/data/$TYPE.sls > $local_salt_dir/pillar/data/tmp.$TYPE.sls
+mv $local_salt_dir/pillar/data/tmp.$TYPE.sls $local_salt_dir/pillar/data/$TYPE.sls
+echo "Deleted $NAME from the tab. Now adding it in again with updated info"
+fi
+echo "  $NAME:" >> $local_salt_dir/pillar/data/$TYPE.sls
+echo "    ip: $IPADDRESS" >> $local_salt_dir/pillar/data/$TYPE.sls
+echo "    manint: $MANINT" >> $local_salt_dir/pillar/data/$TYPE.sls
+echo "    totalcpus: $CPUS" >> $local_salt_dir/pillar/data/$TYPE.sls
+echo "    guid: $GUID" >> $local_salt_dir/pillar/data/$TYPE.sls
+echo "    rootfs: $ROOTFS" >> $local_salt_dir/pillar/data/$TYPE.sls
+echo "    nsmfs: $NSM" >> $local_salt_dir/pillar/data/$TYPE.sls
+if [ $TYPE == 'sensorstab' ]; then
+  echo "    monint: bond0" >> $local_salt_dir/pillar/data/$TYPE.sls
+fi
+if [ $TYPE == 'evaltab' ] || [ $TYPE == 'standalonetab' ]; then
+  echo "    monint: bond0" >> $local_salt_dir/pillar/data/$TYPE.sls
+  if [ ! $10 ]; then
+    salt-call state.apply utility queue=True
+  fi
+fi
+if [ $TYPE == 'nodestab' ]; then
+  salt-call state.apply elasticsearch queue=True
+#  echo "    nodetype: $NODETYPE" >> $local_salt_dir/pillar/data/$TYPE.sls
+#  echo "    hotname: $HOTNAME" >> $local_salt_dir/pillar/data/$TYPE.sls
+fi

pillar/top.sls

@@ -3,8 +3,6 @@ base:
     - ca
     - global.soc_global
     - global.adv_global
-    - salt.soc_salt
-    - salt.adv_salt
     - docker.soc_docker
     - docker.adv_docker
     - influxdb.token

salt/_beacons/pillar_db.py

@@ -6,7 +6,7 @@
 # Custom salt beacon that watches the SOC audit_settings table in postgres for
 # new settings changes and emits a beacon event per new row. This replaces the
 # inotify watch on /opt/so/saltstack/local/pillar -- instead of monitoring pillar
-# files on disk, we monitor the securityonion.audit_settings table that SOC writes to.
+# files on disk, we monitor the so_soc.audit_settings table that SOC writes to.
 #
 # Detection is poll-based with a monotonic `id` watermark persisted to
 # WATERMARK_FILE: each pass selects rows with id greater than the last id seen,
@@ -22,9 +22,9 @@ import subprocess
 
 log = logging.getLogger(__name__)
 
-WATERMARK_FILE = '/opt/so/state/postgres_pillar_beacon_watch.id'
+WATERMARK_FILE = '/opt/so/state/pillar_db_watch.id'
 CONTAINER = 'so-postgres'
-DATABASE = 'securityonion'
+DATABASE = 'so_soc'
 
 # Unaligned, tuples-only psql output with a field separator that cannot appear in
 # an id/setting_id/node_id, so we can split each row reliably.
@@ -56,11 +56,11 @@ def _write_watermark(value):
             f.write(str(int(value)))
         os.rename(tmp, WATERMARK_FILE)
     except OSError:
-        log.exception('postgres_pillar_beacon: failed to persist watermark to %s', WATERMARK_FILE)
+        log.exception('pillar_db beacon: failed to persist watermark to %s', WATERMARK_FILE)
 
 
 def _query(sql):
-    # Run a query against securityonion inside the so-postgres container over the unix
+    # Run a query against so_soc inside the so-postgres container over the unix
     # socket (trust auth, no password). Returns stdout on success, or None on any
     # failure so the caller can no-op and retry on the next interval.
     cmd = [
@@ -71,19 +71,19 @@ def _query(sql):
     try:
         result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
     except subprocess.TimeoutExpired:
-        log.warning('postgres_pillar_beacon: psql timed out')
+        log.warning('pillar_db beacon: psql timed out')
         return None
     except Exception:
-        log.exception('postgres_pillar_beacon: failed to exec psql')
+        log.exception('pillar_db beacon: failed to exec psql')
         return None
     if result.returncode != 0:
-        log.warning('postgres_pillar_beacon: psql failed (rc=%s): %s',
+        log.warning('pillar_db beacon: psql failed (rc=%s): %s',
                     result.returncode, (result.stderr or '').strip())
         return None
     return result.stdout
 
 
-def beacon(config):  # noqa: C901
+def beacon(config):
     retval = []
 
     watermark = _read_watermark()
@@ -97,7 +97,7 @@ def beacon(config):  # noqa: C901
         try:
             _write_watermark(int((seed or '0').strip() or 0))
         except ValueError:
-            log.warning('postgres_pillar_beacon: could not parse MAX(id) seed: %r', seed)
+            log.warning('pillar_db beacon: could not parse MAX(id) seed: %r', seed)
         return retval
 
     rows = _query(
@@ -116,12 +116,12 @@ def beacon(config):  # noqa: C901
             continue
         parts = line.split(FIELD_SEP)
         if len(parts) < 3:
-            log.warning('postgres_pillar_beacon: skipping malformed row: %r', line)
+            log.warning('pillar_db beacon: skipping malformed row: %r', line)
             continue
         try:
             row_id = int(parts[0])
         except ValueError:
-            log.warning('postgres_pillar_beacon: skipping row with non-int id: %r', line)
+            log.warning('pillar_db beacon: skipping row with non-int id: %r', line)
             continue
         setting_id = parts[1]
         node_id = parts[2]
@@ -136,7 +136,7 @@ def beacon(config):  # noqa: C901
 
     if max_id > watermark:
         _write_watermark(max_id)
-        log.info('postgres_pillar_beacon: emitted %d change(s), watermark %d -> %d',
+        log.info('pillar_db beacon: emitted %d change(s), watermark %d -> %d',
                  len(retval), watermark, max_id)
 
     return retval

salt/_beacons/postgres_pillar_beacon_test.py

@@ -1,165 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-import os
-import shutil
-import subprocess
-import tempfile
-import unittest
-from unittest.mock import patch
-
-import postgres_pillar_beacon
-
-
-class TestPostgresPillarBeacon(unittest.TestCase):
-
-    def setUp(self):
-        # Point WATERMARK_FILE at a throwaway dir so the real read/write helpers
-        # (and their os.makedirs/os.rename) run against actual files, then clean
-        # it all up in tearDown.
-        self.tmpdir = tempfile.mkdtemp()
-        self.watermark = os.path.join(self.tmpdir, 'state', 'watch.id')
-        patcher = patch.object(postgres_pillar_beacon, 'WATERMARK_FILE', self.watermark)
-        patcher.start()
-        self.addCleanup(patcher.stop)
-
-    def tearDown(self):
-        shutil.rmtree(self.tmpdir, ignore_errors=True)
-
-    # -- trivial contract -------------------------------------------------
-
-    def test_virtual_returns_true(self):
-        self.assertTrue(postgres_pillar_beacon.__virtual__())
-
-    def test_validate_returns_valid(self):
-        self.assertEqual(postgres_pillar_beacon.validate({}), (True, 'valid'))
-
-    # -- _read_watermark --------------------------------------------------
-
-    def test_read_watermark_valid(self):
-        postgres_pillar_beacon._write_watermark(42)
-        self.assertEqual(postgres_pillar_beacon._read_watermark(), 42)
-
-    def test_read_watermark_missing_file_returns_none(self):
-        # tmp watermark file was never created
-        self.assertIsNone(postgres_pillar_beacon._read_watermark())
-
-    def test_read_watermark_garbage_returns_none(self):
-        os.makedirs(os.path.dirname(self.watermark), exist_ok=True)
-        with open(self.watermark, 'w') as f:
-            f.write('nope')
-        self.assertIsNone(postgres_pillar_beacon._read_watermark())
-
-    # -- _write_watermark -------------------------------------------------
-
-    def test_write_watermark_round_trip(self):
-        postgres_pillar_beacon._write_watermark(7)
-        with open(self.watermark) as f:
-            self.assertEqual(f.read(), '7')
-
-    def test_write_watermark_swallows_oserror(self):
-        with patch.object(postgres_pillar_beacon.os, 'makedirs', side_effect=OSError):
-            # Must not raise; failure is logged and the beacon retries next pass.
-            postgres_pillar_beacon._write_watermark(5)
-        self.assertFalse(os.path.exists(self.watermark))
-
-    # -- _query -----------------------------------------------------------
-
-    def test_query_success_returns_stdout_and_builds_argv(self):
-        completed = subprocess.CompletedProcess(args=[], returncode=0, stdout='rows', stderr='')
-        with patch.object(postgres_pillar_beacon.subprocess, 'run', return_value=completed) as mock_run:
-            result = postgres_pillar_beacon._query('SELECT 1;')
-        self.assertEqual(result, 'rows')
-        argv = mock_run.call_args[0][0]
-        self.assertEqual(argv[:5], ['docker', 'exec', 'so-postgres', 'psql', '-U'])
-        self.assertIn('SELECT 1;', argv)
-        self.assertFalse(mock_run.call_args[1].get('shell', False))
-
-    def test_query_timeout_returns_none(self):
-        with patch.object(postgres_pillar_beacon.subprocess, 'run',
-                          side_effect=subprocess.TimeoutExpired(cmd='psql', timeout=30)):
-            self.assertIsNone(postgres_pillar_beacon._query('SELECT 1;'))
-
-    def test_query_generic_exception_returns_none(self):
-        with patch.object(postgres_pillar_beacon.subprocess, 'run', side_effect=Exception('boom')):
-            self.assertIsNone(postgres_pillar_beacon._query('SELECT 1;'))
-
-    def test_query_nonzero_returncode_returns_none(self):
-        completed = subprocess.CompletedProcess(args=[], returncode=1, stdout='', stderr='bad')
-        with patch.object(postgres_pillar_beacon.subprocess, 'run', return_value=completed):
-            self.assertIsNone(postgres_pillar_beacon._query('SELECT 1;'))
-
-    # -- beacon: first run / seeding --------------------------------------
-
-    def test_beacon_seeds_when_postgres_not_ready(self):
-        with patch.object(postgres_pillar_beacon, '_read_watermark', return_value=None), \
-             patch.object(postgres_pillar_beacon, '_query', return_value=None), \
-             patch.object(postgres_pillar_beacon, '_write_watermark') as mock_write:
-            self.assertEqual(postgres_pillar_beacon.beacon({}), [])
-        mock_write.assert_not_called()
-
-    def test_beacon_seeds_to_max_id_and_emits_nothing(self):
-        with patch.object(postgres_pillar_beacon, '_read_watermark', return_value=None), \
-             patch.object(postgres_pillar_beacon, '_query', return_value='7\n'), \
-             patch.object(postgres_pillar_beacon, '_write_watermark') as mock_write:
-            self.assertEqual(postgres_pillar_beacon.beacon({}), [])
-        mock_write.assert_called_once_with(7)
-
-    def test_beacon_seed_unparseable_is_swallowed(self):
-        with patch.object(postgres_pillar_beacon, '_read_watermark', return_value=None), \
-             patch.object(postgres_pillar_beacon, '_query', return_value='abc'), \
-             patch.object(postgres_pillar_beacon, '_write_watermark') as mock_write:
-            self.assertEqual(postgres_pillar_beacon.beacon({}), [])
-        mock_write.assert_not_called()
-
-    # -- beacon: steady state ---------------------------------------------
-
-    def test_beacon_query_failure_returns_empty(self):
-        with patch.object(postgres_pillar_beacon, '_read_watermark', return_value=10), \
-             patch.object(postgres_pillar_beacon, '_query', return_value=None), \
-             patch.object(postgres_pillar_beacon, '_write_watermark') as mock_write:
-            self.assertEqual(postgres_pillar_beacon.beacon({}), [])
-        mock_write.assert_not_called()
-
-    def test_beacon_emits_events_and_advances_watermark(self):
-        sep = postgres_pillar_beacon.FIELD_SEP
-        rows = '11%s5%snode1\n12%s6%s\n' % (sep, sep, sep, sep)
-        with patch.object(postgres_pillar_beacon, '_read_watermark', return_value=10), \
-             patch.object(postgres_pillar_beacon, '_query', return_value=rows), \
-             patch.object(postgres_pillar_beacon, '_write_watermark') as mock_write:
-            result = postgres_pillar_beacon.beacon({})
-        self.assertEqual(result, [
-            {'tag': 'audit_settings', 'id': 11, 'setting_id': '5', 'node_id': 'node1'},
-            {'tag': 'audit_settings', 'id': 12, 'setting_id': '6', 'node_id': ''},
-        ])
-        mock_write.assert_called_once_with(12)
-
-    def test_beacon_skips_malformed_blank_and_noninteger_rows(self):
-        sep = postgres_pillar_beacon.FIELD_SEP
-        rows = (
-            '\n'                       # blank line -> skipped
-            '13%s7\n'                  # too few fields -> skipped
-            'abc%s8%snodeX\n'          # non-integer id -> skipped
-            '14%s9%snodeY\n'           # the one good row
-        ) % (sep, sep, sep, sep, sep)
-        with patch.object(postgres_pillar_beacon, '_read_watermark', return_value=10), \
-             patch.object(postgres_pillar_beacon, '_query', return_value=rows), \
-             patch.object(postgres_pillar_beacon, '_write_watermark') as mock_write:
-            result = postgres_pillar_beacon.beacon({})
-        self.assertEqual(result, [
-            {'tag': 'audit_settings', 'id': 14, 'setting_id': '9', 'node_id': 'nodeY'},
-        ])
-        mock_write.assert_called_once_with(14)
-
-    def test_beacon_no_new_rows_does_not_advance_watermark(self):
-        with patch.object(postgres_pillar_beacon, '_read_watermark', return_value=10), \
-             patch.object(postgres_pillar_beacon, '_query', return_value=''), \
-             patch.object(postgres_pillar_beacon, '_write_watermark') as mock_write:
-            self.assertEqual(postgres_pillar_beacon.beacon({}), [])
-        mock_write.assert_not_called()
-
-
-if __name__ == '__main__':
-    unittest.main()

salt/_beacons/rules_beacon.py

@@ -1,139 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Custom salt beacon that watches the suricata/strelka rule directories for changes
-# and emits a beacon event per changed directory. This replaces the stock salt
-# `inotify` beacon, which leaks a kernel inotify instance every time the minion
-# rebuilds the beacon loader's __context__ (orphaning the old pyinotify.Notifier
-# without closing it) until fs.inotify.max_user_instances is exhausted and the
-# beacon dies with EMFILE. Polling holds zero inotify instances, so the leak is
-# impossible, and it keeps firing during state runs (no blackout).
-#
-# Detection is poll-based with a per-directory fingerprint persisted to
-# WATERMARK_DIR: each pass walks the directory and hashes every file's
-# (relpath, st_mtime_ns, st_size), which catches content writes, additions,
-# moves, and deletions. A change in the digest emits one event; an unchanged
-# digest emits nothing. This makes it self-healing (a missed poll simply catches
-# up on the next one).
-#
-# Each emitted event carries the watched directory path under the configured tag
-# (e.g. salt/beacon/<minion>/rules_beacon/suricata); the push_suricata / push_strelka
-# reactors write a push intent, after which the existing so-push-drainer /
-# orch.push_batch pipeline takes over unchanged.
-
-import hashlib
-import logging
-import os
-import re
-
-log = logging.getLogger(__name__)
-
-WATERMARK_DIR = '/opt/so/state'
-
-# Temp/editor files that should not trigger a push. Mirrors the exclude regexes
-# the inotify beacon used. Matched against the full pathname.
-EXCLUDES = [
-    re.compile(r'\.sw[a-z]$'),
-    re.compile(r'~$'),
-    re.compile(r'/4913$'),
-    re.compile(r'/\.#'),
-]
-
-
-def __virtual__():
-    return True
-
-
-def validate(config):
-    return True, 'valid'
-
-
-def _paths_from_config(config):
-    # The beacon config arrives as a list of single-key dicts (salt beacon style).
-    # Merge it and return the {dir: tag} mapping under the 'paths' key.
-    merged = {}
-    if isinstance(config, list):
-        for item in config:
-            if isinstance(item, dict):
-                merged.update(item)
-    elif isinstance(config, dict):
-        merged = config
-    paths = merged.get('paths', {})
-    return paths if isinstance(paths, dict) else {}
-
-
-def _excluded(pathname):
-    for pattern in EXCLUDES:
-        if pattern.search(pathname):
-            return True
-    return False
-
-
-def _fingerprint(directory):
-    # Stat-only walk; hash each file's (relpath, mtime_ns, size). Returns a hex
-    # digest, or the digest of an empty tree if the directory does not exist.
-    h = hashlib.sha1()
-    if os.path.isdir(directory):
-        entries = []
-        for root, _dirs, files in os.walk(directory):
-            for name in files:
-                full = os.path.join(root, name)
-                if _excluded(full):
-                    continue
-                try:
-                    st = os.stat(full)
-                except OSError:
-                    continue
-                rel = os.path.relpath(full, directory)
-                entries.append('%s\0%d\0%d' % (rel, st.st_mtime_ns, st.st_size))
-        for line in sorted(entries):
-            h.update(line.encode('utf-8', 'surrogateescape'))
-            h.update(b'\n')
-    return h.hexdigest()
-
-
-def _watermark_file(tag):
-    return os.path.join(WATERMARK_DIR, 'rules_beacon_%s.hash' % tag)
-
-
-def _read_watermark(tag):
-    try:
-        with open(_watermark_file(tag), 'r') as f:
-            return (f.read() or '').strip() or None
-    except IOError:
-        return None
-
-
-def _write_watermark(tag, digest):
-    path = _watermark_file(tag)
-    try:
-        os.makedirs(WATERMARK_DIR, exist_ok=True)
-        tmp = path + '.tmp'
-        with open(tmp, 'w') as f:
-            f.write(digest)
-        os.rename(tmp, path)
-    except OSError:
-        log.exception('rules_beacon: failed to persist watermark to %s', path)
-
-
-def beacon(config):
-    retval = []
-
-    for directory, tag in _paths_from_config(config).items():
-        digest = _fingerprint(directory)
-        previous = _read_watermark(tag)
-
-        # First run / missing watermark: seed the digest and emit nothing so a
-        # fresh host does not fire a spurious fleetwide push.
-        if previous is None:
-            _write_watermark(tag, digest)
-            continue
-
-        if digest != previous:
-            _write_watermark(tag, digest)
-            retval.append({'tag': tag, 'path': directory})
-            log.info('rules_beacon: change detected in %s, emitting %s', directory, tag)
-
-    return retval

salt/_beacons/rules_beacon_test.py

@@ -1,172 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-import hashlib
-import os
-import shutil
-import tempfile
-import unittest
-from unittest.mock import patch
-
-import rules_beacon
-
-
-class TestRulesBeacon(unittest.TestCase):
-
-    def setUp(self):
-        # Isolate all on-disk state (watermarks and the dirs we fingerprint) in a
-        # throwaway tree, and point WATERMARK_DIR at it so the real read/write
-        # helpers run against actual files.
-        self.tmpdir = tempfile.mkdtemp()
-        self.state = os.path.join(self.tmpdir, 'state')
-        patcher = patch.object(rules_beacon, 'WATERMARK_DIR', self.state)
-        patcher.start()
-        self.addCleanup(patcher.stop)
-
-    def tearDown(self):
-        shutil.rmtree(self.tmpdir, ignore_errors=True)
-
-    def _make_dir(self, name, files=None):
-        path = os.path.join(self.tmpdir, name)
-        os.makedirs(path, exist_ok=True)
-        for fname, content in (files or {}).items():
-            with open(os.path.join(path, fname), 'w') as f:
-                f.write(content)
-        return path
-
-    # -- trivial contract -------------------------------------------------
-
-    def test_virtual_returns_true(self):
-        self.assertTrue(rules_beacon.__virtual__())
-
-    def test_validate_returns_valid(self):
-        self.assertEqual(rules_beacon.validate({}), (True, 'valid'))
-
-    # -- _paths_from_config -----------------------------------------------
-
-    def test_paths_from_config_list_of_dicts(self):
-        config = [{'interval': 10}, {'paths': {'/a': 'suricata', '/b': 'strelka'}}]
-        self.assertEqual(
-            rules_beacon._paths_from_config(config),
-            {'/a': 'suricata', '/b': 'strelka'},
-        )
-
-    def test_paths_from_config_plain_dict(self):
-        self.assertEqual(
-            rules_beacon._paths_from_config({'paths': {'/a': 'suricata'}}),
-            {'/a': 'suricata'},
-        )
-
-    def test_paths_from_config_skips_non_dict_items(self):
-        self.assertEqual(rules_beacon._paths_from_config(['bogus', 42]), {})
-
-    def test_paths_from_config_paths_not_a_dict(self):
-        self.assertEqual(rules_beacon._paths_from_config({'paths': 'nope'}), {})
-
-    def test_paths_from_config_unexpected_type(self):
-        self.assertEqual(rules_beacon._paths_from_config('nonsense'), {})
-
-    # -- _excluded --------------------------------------------------------
-
-    def test_excluded_matches_temp_and_editor_files(self):
-        for pathname in ('/rules/foo.swp', '/rules/foo~', '/rules/4913', '/rules/.#foo'):
-            self.assertTrue(rules_beacon._excluded(pathname), pathname)
-
-    def test_excluded_allows_real_rule_files(self):
-        self.assertFalse(rules_beacon._excluded('/rules/suricata.rules'))
-
-    # -- _fingerprint -----------------------------------------------------
-
-    def test_fingerprint_missing_dir_is_empty_tree_digest(self):
-        missing = os.path.join(self.tmpdir, 'does-not-exist')
-        self.assertEqual(rules_beacon._fingerprint(missing), hashlib.sha1().hexdigest())
-
-    def test_fingerprint_changes_when_content_changes(self):
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        before = rules_beacon._fingerprint(d)
-        with open(os.path.join(d, 'a.rules'), 'w') as f:
-            f.write('alert tcp any any -> any any')  # different size
-        self.assertNotEqual(rules_beacon._fingerprint(d), before)
-
-    def test_fingerprint_ignores_excluded_files(self):
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        before = rules_beacon._fingerprint(d)
-        with open(os.path.join(d, 'a.rules.swp'), 'w') as f:
-            f.write('editor swap')
-        self.assertEqual(rules_beacon._fingerprint(d), before)
-
-    def test_fingerprint_skips_unstatable_entries(self):
-        # A dangling symlink appears in os.walk's file list but os.stat raises
-        # OSError, exercising the except-continue path.
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        good = rules_beacon._fingerprint(d)
-        os.symlink(os.path.join(d, 'missing-target'), os.path.join(d, 'broken.link'))
-        self.assertEqual(rules_beacon._fingerprint(d), good)
-
-    # -- _read_watermark / _write_watermark -------------------------------
-
-    def test_watermark_round_trip(self):
-        rules_beacon._write_watermark('suricata', 'deadbeef')
-        self.assertEqual(rules_beacon._read_watermark('suricata'), 'deadbeef')
-
-    def test_read_watermark_missing_returns_none(self):
-        self.assertIsNone(rules_beacon._read_watermark('suricata'))
-
-    def test_read_watermark_empty_file_returns_none(self):
-        os.makedirs(self.state, exist_ok=True)
-        with open(rules_beacon._watermark_file('suricata'), 'w') as f:
-            f.write('')
-        self.assertIsNone(rules_beacon._read_watermark('suricata'))
-
-    def test_write_watermark_swallows_oserror(self):
-        with patch.object(rules_beacon.os, 'makedirs', side_effect=OSError):
-            rules_beacon._write_watermark('suricata', 'deadbeef')
-        self.assertIsNone(rules_beacon._read_watermark('suricata'))
-
-    # -- beacon -----------------------------------------------------------
-
-    def _config(self, mapping):
-        return [{'paths': mapping}]
-
-    def test_beacon_seeds_first_run_and_emits_nothing(self):
-        with patch.object(rules_beacon, '_fingerprint', return_value='hash1'), \
-             patch.object(rules_beacon, '_read_watermark', return_value=None), \
-             patch.object(rules_beacon, '_write_watermark') as mock_write:
-            result = rules_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
-        self.assertEqual(result, [])
-        mock_write.assert_called_once_with('suricata', 'hash1')
-
-    def test_beacon_emits_on_change(self):
-        with patch.object(rules_beacon, '_fingerprint', return_value='newhash'), \
-             patch.object(rules_beacon, '_read_watermark', return_value='oldhash'), \
-             patch.object(rules_beacon, '_write_watermark') as mock_write:
-            result = rules_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
-        self.assertEqual(result, [{'tag': 'suricata', 'path': '/rules/suricata'}])
-        mock_write.assert_called_once_with('suricata', 'newhash')
-
-    def test_beacon_no_change_emits_nothing(self):
-        with patch.object(rules_beacon, '_fingerprint', return_value='samehash'), \
-             patch.object(rules_beacon, '_read_watermark', return_value='samehash'), \
-             patch.object(rules_beacon, '_write_watermark') as mock_write:
-            result = rules_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
-        self.assertEqual(result, [])
-        mock_write.assert_not_called()
-
-    def test_beacon_end_to_end_with_real_files(self):
-        # Exercise the full stack (real fingerprint + real watermark files) across
-        # two poll passes: first seeds silently, second fires after a write.
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        config = self._config({d: 'suricata'})
-
-        self.assertEqual(rules_beacon.beacon(config), [])  # seed pass
-        self.assertEqual(rules_beacon.beacon(config), [])  # unchanged pass
-
-        with open(os.path.join(d, 'b.rules'), 'w') as f:
-            f.write('alert tcp any any -> any any')
-        self.assertEqual(rules_beacon.beacon(config), [{'tag': 'suricata', 'path': d}])
-
-
-if __name__ == '__main__':
-    unittest.main()

salt/_beacons/zeek.py

@@ -4,7 +4,7 @@ import logging
 def status():
 
  cmd = "runuser -l zeek -c '/opt/zeek/bin/zeekctl status'"
-    retval = __salt__['docker.run']('so-zeek', cmd)  # noqa: F821
+ retval = __salt__['docker.run']('so-zeek', cmd)
  logging.info('zeekctl_module: zeekctl.status retval: %s' % retval)
 
  return retval
@@ -14,7 +14,7 @@ def beacon(config):
 
   retval = []
 
-    is_enabled = __salt__['healthcheck.is_enabled']()  # noqa: F821
+  is_enabled = __salt__['healthcheck.is_enabled']()
   logging.info('zeek_beacon: healthcheck_is_enabled: %s' % is_enabled)
 
   if is_enabled:
@@ -25,8 +25,9 @@ def beacon(config):
     else:
      zeek_restart = False
 
-        __salt__['telegraf.send']('healthcheck zeek_restart=%s' % str(zeek_restart))  # noqa: F821
+    __salt__['telegraf.send']('healthcheck zeek_restart=%s' % str(zeek_restart))
     retval.append({'zeek_restart': zeek_restart})
     logging.info('zeek_beacon: retval: %s' % str(retval))
 
   return retval
+

salt/_beacons/zeek_test.py

@@ -1,59 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-import unittest
-from unittest.mock import MagicMock
-
-import zeek
-
-ZEEKCTL_CMD = "runuser -l zeek -c '/opt/zeek/bin/zeekctl status'"
-
-
-class TestZeekBeacon(unittest.TestCase):
-
-    def setUp(self):
-        # zeek.py relies on the __salt__ dunder that Salt injects at load time.
-        # Nothing defines it under test, so we attach a dict of mock loader
-        # functions to the module and remove it again afterwards.
-        self.salt = {
-            'docker.run': MagicMock(return_value='Zeek is running'),
-            'healthcheck.is_enabled': MagicMock(return_value=True),
-            'telegraf.send': MagicMock(),
-        }
-        zeek.__salt__ = self.salt
-        self.addCleanup(lambda: delattr(zeek, '__salt__'))
-
-    # -- status -----------------------------------------------------------
-
-    def test_status_runs_zeekctl_and_returns_output(self):
-        self.salt['docker.run'].return_value = 'Zeek is running'
-        result = zeek.status()
-        self.assertEqual(result, 'Zeek is running')
-        self.salt['docker.run'].assert_called_once_with('so-zeek', ZEEKCTL_CMD)
-
-    # -- beacon -----------------------------------------------------------
-
-    def test_beacon_disabled_returns_empty_and_skips_telegraf(self):
-        self.salt['healthcheck.is_enabled'].return_value = False
-        self.assertEqual(zeek.beacon({}), [])
-        self.salt['telegraf.send'].assert_not_called()
-
-    def test_beacon_running_reports_no_restart(self):
-        self.salt['docker.run'].return_value = 'Zeek is running'
-        self.assertEqual(zeek.beacon({}), [{'zeek_restart': False}])
-        self.salt['telegraf.send'].assert_called_once_with('healthcheck zeek_restart=False')
-
-    def test_beacon_unhealthy_status_triggers_restart(self):
-        # Each of these status tokens should flag a restart (the or-chain in beacon).
-        for status_text in ('Zeek is stopped', 'Zeek crashed', 'Zeek error state', 'Zeek error:'):
-            with self.subTest(status=status_text):
-                self.salt['docker.run'].return_value = status_text
-                self.salt['telegraf.send'].reset_mock()
-                self.assertEqual(zeek.beacon({}), [{'zeek_restart': True}])
-                self.salt['telegraf.send'].assert_called_once_with('healthcheck zeek_restart=True')
-
-
-if __name__ == '__main__':
-    unittest.main()

salt/allowed_states.map.jinja

@@ -37,7 +37,8 @@
     'elasticfleet',
     'elasticfleet.manager',
     'elasticsearch.cluster',
-    'elastic-fleet-package-registry'
+    'elastic-fleet-package-registry',
+    'utility'
 ] %}
 
 {% set sensor_states = [

salt/common/tools/sbin/so-common

@@ -291,20 +291,6 @@ download_and_verify() {
 	fi
 }
 
-# check if container with name is running and optionally stop it
-docker_check_running() {
-    # show running containers, only names
-    if docker ps --format '{{.Names}}' | grep -q "^so-${1}$"; then
-        if [[ "$2" == "--stop" ]]; then
-            docker stop "so-${1}"
-        fi
-
-        return 0
-    else
-        return 1
-    fi
-}
-
 elastic_license() {
 
 read -r -d '' message <<- EOM
@@ -602,6 +588,42 @@ run_check_net_err() {
 	fi
 }
 
+wait_for_salt_minion() {
+    local minion="$1"
+    local max_wait="${2:-30}"
+    local interval="${3:-2}"
+    local logfile="${4:-'/dev/stdout'}"
+    local elapsed=0
+
+	echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Waiting for salt-minion '$minion' to be ready..."
+
+    while [ $elapsed -lt $max_wait ]; do
+        # Check if service is running
+		echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Check if salt-minion service is running"
+        if ! systemctl is-active --quiet salt-minion; then
+            echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion service not running (elapsed: ${elapsed}s)"
+            sleep $interval
+            elapsed=$((elapsed + interval))
+            continue
+        fi
+		echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion service is running"
+
+        # Check if minion responds to ping
+		echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Check if $minion responds to ping"
+        if salt "$minion" test.ping --timeout=3 --out=json 2>> "$logfile" | grep -q "true"; then
+            echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion '$minion' is connected and ready!"
+            return 0
+        fi
+        
+        echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Waiting... (${elapsed}s / ${max_wait}s)"
+        sleep $interval
+        elapsed=$((elapsed + interval))
+    done
+
+    echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - ERROR: salt-minion '$minion' not ready after $max_wait seconds"
+    return 1
+}
+
 salt_minion_count() {
 	local MINIONDIR="/opt/so/saltstack/local/pillar/minions"
 	MINIONCOUNT=$(ls -la $MINIONDIR/*.sls | grep -v adv_ | wc -l)
@@ -666,7 +688,7 @@ systemctl_func() {
 
 	echo ""
 	echo "${echo_action^}ing $service_name service at $(date +"%T.%6N")"
-    systemctl $action $service_name && echo "Successfully ${echo_action}ed $service_name at $(date +"%T.%6N")." || echo "Failed to $action $service_name at $(date +"%T.%6N")."
+    systemctl $action $service_name && echo "Successfully ${echo_action}ed $service_name." || echo "Failed to $action $service_name."
 	echo ""
 }
 

salt/common/tools/sbin/so-kernel-upgrade

@@ -1,243 +0,0 @@
-#!/bin/bash
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-#
-# so-kernel-upgrade — install the UEK8 (6.x) kernel and make it the boot default.
-#
-# Security Onion is moving off the EL9 stock kernel (RHCK, 5.14) and UEK7 (5.15) onto UEK8
-# (6.x). Three things have to happen, and the tool has to drive each one:
-#
-#   1. Populate. The manager mirrors the UEK8 packages into /nsm/kernelrepo via so-repo-sync,
-#      and serves them to the grid over https://<manager>/kernelrepo. Until that sync runs the
-#      repo is valid but EMPTY -- dnf resolves it happily and installs nothing, with no error.
-#   2. Install. A node on RHCK has no kernel-uek* package at all, so there is nothing for
-#      'dnf update' to upgrade. A node on UEK7 does have kernel-uek installed, so
-#      'dnf install kernel-uek' reports "Nothing to do" and exits 0 without installing 6.x.
-#      Both cases need an explicit install of the UEK8 NEVRA.
-#   3. Boot it. Whether a newly installed UEK8 kernel becomes the boot default depends on the
-#      RUNNING kernel's flavor. kernel-install/grubby (with UPDATEDEFAULT=yes) only auto-promote
-#      within the running kernel's flavor lineage:
-#        - From UEK7 (5.x, kernel-uek) the install stays in the kernel-uek lineage and IS
-#          auto-promoted, so no grubby change is needed -- just make sure the repo is populated
-#          and install UEK8.
-#        - From the stock EL9 kernel (RHCK, 5.14, no UEK) it is a flavor CROSS that is NOT
-#          auto-promoted, so the box keeps booting RHCK until grubby is told otherwise.
-#      This tool inspects the running kernel and only runs 'grubby --set-default' for RHCK.
-#
-# Every one of those failure modes is silent by default. This tool handles each case and fails
-# loudly when it cannot, rather than reporting success while changing nothing.
-#
-# Manager vs minion: only the manager owns /nsm/kernelrepo, so only the manager can populate
-# it. If the repo is empty here, a manager runs so-repo-sync itself; a minion has no way to
-# fix it and exits non-zero telling the admin to sync the manager first.
-#
-# Idempotent: an already-installed, already-default UEK8 kernel is left alone. It only sets
-# the boot default; it does NOT reboot -- the admin reboots the node on their own schedule.
-
-. /usr/sbin/so-common
-
-# Client-side repo id (what dnf enables on this node, from repo/client/oracle.sls) vs the
-# reposync-side section in repodownload.conf that the manager mirrors from (mirrors the
-# securityonion/securityonionsync split for the main repo).
-KERNEL_REPO="securityonionkernel"
-KERNEL_REPO_SYNC="securityonionkernelsync"
-KERNEL_PKG="kernel-uek"
-KERNEL_REPO_DIR="/nsm/kernelrepo"
-REPOSYNC_CONF="/opt/so/conf/reposync/repodownload.conf"
-GLOBAL_PILLAR="/opt/so/saltstack/local/pillar/global/soc_global.sls"
-
-log() { echo "[so-kernel-upgrade] $*"; }
-die() { echo "[so-kernel-upgrade] ERROR: $*" >&2; exit 1; }
-
-command -v grubby >/dev/null 2>&1 || die "grubby not found"
-command -v dnf >/dev/null 2>&1 || die "dnf not found"
-
-ARCH="$(rpm -E '%{_arch}')"
-
-is_airgap() {
-    [ -f "$GLOBAL_PILLAR" ] && grep -q 'airgap: *[Tt]rue' "$GLOBAL_PILLAR"
-}
-
-# Newest installed UEK8 (6.x) kernel known to the bootloader. UEK8 vmlinuz paths look like
-# /boot/vmlinuz-6.12.0-204.92.4.2.el9uek.x86_64; UEK7 (5.15) and RHCK (5.14) won't match.
-find_uek8() {
-    grubby --info=ALL 2>/dev/null \
-        | sed -n 's/^kernel="\(.*\)"$/\1/p' \
-        | grep -E '/vmlinuz-6\.[0-9]+.*uek' \
-        | sort -V | tail -1
-}
-
-# Classify the RUNNING kernel (uname -r) -- this, not what's installed, is what decides whether
-# a UEK8 install auto-promotes to the boot default:
-#   uek8  6.x UEK    already on the target line; nothing to do
-#   uek7  5.x UEK    a UEK8 install stays in the kernel-uek lineage and auto-promotes (no grubby)
-#   rhck  5.14 EL9   crossing into the UEK flavor does NOT auto-promote (needs grubby --set-default)
-running_flavor() {
-    case "$(uname -r)" in
-        6.*uek*) echo uek8 ;;
-        *uek*)   echo uek7 ;;
-        *)       echo rhck ;;
-    esac
-}
-
-# Newest UEK8 kernel-uek NEVRA offered by the kernel repo, empty if the repo has none.
-# Restricted to the kernel repo so a UEK7 kernel-uek in the main repo can't be picked up,
-# and filtered to 6.x so we never "succeed" by reinstalling the 5.15 we already have.
-uek8_available() {
-    dnf -q repoquery --disablerepo='*' --enablerepo="$KERNEL_REPO" \
-        --arch="$ARCH" --latest-limit=1 \
-        --qf '%{name}-%{evr}.%{arch}\n' "$KERNEL_PKG" 2>/dev/null \
-        | grep -E "^${KERNEL_PKG}-6\." | tail -1
-}
-
-kernelrepo_rpm_count() {
-    find "$KERNEL_REPO_DIR" -maxdepth 1 -name '*.rpm' 2>/dev/null | wc -l
-}
-
-# The kernel repo starts life as valid-but-empty (kernelrepo_init_empty in
-# salt/manager/init.sls) and is filled by so-repo-sync. During a soup, so-repo-sync runs
-# BEFORE the highstate deploys the [securityonionkernelsync] section into repodownload.conf, so
-# the first kernel-aware soup leaves the repo empty until the next nightly sync.
-sync_kernel_repo() {
-    if is_airgap; then
-        log "airgap install: $KERNEL_REPO_DIR is populated from the airgap ISO, not by so-repo-sync."
-        return 1
-    fi
-    if ! grep -q "^\[${KERNEL_REPO_SYNC}\]" "$REPOSYNC_CONF" 2>/dev/null; then
-        log "$REPOSYNC_CONF has no [${KERNEL_REPO_SYNC}] section -- run a highstate to deploy it."
-        return 1
-    fi
-
-    log "populating $KERNEL_REPO_DIR with so-repo-sync (mirrors upstream; can take several minutes)"
-    su socore -c '/usr/sbin/so-repo-sync' || { log "so-repo-sync failed"; return 1; }
-
-    dnf -q clean expire-cache >/dev/null 2>&1
-    return 0
-}
-
-# Make the kernel repo actually able to serve a UEK8 package, or fail trying.
-ensure_kernel_repo() {
-    # The repo is assigned by the repo.client highstate, and only once NICs are pinned by MAC
-    # (/opt/so/state/nic_names_pinned) so the kernel swap can't renumber interfaces SO binds
-    # by name. skip_if_unavailable=1 means a broken repo is silently ignored, so check first.
-    if ! dnf -q repolist --enabled 2>/dev/null | awk '{print $1}' | grep -qx "$KERNEL_REPO"; then
-        log "repo '$KERNEL_REPO' is not enabled on this node."
-        log "Run a highstate first; the repo is skipped until /opt/so/state/nic_names_pinned"
-        log "exists (run so-nic-pin) and this node's salt matches the version this release ships."
-        die "kernel repo unavailable"
-    fi
-
-    [ -n "$(uek8_available)" ] && return 0
-
-    log "repo '$KERNEL_REPO' is enabled but offers no UEK8 $KERNEL_PKG package"
-
-    if ! is_manager_node; then
-        log "This is a minion; it consumes the kernel repo from the manager and cannot populate it."
-        log "On the manager, run:  su socore -c /usr/sbin/so-repo-sync"
-        log "then re-run this script here."
-        die "manager's kernel repo is empty"
-    fi
-
-    log "this is a manager and $KERNEL_REPO_DIR holds $(kernelrepo_rpm_count) rpm(s)"
-    sync_kernel_repo || die "could not populate $KERNEL_REPO_DIR"
-
-    [ -n "$(uek8_available)" ] \
-        || die "so-repo-sync completed but $KERNEL_REPO still offers no UEK8 $KERNEL_PKG"
-}
-
-reboot_notice() {
-    [ "$(uname -r)" = "$(basename "$1" | sed 's/^vmlinuz-//')" ] \
-        || log "REBOOT REQUIRED to start using the UEK8 kernel (currently running $(uname -r))."
-}
-
-# Keep future kernel updates on the UEK line rather than falling back to RHCK. Oracle ships
-# /etc/sysconfig/kernel; only rewrite it when it's actually pointing somewhere else.
-set_default_kernel_conf() {
-    if [ -f /etc/sysconfig/kernel ] && ! grep -q '^DEFAULTKERNEL=kernel-uek-core$' /etc/sysconfig/kernel; then
-        log "setting DEFAULTKERNEL=kernel-uek-core in /etc/sysconfig/kernel"
-        sed -i 's/^DEFAULTKERNEL=.*/DEFAULTKERNEL=kernel-uek-core/' /etc/sysconfig/kernel
-    fi
-}
-
-# Make sure a UEK8 kernel is installed, leaving its boot entry in INSTALLED_UEK8. If one is
-# already present we leave the repo alone -- it may be disabled or empty and we don't need it
-# just to flip the boot default. Otherwise install the explicit NEVRA, not the bare package
-# name: on a UEK7 node 'dnf install kernel-uek' sees 5.15 already present, prints "Nothing to
-# do" and exits 0 without installing 6.x.
-ensure_uek8_installed() {
-    INSTALLED_UEK8="$(find_uek8)"
-    if [ -n "$INSTALLED_UEK8" ]; then
-        log "UEK8 kernel already installed: $INSTALLED_UEK8"
-        return 0
-    fi
-
-    ensure_kernel_repo
-    local nevra; nevra="$(uek8_available)"
-    log "installing $nevra from $KERNEL_REPO"
-    dnf -y install "$nevra" || die "failed to install $nevra"
-
-    INSTALLED_UEK8="$(find_uek8)"
-    [ -n "$INSTALLED_UEK8" ] || die "$nevra installed but no 6.x UEK boot entry appeared -- check 'grubby --info=ALL'"
-    log "installed UEK8 kernel: $INSTALLED_UEK8"
-}
-
-case "$(running_flavor)" in
-uek8)
-    # Already on the 6.x UEK line. A plain 'dnf update' keeps this node current within the
-    # lineage and auto-promotes newer builds, so there is nothing for this tool to do.
-    log "already running a UEK8 kernel ($(uname -r)); nothing to do."
-    exit 0
-    ;;
-
-uek7)
-    # On a 5.x UEK kernel. Installing UEK8 stays inside the kernel-uek lineage, so dnf/grubby
-    # (UPDATEDEFAULT=yes) auto-promote it and we do NOT touch grubby. A node still on UEK7
-    # usually means the kernel repo was empty when it last updated, so populate it and install.
-    log "running UEK7 kernel ($(uname -r)); the kernel repo was likely not yet populated when"
-    log "this node last updated. Populating it and installing UEK8 -- the update stays on the"
-    log "kernel-uek line, so it becomes the boot default automatically (no grubby change needed)."
-    set_default_kernel_conf
-    ensure_uek8_installed
-
-    now="$(grubby --default-kernel 2>/dev/null)"
-    if [ "$now" = "$INSTALLED_UEK8" ]; then
-        log "boot default auto-promoted to UEK8 kernel: $INSTALLED_UEK8"
-    else
-        log "WARNING: expected the UEK8 kernel to auto-promote but the default is still"
-        log "'${now:-unknown}'. Run 'grubby --set-default=$INSTALLED_UEK8' to force it."
-    fi
-    reboot_notice "$INSTALLED_UEK8"
-    ;;
-
-rhck)
-    # On the stock EL9 kernel (5.14, no UEK installed). Crossing from RHCK into the UEK flavor
-    # does NOT auto-promote -- kernel-install/grubby only auto-promote within the running
-    # kernel's flavor lineage -- so after installing we must set the boot default explicitly.
-    log "running stock EL9 (RHCK) kernel ($(uname -r)); installing UEK8 and setting it as the"
-    log "boot default explicitly (a RHCK->UEK flavor change does not auto-promote)."
-    set_default_kernel_conf
-    ensure_uek8_installed
-    target="$INSTALLED_UEK8"
-
-    current="$(grubby --default-kernel 2>/dev/null)"
-    if [ "$current" = "$target" ]; then
-        log "UEK8 kernel is already the boot default: $target"
-        reboot_notice "$target"
-        exit 0
-    fi
-
-    log "current default kernel: ${current:-unknown}"
-    log "switching boot default to UEK8 kernel: $target"
-    grubby --set-default="$target" || die "grubby --set-default failed for $target"
-
-    # Verify the change actually took before claiming success.
-    now="$(grubby --default-kernel 2>/dev/null)"
-    [ "$now" = "$target" ] || die "default kernel is still '${now:-unknown}' after set-default"
-
-    log "boot default is now $target"
-    reboot_notice "$target"
-    ;;
-esac

salt/common/tools/sbin/so-log-check

@@ -132,7 +132,6 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|HTTP 404: Not Found"          # Salt loops until Kratos returns 200, during startup Kratos may not be ready
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Cancelling deferred write event maybeFenceReplicas because the event queue is now closed" # Kafka controller log during shutdown/restart
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Redis may have been restarted" # Redis likely restarted by salt
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|file already closed"          # Go logging race condition during container restart
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then

salt/common/tools/sbin/so-restart

@@ -5,44 +5,27 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
+
+
+# Usage: so-restart  kibana | playbook 
+
 . /usr/sbin/so-common
 
-usage() {
+if [ $# -ge 1 ]; then
-    echo "Usage: $0 <component> [args]"
-    echo ""
-    echo "Supported args:"
-    echo "  --force | -f          Force stop all Salt jobs before starting component."
-    echo ""
-    echo "Examples:"
-    echo "  $0 kibana             Restart Kibana"
-    echo "  $0 kibana --force     Force stop all Salt jobs before restarting Kibana"
-    exit 1
-}
 
-if [[ $# -lt 1 ]]; then
+        echo $banner
-    usage
+        printf "Restarting $1...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n"
-fi
+        echo $banner
 
-#shellcheck disable=SC2154
+        if [ "$2" = "--force" ]; then
-echo "$banner"
-printf "Restarting %s...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n" "$1"
-echo "$banner"
-if [[ "$2" = "--force" ]] || [[ "$2" = "-f" ]]; then
                 printf "\nForce-stopping all Salt jobs before proceeding\n\n"
                 salt-call saltutil.kill_all_jobs
-fi
+        fi
-case $1 in
-    "elastic-fleet"|"elasticfleet")
-        docker_check_running "elastic-fleet" "--stop"
-        docker rm "so-elastic-fleet" 2> /dev/null
-        # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy
-        rm -rf /opt/so/conf/elastic-fleet/state
 
-        salt-call state.apply elasticfleet queue=True
+        case $1 in
-        ;;
+                "elastic-fleet") docker stop so-elastic-fleet && docker rm so-elastic-fleet && salt-call state.apply elasticfleet queue=True;;
-    *)
+                *)  docker stop so-$1 ; docker rm so-$1 ; salt-call state.apply $1 queue=True;;
-        docker_check_running "$1" "--stop"
+        esac
-        docker rm "so-${1}" 2> /dev/null
+else
-        salt-call state.apply "$1" queue=True
+        echo -e "\nPlease provide an argument by running like so-restart $component, or by using the component-specific script.\nEx. so-restart logstash, or so-logstash-restart\n"
-        ;;
+fi
-esac

salt/common/tools/sbin/so-start

@@ -5,54 +5,27 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-# shellcheck disable=SC1091
+
+
+# Usage: so-start  all | kibana | playbook 
+
 . /usr/sbin/so-common
 
-usage() {
+if [ $# -ge 1 ]; then
-    echo "Usage: $0 <component> [args]"
+	echo $banner
-    echo ""
+	printf "Starting $1...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n"
-    echo "Supported args:"
+	echo $banner
-    echo "  --force | -f          Force stop all Salt jobs before starting component."
-    echo ""
-    echo "Examples:"
-    echo "  $0 kibana             Start Kibana"
-    echo "  $0 kibana --force     Force stop all Salt jobs before starting Kibana"
-    exit 1
-}
 
-if [[ $# -lt 1 ]]; then
+	if [ "$2" = "--force" ]; then
-    usage
-fi
-
-#shellcheck disable=SC2154
-echo "$banner"
-printf "Starting %s...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n" "$1"
-echo "$banner"
-if [[ "$2" = "--force" ]] || [[ "$2" == "-f" ]]; then
    		printf "\nForce-stopping all Salt jobs before proceeding\n\n"
    		salt-call saltutil.kill_all_jobs
-fi
+	fi
 
-case "$1" in
+	case $1 in
-    "all")
+   		"all") salt-call state.highstate queue=True;;
-        salt-call state.highstate queue=True
+   		"elastic-fleet") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply elasticfleet queue=True; fi ;; 
-        ;;
+   		*) if docker ps | grep -E -q '^so-$1$'; then printf "\n$1 is already running\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply $1 queue=True; fi ;; 
-   	"elastic-fleet"|"elasticfleet")
+	esac
-        if docker_check_running "elastic-fleet"; then
+else
-            printf "\nso-%s is already running!\n\n" "elastic-fleet"
+	echo -e "\nPlease provide an argument by running like so-start $component, or by using the component-specific script.\nEx. so-start logstash, or so-logstash-start\n"	
-            /usr/sbin/so-status
+fi
-        else
-            docker rm "so-elastic-fleet" 2> /dev/null
-            salt-call state.apply elasticfleet queue=True
-        fi
-        ;;
-   	*)
-        if docker_check_running "$1"; then
-            printf "\nso-%s is already running\n\n" "$1"
-            /usr/sbin/so-status
-        else
-            docker rm "so-${1}" 2> /dev/null
-            salt-call state.apply "$1" queue=True
-        fi
-        ;;
-esac

salt/common/tools/sbin/so-status

@@ -74,13 +74,13 @@ def output(options, console, code, data):
     summary = { "status_code": code, "containers": data }
     print(json.dumps(summary))
   elif "-q" not in options:
-    if code == 99:
+    if code == 2:
+      console.print(" [bold yellow]:hourglass: [bold white]System appears to be starting. No highstate has completed since the system was restarted.")
+    elif code == 99:
       console.print(" [bold red]:exclamation: [bold white]Installation does not appear to be complete. A highstate has not fully completed.")
     elif code == 100:
       console.print(" [bold red]:exclamation: [bold white]Installation encountered errors.")
     else:
-      if code == 2:
-        console.print(" [bold yellow]:hourglass: [bold white]System appears to be starting. No highstate has completed since the system was restarted. Container status is shown below.")
       table = Table(title = "Security Onion Status", show_edge = False, safe_box = True, box = box.MINIMAL)
       table.add_column("Container", justify="right", style="white", no_wrap=True)
       table.add_column("Status", justify="left", style="green", no_wrap=True)
@@ -154,14 +154,8 @@ def check_status(options, console):
   code = check_installation_status(options, console)
   if code == 0:
     code = check_system_status(options, console)
-  # Containers now start on boot without a highstate, so gather/display their
-  # status even when the system is still "starting" (code 2). Keep the starting
-  # code as the exit/status_code so SOC keeps showing the "restarting" message
-  # on the Grid until a highstate completes.
-  if code == 0 or code == 2:
-    container_code, container_list = check_container_status(options, console)
   if code == 0:
-      code = container_code
+    code, container_list = check_container_status(options, console)
   output(options, console, code, container_list)
   return code
 
@@ -186,3 +180,4 @@ def main():
 
 if __name__ == "__main__":
   main()
+

salt/common/tools/sbin/so-stop

@@ -5,35 +5,21 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-# shellcheck disable=SC1091
+
+
+# Usage: so-stop kibana | playbook | thehive
+
 . /usr/sbin/so-common
 
-usage() {
+if [ $# -ge 1 ]; then
-    echo "Usage: $0 <component>"
+	echo $banner
-    echo ""
+	printf "Stopping $1...\n"
-    echo "Examples:"
+	echo $banner
-    echo "  $0 kibana             Stop Kibana"
-    exit 1
-}
 
-if [[ $# -lt 1 ]]; then
+	case $1 in
-    usage
+    		*) docker stop so-$1 ; docker rm so-$1 ;;
+	esac
+else
+	echo -e "\nPlease provide an argument by running like so-stop $component, or by using the component-specific script.\nEx. so-stop logstash, or so-logstash-stop\n"	
 fi
 
-
-#shellcheck disable=SC2154
-echo "$banner"
-printf "Stopping %s...\n" "$1"
-echo "$banner"
-case $1 in
-	"elasticfleet"|"elastic-fleet")
-        docker_check_running "elastic-fleet" "--stop"
-        docker rm "so-elastic-fleet" 2> /dev/null
-        # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy
-        rm -rf /opt/so/conf/elastic-fleet/state
-        ;;
-    *)
-        docker_check_running "$1" "--stop"
-        docker rm "so-${1}" 2> /dev/null
-    ;;
-esac

salt/common/tools/sbin_jinja/so-import-pcap

@@ -63,8 +63,7 @@ function status {
 function pcapinfo() {
   PCAP=$1
   ARGS=$2
-  docker run --rm -v "$PCAP:/input.pcap" --entrypoint capinfos {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} /input.pcap -ae $ARGS |\
+  docker run --rm -v "$PCAP:/input.pcap" --entrypoint capinfos {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} /input.pcap -ae $ARGS
-    sed 's/First packet/Earliest packet/g' | sed 's/Last packet/Latest packet/g'
 }
 
 function pcapfix() {

salt/common/tools/sbin_jinja/so-salt-minion-check

@@ -5,7 +5,7 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-{% from 'salt/schedule.map.jinja' import SCHEDULEMERGED %}
+
 
 # this script checks the time the file /opt/so/log/salt/state-apply-test was last modified and restarts the salt-minion service if it is outside a threshold date/time
 # the file is modified via file.touch using a scheduled job healthcheck.salt-minion.state-apply-test that runs a state.apply. 
@@ -18,14 +18,13 @@
 
 QUIET=false
 UPTIME_REQ=1800 #in seconds, how long the box has to be up before considering restarting salt-minion due to /opt/so/log/salt/state-apply-test not being touched
-HIGHSTATE_UPTIME_REQ=900 #in seconds; if the box has been up this long and no highstate has completed since boot, force one
 CURRENT_TIME=$(date +%s)
 SYSTEM_START_TIME=$(date -d "$(</proc/uptime awk '{print $1}') seconds ago" +%s)
 LAST_HIGHSTATE_END=$([ -e "/opt/so/log/salt/lasthighstate" ] && date -r /opt/so/log/salt/lasthighstate +%s || echo 0)
 LAST_HEALTHCHECK_STATE_APPLY=$([ -e "/opt/so/log/salt/state-apply-test" ] && date -r /opt/so/log/salt/state-apply-test +%s || echo 0)
 # SETTING THRESHOLD TO ANYTHING UNDER 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default
-# THRESHOLD is derived from the salt schedule highstate interval + 1 hour, so the minion-check grace period tracks the schedule automatically.
+# THRESHOLD is derived from the global push highstate interval + 1 hour, so the minion-check grace period tracks the schedule automatically.
-THRESHOLD=$(( ({{ SCHEDULEMERGED.highstate_interval_hours }} + 1) * 3600 )) #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted
+THRESHOLD=$(( ({{ salt['pillar.get']('global:push:highstate_interval_hours', 2) }} + 1) * 3600 )) #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted
 THRESHOLD_DATE=$((LAST_HEALTHCHECK_STATE_APPLY+THRESHOLD))
 
 logCmd() {
@@ -77,50 +76,24 @@ done
 
 log "running so-salt-minion-check"
 
-RESTARTED=false
-
-# Check 1 (minion-restart-check): if the minion has stopped applying states (the
-# state-apply-test healthcheck file has gone stale), restart the salt-minion service.
 if [ $CURRENT_TIME -ge $((SYSTEM_START_TIME+$UPTIME_REQ))  ]; then
   if [ $THRESHOLD_DATE -le $CURRENT_TIME ]; then
-    log "[minion-restart-check] salt-minion is unable to apply states; restarting salt-minion" E
+    log "salt-minion is unable to apply states" E
-    log "[minion-restart-check] state-apply-test not touched by required date `date -d @$THRESHOLD_DATE`, last touched `date -d @$LAST_HEALTHCHECK_STATE_APPLY`" I
+    log "/opt/so/log/salt/healthcheck-state-apply not touched by required date: `date -d @$THRESHOLD_DATE`, last touched: `date -d @$LAST_HEALTHCHECK_STATE_APPLY`" I
-    log "[minion-restart-check] last highstate completed at `date -d @$LAST_HIGHSTATE_END`" I
+    log "last highstate completed at `date -d @$LAST_HIGHSTATE_END`" I
-    log "[minion-restart-check] checking if any jobs are running" I
+    log "checking if any jobs are running" I
     logCmd "salt-call --local saltutil.running" I
-    log "[minion-restart-check] ensure salt.minion-state-apply-test is enabled" I
+    log "ensure salt.minion-state-apply-test is enabled" I
     logCmd "salt-call state.enable salt.minion-state-apply-test" I
-    log "[minion-restart-check] ensure highstate is enabled" I
+    log "ensure highstate is enabled" I
     logCmd "salt-call state.enable highstate" I
-    log "[minion-restart-check] killing all salt-minion processes" I
+    log "killing all salt-minion processes" I
     logCmd "pkill -9 -ef /usr/bin/salt-minion" I
-    log "[minion-restart-check] starting salt-minion service" I
+    log "starting salt-minion service" I
     logCmd "systemctl start salt-minion" I
-    log "[minion-restart-check] waiting for salt-minion to become ready, then applying highstate in the background (queued)" I
-    nohup bash -c '/usr/sbin/so-salt-minion-wait; salt-call state.highstate queue=True' >> "/opt/so/log/salt/so-salt-minion-check" 2>&1 &
-    RESTARTED=true
   else
-    log "[minion-restart-check] healthy: state-apply-test last touched `date -d @$LAST_HEALTHCHECK_STATE_APPLY`, must go stale past `date -d @$THRESHOLD_DATE` to trigger a salt-minion restart" I
+    log "/opt/so/log/salt/healthcheck-state-apply last touched: `date -d @$LAST_HEALTHCHECK_STATE_APPLY` must be touched by `date -d @$THRESHOLD_DATE` to avoid salt-minion restart" I
   fi
 else
-  log "[minion-restart-check] skipped: system uptime $((CURRENT_TIME-SYSTEM_START_TIME))s is below the ${UPTIME_REQ}s minimum required before a salt-minion restart" I
+  log "system uptime only $((CURRENT_TIME-SYSTEM_START_TIME)) seconds does not meet $UPTIME_REQ second requirement." I
-fi
-
-# Check 2 (boot-highstate-check): if the host has been up long enough but no highstate
-# has completed since this boot, force one. This recovers a host whose boot highstate
-# (so-boot-highstate.service) failed or was skipped, even while the minion is otherwise
-# healthy (touching state-apply-test). We deliberately do NOT enable highstate here: if
-# soup has disabled it during an upgrade, Salt will refuse the highstate and we avoid
-# forcing one mid-upgrade.
-if $RESTARTED; then
-  log "[boot-highstate-check] skipped: minion-restart-check already queued a highstate this run" I
-elif [ $CURRENT_TIME -lt $((SYSTEM_START_TIME+HIGHSTATE_UPTIME_REQ)) ]; then
-  log "[boot-highstate-check] skipped: system uptime $((CURRENT_TIME-SYSTEM_START_TIME))s is below the ${HIGHSTATE_UPTIME_REQ}s minimum required before forcing a highstate" I
-elif [ $LAST_HIGHSTATE_END -ge $SYSTEM_START_TIME ]; then
-  log "[boot-highstate-check] healthy: a highstate completed at `date -d @$LAST_HIGHSTATE_END`, after this boot at `date -d @$SYSTEM_START_TIME`" I
-elif salt-call --local saltutil.running 2>/dev/null | grep -q 'state.highstate'; then
-  log "[boot-highstate-check] no highstate has completed since boot, but one is already running; skipping" I
-else
-  log "[boot-highstate-check] no highstate has completed since boot after $((CURRENT_TIME-SYSTEM_START_TIME))s uptime; applying highstate" E
-  nohup bash -c 'salt-call state.highstate -l info queue=True' >> "/opt/so/log/salt/so-salt-minion-check" 2>&1 &
 fi

salt/elasticfleet/config.sls

@@ -173,7 +173,7 @@ eaoptionalintegrationsdir:
 
 {% for minion in node_data %}
 {% set role = node_data[minion]["role"] %}
-{% if role in [ "eval","fleet","import","manager", "managerhype", "managersearch","standalone" ] %}
+{% if role in [ "eval","fleet","heavynode","import","manager", "managerhype", "managersearch","standalone" ] %}
 {% set optional_integrations = ELASTICFLEETMERGED.optional_integrations %}
 {% set integration_keys = optional_integrations.keys() %}
 fleet_server_integrations_{{ minion }}:

salt/elasticfleet/defaults.yaml

@@ -1,5 +1,6 @@
 elasticfleet:
   enabled: False
+  patch_version: 9.3.3+build202604082258 # Elastic Agent specific patch release.
   enable_manager_output: True
   config:
     server:

salt/elasticfleet/enabled.sls

@@ -11,10 +11,6 @@
 
 {#   This value is generated during node install and stored in minion pillar #}
 {%   set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %}
-{#   Prevent Elastic Agent from re-enrolling with a new agent.id everytime the container starts up.
-       - if a fresh enrollment is needed use 'so-stop elasticfleet'
-       #}
-{%   set ENROLLED = salt['file.file_exists']('/opt/so/conf/elastic-fleet/state/fleet.enc') %}
 
 include:
   - ca
@@ -70,7 +66,6 @@ so-elastic-fleet:
       - /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro
       - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro
       - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro
-      - /opt/so/conf/elastic-fleet/state:/usr/share/elastic-agent/state
       - /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs 
      {% if DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
         {% for BIND in DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
@@ -78,7 +73,6 @@ so-elastic-fleet:
         {% endfor %}
       {% endif %}      
     - environment:
-      {% if not ENROLLED %}
       - FLEET_SERVER_ENABLE=true
       - FLEET_URL=https://{{ GLOBALS.hostname }}:8220
       - FLEET_SERVER_ELASTICSEARCH_HOST=https://{{ GLOBALS.manager }}:9200
@@ -88,9 +82,6 @@ so-elastic-fleet:
       - FLEET_SERVER_CERT_KEY=/etc/pki/elasticfleet-server.key
       - FLEET_CA=/etc/pki/tls/certs/intca.crt     
       - FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt
-      {% endif %}
-      - STATE_PATH=/usr/share/elastic-agent/state
-      - CONFIG_PATH=/usr/share/elastic-agent/state
       - LOGS_PATH=logs
       {% if DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
         {% for XTRAENV in DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
@@ -109,7 +100,6 @@ so-elastic-fleet:
       - x509: etc_elasticfleet_crt
     - require:
       - file: trusttheca
-      - file: eastatedir
       - x509: etc_elasticfleet_key
       - x509: etc_elasticfleet_crt
 

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -5,7 +5,7 @@
   "package": {
     "name": "endpoint",
     "title": "Elastic Defend",
-    "version": "9.3.1",
+    "version": "9.3.0",
     "requires_root": true
   },
   "enabled": true,

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -29,7 +29,7 @@
               "\\.gz$"
             ],
             "include_files": [],
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.20.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.3\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.20.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.20.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.3\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.15.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.15.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.15.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
             ],

salt/elasticfleet/install_agent_grid.sls

@@ -10,15 +10,6 @@
 {% set AGENT_STATUS = salt['service.available']('elastic-agent') %}
 {% set AGENT_EXISTS = salt['file.file_exists']('/opt/Elastic/Agent/elastic-agent') %}
 
-so-elastic-agent-install:
-  file.managed:
-    - name: /usr/sbin/so-elastic-agent-install
-    - source: salt://elasticfleet/tools/sbin/so-elastic-agent-install
-    - user: 947
-    - group: 939
-    - mode: 755
-    - show_changes: False
-
 {% if not AGENT_STATUS or not AGENT_EXISTS %}
 
 pull_agent_installer:
@@ -30,9 +21,11 @@ pull_agent_installer:
 
 run_installer:
   cmd.run:
-    - name: /usr/sbin/so-elastic-agent-install "{{ GRIDNODETOKEN }}"
+    - name: ./so-elastic-agent_linux_amd64 -token={{ GRIDNODETOKEN }} -force
-    - require:
+    - cwd: /opt/so
-      - file: pull_agent_installer
+    - retry:
+        attempts: 3
+        interval: 20
 
 cleanup_agent_installer:
   file.absent:

salt/elasticfleet/manager.sls

@@ -67,6 +67,8 @@ so-elastic-fleet-package-upgrade:
         interval: 30
     - require:
       - http: wait_for_so-kibana
+    - onchanges:
+      - file: /opt/so/state/elastic_fleet_packages.txt
 
 so-elastic-fleet-integrations:
   cmd.run:

salt/elasticfleet/tools/sbin/so-elastic-agent-install

@@ -1,100 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-elastic-fleet-common
-
-
-# passed in as arg from elasticfleet/install_agent_grid.sls, else pulled from pillar later
-GRIDNODETOKEN="$1"
-LOGFILE="/opt/so/SO-Elastic-Agent_Installer_Health.log"
-
-check_agent_health() {
-    timeout=300
-    interval=10
-    start=$SECONDS
-
-    while (( SECONDS - start < timeout )); do
-        agent_status=$(elastic-agent status 2>&1)
-        echo -e "\n$(date)\n$agent_status\n" >> "$LOGFILE"
-        if echo "$agent_status" | grep -A1 'elastic-agent$' | grep -q 'status: (HEALTHY)'; then
-            return 0
-        fi
-        echo "The Elastic Agent is not yet healthy. Waiting for ${interval} seconds before checking again..."
-        sleep "$interval"
-    done
-
-    echo "The Elastic Agent did not become healthy within ${timeout} seconds"
-    return 1
-}
-
-uninstall_agent() {
-    if command -v elastic-agent >/dev/null 2>&1; then
-        elastic-agent uninstall -f
-    fi
-}
-
-if [[ -z "$GRIDNODETOKEN" ]]; then
-    noderole=$(so-yaml.py get -r /etc/salt/grains role)
-    if [[ "$noderole" == "so-heavynode" ]]; then
-        GRIDNODETOKEN=$(salt-call pillar.get global:fleet_grid_enrollment_token_heavy --out=newline_values_only)
-    else
-        GRIDNODETOKEN=$(salt-call pillar.get global:fleet_grid_enrollment_token_general --out=newline_values_only)
-    fi
-fi
-
-if [[ -z "$GRIDNODETOKEN" ]]; then
-    echo "Unable to determine Elastic Fleet enrollment token. Exiting."
-
-    exit 1
-fi
-
-if [[ ! -x /opt/so/so-elastic-agent_linux_amd64 ]]; then
-    echo "Downloading so-elastic-agent installer... This could take a while if another Salt job is running."
-
-    # When running outside of elasticfleet/install_agent_grid.sls we need to download the installer independently.
-    #  PYTHONWARNINGS="ignore" to avoid messages like the following when running salt-call:
-    #  '/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/base.py:129: TransportWarning: Unclosed transport! <salt.transport.zeromq.RequestClient object at 0x7fc5f0ee7a30>
-    #    File "/bin/salt-call", line 12, in <module>
-    #      sys.exit(salt_call())'
-
-    PYTHONWARNINGS="ignore" salt-call state.single file.managed name=/opt/so/so-elastic-agent_linux_amd64 source=salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64 mode=755 makedirs=True queue=True
-
-fi
-
-if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then
-    attempts=0
-    cd /opt/so/ || exit 1
-
-    truncate -s 0 "$LOGFILE"
-
-    uninstall_agent
-
-    while [[ $attempts -lt 3 ]]; do
-        if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force && echo "Verifying Elastic Agent health..." && check_agent_health; then
-            rm -f /opt/so/so-elastic-agent_linux_amd64
-            elastic-agent status
-
-            exit 0
-        fi
-
-        attempts=$((attempts + 1))
-
-        if [[ $attempts -lt 3 ]]; then
-            echo "Unable to verify Elastic Agent health... Retrying in 20 seconds..."
-            sleep 20
-        fi
-    done
-
-    uninstall_agent
-    rm -f /opt/so/so-elastic-agent_linux_amd64
-    echo "The so-elastic-agent installer failed after 3 attempts. Exiting."
-
-    exit 1
-else
-    echo "Unable to locate so-elastic-agent installer. Exiting."
-
-    exit 1
-fi

salt/elasticfleet/tools/sbin/so-elastic-fleet-common

@@ -30,94 +30,6 @@ fleet_api() {
     curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/${QUERYPATH}" "$@" --retry 3 --retry-delay 10 --fail 2>/dev/null
 }
 
-# Max number of concurrent Fleet write jobs (create/update). Override via env if needed.
-MAX_FLEET_JOBS=${MAX_FLEET_JOBS:-10}
-
-# Block until fewer than MAX_FLEET_JOBS background jobs are running.
-elastic_fleet_throttle() {
-    while (( $(jobs -rp | wc -l) >= MAX_FLEET_JOBS )); do
-        wait -n || true
-    done
-}
-
-# Load every integration JSON in a directory into a single agent policy.
-# The agent policy is fetched ONCE (not per file), and the create/update writes
-# are dispatched as throttled background jobs.
-#   $1 AGENT_POLICY     - the agent policy id/name to load integrations into
-#   $2 DIR              - directory of integration *.json files
-#   $3 LABEL           - human-readable label for log output
-#   $4 SKIP_CREATE_NAME - (optional) integration name to skip when creating (still updated if present)
-# Returns 1 if the policy cannot be fetched or if any integration failed to create/update.
-elastic_fleet_load_integrations_dir() {
-    local AGENT_POLICY=$1
-    local DIR=$2
-    local LABEL=$3
-    local SKIP_CREATE_NAME=$4
-    local POLICY_JSON FAIL_FILE OUT_DIR INTEGRATION NAME ID i
-
-    FAIL_FILE=$(mktemp)
-    # Each job buffers its full output (header + API response) into its own file so the
-    # parent can print them grouped and in submission order after concurrent writes finish.
-    OUT_DIR=$(mktemp -d)
-    i=0
-
-    # Fetch the agent policy a single time; we look up integration ids locally below.
-    if ! POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY"); then
-        echo "Error: Failed to retrieve agent policy '$AGENT_POLICY'."
-        rm -f "$FAIL_FILE"
-        rm -rf "$OUT_DIR"
-        return 1
-    fi
-
-    if ! jq -e '.item.package_policies' <<<"$POLICY_JSON" >/dev/null 2>&1; then
-        echo "Error: Invalid agent policy response for '$AGENT_POLICY'."
-        rm -f "$FAIL_FILE"
-        rm -rf "$OUT_DIR"
-        return 1
-    fi
-
-    for INTEGRATION in "$DIR"/*.json; do
-        [ -e "$INTEGRATION" ] || continue
-        NAME=$(jq -r .name "$INTEGRATION")
-        ID=$(jq -r --arg n "$NAME" '.item.package_policies[]? | select(.name==$n) | .id' <<<"$POLICY_JSON")
-
-        elastic_fleet_throttle
-        {
-            local RESP
-            if [ -n "$ID" ]; then
-                printf "\n\n%s - Updating integration %s\n" "$LABEL" "$NAME"
-                if ! RESP=$(elastic_fleet_integration_update "$ID" "@$INTEGRATION"); then
-                    flock 9; echo "update ${INTEGRATION##*/}" >&9
-                fi
-                printf '%s\n' "$RESP"
-            elif [ -n "$SKIP_CREATE_NAME" ] && [ "$NAME" == "$SKIP_CREATE_NAME" ]; then
-                printf "\n\n%s - Skipping creation of %s\n" "$LABEL" "$NAME"
-            else
-                printf "\n\n%s - Creating integration %s\n" "$LABEL" "$NAME"
-                if ! RESP=$(elastic_fleet_integration_create "@$INTEGRATION"); then
-                    flock 9; echo "create ${INTEGRATION##*/}" >&9
-                fi
-                printf '%s\n' "$RESP"
-            fi
-        } >"$OUT_DIR/$(printf '%03d' "$i")" 9>>"$FAIL_FILE" &
-        i=$((i+1))
-    done
-    wait || true
-
-    # Emit per-integration output grouped and in submission order (glob sorts numerically).
-    cat "$OUT_DIR"/* 2>/dev/null
-    rm -rf "$OUT_DIR"
-
-    local rc=0
-    if [ -s "$FAIL_FILE" ]; then
-        printf "\n%s: failed integrations:\n" "$LABEL"
-        cat "$FAIL_FILE"
-        rc=1
-    fi
-    rm -f "$FAIL_FILE"
-    return $rc
-}
-
 elastic_fleet_integration_check() {
 
     AGENT_POLICY=$1
@@ -134,9 +46,7 @@ elastic_fleet_integration_create() {
 
     JSON_STRING=$1
 
-    # --retry-all-errors so transient 409 conflicts (concurrent writes to the same agent
+    if ! fleet_api "package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPOST -d "$JSON_STRING"; then
-    # policy) are retried; curl --retry alone does not retry 409.
-    if ! fleet_api "package_policies" --retry-all-errors -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPOST -d "$JSON_STRING"; then
         return 1
     fi
 }
@@ -167,9 +77,7 @@ elastic_fleet_integration_update() {
 
     JSON_STRING=$2
 
-    # --retry-all-errors so transient 409 conflicts (concurrent writes to the same agent
+    if ! fleet_api "package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPUT -d "$JSON_STRING"; then
-    # policy) are retried; curl --retry alone does not retry 409.
-    if ! fleet_api "package_policies/$UPDATE_ID" --retry-all-errors -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPUT -d "$JSON_STRING"; then
         return 1
     fi
 }

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -9,36 +9,102 @@
 RETURN_CODE=0
 
 if [ ! -f /opt/so/state/eaintegrations.txt ]; then
+  # First, check for any package upgrades
+  /usr/sbin/so-elastic-fleet-package-upgrade
 
-  # update Fleet Server policies
+  # Second, update Fleet Server policies
   /usr/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server
 
-  # configure Elastic Defend Integration separately
+  # Third, configure Elastic Defend Integration seperately
   /usr/sbin/so-elastic-fleet-integration-policy-elastic-defend
 
-  # Each group fetches its agent policy once and dispatches create/update writes concurrently.
-
   # Initial Endpoints
-  elastic_fleet_load_integrations_dir "endpoints-initial" \
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/endpoints-initial/*.json; do
-    /opt/so/conf/elastic-fleet/integrations/endpoints-initial "Initial Endpoints Policy" || RETURN_CODE=1
+    printf "\n\nInitial Endpoints Policy - Loading $INTEGRATION\n"
+    elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
+    if [ -n "$INTEGRATION_ID" ]; then
+      printf "\n\nIntegration $NAME exists - Updating integration\n"
+      if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+        echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
+    else
+      printf "\n\nIntegration does not exist - Creating integration\n"
+      if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
+    fi
+  done
 
   # Grid Nodes - General
-  elastic_fleet_load_integrations_dir "so-grid-nodes_general" \
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_general/*.json; do
-    /opt/so/conf/elastic-fleet/integrations/grid-nodes_general "Grid Nodes Policy_General" || RETURN_CODE=1
+    printf "\n\nGrid Nodes Policy_General - Loading $INTEGRATION\n"
+    elastic_fleet_integration_check "so-grid-nodes_general" "$INTEGRATION"
+    if [ -n "$INTEGRATION_ID" ]; then
+      printf "\n\nIntegration $NAME exists - Updating integration\n"
+      if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+        echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
+    else
+      printf "\n\nIntegration does not exist - Creating integration\n"
+      if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
+    fi
+  done
 
   # Grid Nodes - Heavy
-  elastic_fleet_load_integrations_dir "so-grid-nodes_heavy" \
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy/*.json; do
-    /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy "Grid Nodes Policy_Heavy" || RETURN_CODE=1
+    printf "\n\nGrid Nodes Policy_Heavy - Loading $INTEGRATION\n"
+    elastic_fleet_integration_check "so-grid-nodes_heavy" "$INTEGRATION"
+    if [ -n "$INTEGRATION_ID" ]; then
+      printf "\n\nIntegration $NAME exists - Updating integration\n"
+      if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+        echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
+    else
+      printf "\n\nIntegration does not exist - Creating integration\n"
+      if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
+    fi
+  done
 
-  # Fleet Server - Optional integrations (adds integration configuration to a given FleetServer_ policy)
+  # Fleet Server - Optional integrations
-  for FLEET_DIR in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/; do
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json; do
-    [ -d "$FLEET_DIR" ] || continue
+    if ! [ "$INTEGRATION" == "/opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json" ]; then
-    INTEGRATIONS=("${FLEET_DIR%/}"/*.json)
+      FLEET_POLICY=`echo "$INTEGRATION"| cut -d'/' -f7`
-    [ -e "${INTEGRATIONS[0]}" ] || continue
+      printf "\n\nFleet Server Policy - Loading $INTEGRATION\n"
-
+      elastic_fleet_integration_check "$FLEET_POLICY" "$INTEGRATION"
-    FLEET_POLICY=$(basename "$FLEET_DIR")
+      if [ -n "$INTEGRATION_ID" ]; then
-    elastic_fleet_load_integrations_dir "$FLEET_POLICY" \
+        printf "\n\nIntegration $NAME exists - Updating integration\n"
-      "${FLEET_DIR%/}" "Fleet Server Policy" "elasticsearch-logs" || RETURN_CODE=1
+        if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+          echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+          RETURN_CODE=1
+          continue
+        fi
+      else
+        printf "\n\nIntegration does not exist - Creating integration\n"
+        if [ "$NAME" != "elasticsearch-logs" ]; then
+          if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+            echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+            RETURN_CODE=1
+            continue
+          fi
+        fi
+      fi
+    fi
   done
 
   # Only create the state file if all policies were created/updated successfully

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers

@@ -30,7 +30,7 @@ done
 if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then 
   printf "\nFleet Host URL, Enrollment Token or Elastic Version empty - exiting..."
   printf "\nFleet Host: $FLEETHOST, Enrollment Token: $ENROLLMENTOKEN\n"
-  exit 1
+  exit 
 fi
 
 OSARCH=( "linux-x86_64" "windows-x86_64" "darwin-x86_64" "darwin-aarch64" )
@@ -62,54 +62,31 @@ do
 done 
 
 GOTARGETOS=( "linux" "windows" "darwin" "darwin/arm64" )
+GOARCH="amd64"
 printf "\n### Generating OS packages using the cleaned up tarballs"
-for GOOS in "${GOTARGETOS[@]}"; do
+for GOOS in "${GOTARGETOS[@]}"
-    GOARCH="amd64"
+do
     if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi
     printf "\n\n### Generating $GOOS/$GOARCH Installer...\n"
     docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \
     --mount type=bind,source=/etc/pki/tls/certs/,target=/workspace/files/cert/ \
     --mount type=bind,source=/nsm/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \
-    --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ \
+    --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \
     {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN"  -o /output/so-elastic-agent_${GOOS}_${GOARCH}
     printf "\n### $GOOS/$GOARCH Installer Generated...\n"
 done
 
 printf "\n\n### Generating MSI...\n"
-cp /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe
+cp /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe
 docker run \
---mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ -w /output \
+--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ -w /output \
 {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs
 printf "\n### MSI Generated...\n"
 
-# Verify installers were created
-for GOOS in "${GOTARGETOS[@]}"; do
-    GOARCH="amd64"
-    if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin"; GOARCH="arm64"; fi
-    if [[ ! -f /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_${GOOS}_${GOARCH} ]]; then
-        printf "\n### ERROR: Installer for %s/%s was not generated. Exiting...\n" "$GOOS" "$GOARCH"
-        exit 1
-    fi
-    # After verifying new installer was generated, move it to so_agent-installers directory
-    mv /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_${GOOS}_${GOARCH} /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/
-done
-
-# Verify MSI installer
-if [[ ! -f /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64_msi ]]; then
-    printf "\n### ERROR: Installer MSI was not generated. Exiting...\n"
-    exit 1
-else
-    # After verifying new installer MSI was generated, move it to so_agent-installers directory
-    mv /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64_msi /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/
-fi
-
 printf "\n### Cleaning up temp files \n"
 rm -rf /nsm/elastic-agent-workspace
-rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe
+rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe
 
 printf "\n### Copying so_agent-installers to /nsm/elastic-fleet/ for nginx.\n"
 \cp -vr /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ /nsm/elastic-fleet/
 chmod 644 /nsm/elastic-fleet/so_agent-installers/*
-
-# if we got here all installers have been generated successfully
-exit 0

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade

@@ -23,90 +23,73 @@ if [ $? -ne 0 ]; then
 fi
 
 default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %})
-# JSON array of the default packages, used by the jq filter below.
-default_packages_json=$(printf '%s\n' "${default_packages[@]}" | jq -R . | jq -s '.')
-
-# Output lock (serializes concurrent job output) and failure file (one marker line per
-# failed integration). Mirrors the pattern used by elastic_fleet_load_integrations_dir.
-OUTPUT_LOCK=$(mktemp)
-FAIL_FILE=$(mktemp)
-trap 'rm -f "$OUTPUT_LOCK" "$FAIL_FILE"' EXIT
-
-# Cache of package name -> latest available version, so the same package is only looked up
-# once instead of once per (policy, integration).
-declare -A LATEST_VERSION_CACHE
 
+ERROR=false
 for AGENT_POLICY in $agent_policies; do
-    # Fetch the agent policy a single time; package name/version and integration id are all
+    if ! integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY"); then
-    # extracted locally below instead of re-fetching the same policy per integration.
-    if ! POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY"); then
         # this script upgrades default integration packages, exit 1 and let salt handle retrying
         exit 1
     fi
-
+    for INTEGRATION in $integrations; do
-    # One jq pass emits name/package.name/package.version/id for every eligible integration.
+        if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then
-    # The endpoint/fleet_server skips and the default-package gate are applied here in jq.
+            # Get package name so we know what package to look for when checking the current and latest available version
-    # $defaults (not $def, a jq reserved keyword) holds the default package list.
+            if ! PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION"); then
-    while IFS=$'\t' read -r INTEGRATION PACKAGE_NAME PACKAGE_VERSION INTEGRATION_ID; do
-        [ -n "$INTEGRATION" ] || continue
-
-        # Look up the latest available version once per package, then memoize it.
-        if [[ -z "${LATEST_VERSION_CACHE[$PACKAGE_NAME]+set}" ]]; then
-            if ! AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME"); then
-                echo "Error: Failed getting latest version for $PACKAGE_NAME"
                 exit 1
             fi
-            LATEST_VERSION_CACHE[$PACKAGE_NAME]=$AVAILABLE_VERSION
+            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+            if [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then
+            {%- endif %}
+                # Get currently installed version of package
+                attempt=0
+                max_attempts=3
+                while [ $attempt -lt $max_attempts ]; do
+                    if PACKAGE_VERSION=$(elastic_fleet_integration_policy_package_version "$AGENT_POLICY" "$INTEGRATION") && AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME"); then
+                        break
+                    fi
+                    attempt=$((attempt + 1))
+                done
+                if [ $attempt -eq $max_attempts ]; then
+                    echo "Error: Failed getting $PACKAGE_VERSION or $AVAILABLE_VERSION"
+                    exit 1
+                fi
+
+                # Get integration ID
+                if ! INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION"); then
+                    exit 1
                 fi
-        AVAILABLE_VERSION=${LATEST_VERSION_CACHE[$PACKAGE_NAME]}
 
                 if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then
-            # Dry run, then (if clean) the actual upgrade, dispatched as a throttled background
+                    # Dry run of the upgrade
-            # job. Each job builds its full log into one block, then flushes it under a single
+                    echo ""
-            # shared lock (OUTPUT_LOCK) so concurrent jobs never interleave on stdout; a failed
+                    echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..."
-            # job also appends a marker line to FAIL_FILE while holding that same lock.
+                    echo "Upgrading $INTEGRATION..."
-            elastic_fleet_throttle
+                    echo "Starting dry run..."
-            {
-                block=$'\n'"Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..."$'\n'
-                block+="Upgrading $INTEGRATION..."$'\n'"Starting dry run..."$'\n'
-                fail=""
                     if ! DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID"); then
-                    block+="Error: Failed to complete dry run for '$INTEGRATION_ID'."$'\n'
+                        exit 1
-                    fail="dryrun $INTEGRATION"
+                    fi
-                elif [[ "$(jq .[].hasErrors <<<"$DRYRUN_OUTPUT")" == "false" ]]; then
+                    DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors)
-                    block+="No errors detected. Proceeding with upgrade..."$'\n'
+
+                    # If no errors with dry run, proceed with actual upgrade
+                    if [[ "$DRYRUN_ERRORS" == "false" ]]; then
+                        echo "No errors detected. Proceeding with upgrade..."
                         if ! elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"; then
-                        block+="Error: Upgrade failed for $PACKAGE_NAME with integration ID '$INTEGRATION_ID'."$'\n'
+                            echo "Error: Upgrade failed for $PACKAGE_NAME with integration ID '$INTEGRATION_ID'."
-                        fail="upgrade $INTEGRATION"
+                            ERROR=true
+                            continue
                         fi
                     else
-                    block+="Errors detected during dry run for $PACKAGE_NAME policy upgrade..."$'\n'
+                        echo "Errors detected during dry run for $PACKAGE_NAME policy upgrade..."
-                    fail="dryrun-errors $INTEGRATION"
+                        ERROR=true
+                        continue
                     fi
-                {
-                    flock 9
-                    printf '%s' "$block"
-                    [ -n "$fail" ] && printf '%s\n' "$fail" >>"$FAIL_FILE"
-                } 9>>"$OUTPUT_LOCK"
-            } &
                 fi
-    done < <(jq -r --argjson defaults "$default_packages_json" '
-        .item.package_policies[]
-        | select(.name != "elastic-defend-endpoints")
-        | select(.name | startswith("fleet_server-") | not)
             {%- if not AUTO_UPGRADE_INTEGRATIONS %}
-        | select(.package.name | IN($defaults[]))
+            fi
             {%- endif %}
-        | [.name, .package.name, .package.version, .id] | @tsv
+        fi
-    ' <<<"$POLICY_JSON")
+    done
 done
-
+if [[ "$ERROR" == "true" ]]; then
-# Barrier: wait for every dispatched dry-run/upgrade job to finish.
-wait
-
-if [ -s "$FAIL_FILE" ]; then
-    printf '\nFailed integration upgrades:\n'
-    cat "$FAIL_FILE"
     exit 1
 fi
 echo

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load

@@ -16,6 +16,7 @@
 STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt
 INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json
 BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
+BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
 BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json
 INTEGRATION_PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
 INPUT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_input_package_components.json
@@ -28,6 +29,29 @@ PENDING_UPDATE=false
 #   Requiring some level of manual Elastic Stack configuration before installation
 EXCLUDED_INTEGRATIONS=('apm')
 
+version_conversion(){
+    version=$1
+    echo "$version" | awk -F '.' '{ printf("%d%03d%03d\n", $1, $2, $3); }'
+}
+
+compare_versions() {
+    version1=$1
+    version2=$2
+
+    # Convert versions to numbers
+    num1=$(version_conversion "$version1")
+    num2=$(version_conversion "$version2")
+
+    # Compare using bc
+    if (( $(echo "$num1 < $num2" | bc -l) )); then
+        echo "less"
+    elif (( $(echo "$num1 > $num2" | bc -l) )); then
+        echo "greater"
+    else
+        echo "equal"
+    fi
+}
+
 IFS=$'\n'
 agent_policies=$(elastic_fleet_agent_policy_ids)
 if [ $? -ne 0 ]; then
@@ -39,23 +63,23 @@ default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.l
 
 in_use_integrations=()
 
-# Fetch each agent policy once; its package_policies[] already contain both the integration name
-#  and the .package.name, so extract all non-default package names locally in a single jq instead
-#  of re-fetching the same policy per integration.
-default_packages_json=$(printf '%s\n' "${default_packages[@]}" | jq -R . | jq -s '.')
 for AGENT_POLICY in $agent_policies; do
 
-    if ! policy_json=$(fleet_api "agent_policies/$AGENT_POLICY"); then
+    if ! integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY"); then
         # skip the agent policy if we can't get required info, let salt retry. Integrations loaded by this script are non-default integrations.
         echo "Skipping $AGENT_POLICY.. "
         continue
     fi
+    for INTEGRATION in $integrations; do
+        if ! PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION"); then
+            echo  "Not adding $INTEGRATION, couldn't get package name"
+            continue
+        fi
         # non-default integrations that are in-use in any policy
-    while IFS= read -r PACKAGE_NAME; do
+        if ! [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then
-        [ -n "$PACKAGE_NAME" ] && in_use_integrations+=("$PACKAGE_NAME")
+            in_use_integrations+=("$PACKAGE_NAME")
-    done < <(jq -r --argjson defaults "$default_packages_json" \
+        fi
-        '.item.package_policies[].package.name | select(. as $n | ($defaults | index($n)) | not)' \
+    done
-        <<<"$policy_json")
 done
 
 if [[ -f $STATE_FILE_SUCCESS  ]]; then
@@ -66,55 +90,72 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
         rm -f $INSTALLED_PACKAGE_LIST
         echo $latest_package_list | jq '{packages: [.items[] | {name: .name, latest_version: .version, installed_version: .installationInfo.version, subscription: .conditions.elastic.subscription }]}' >> $INSTALLED_PACKAGE_LIST
 
-        # Build the bulk install list and the per-package status messages with two jq passes
+        while read -r package; do
-        #  instead of a per-package bash loop. The old loop forked ~10 processes per package
+            # get package details
-        #  (5 jq + awk/bc for the version compare) and re-parsed/rewrote a growing JSON file on
+            package_name=$(echo "$package" | jq -r '.name')
-        #  every add (O(n^2)). Selection and messages below are identical to that logic.
+            latest_version=$(echo "$package" | jq -r '.latest_version')
-        SUB={% if SUB %}true{% else %}false{% endif %}
+            installed_version=$(echo "$package" | jq -r '.installed_version')
-        AUTOUP={% if AUTO_UPGRADE_INTEGRATIONS %}true{% else %}false{% endif %}
+            subscription=$(echo "$package" | jq -r '.subscription')
-        EXCLUDED_JSON=$(printf '%s\n' "${EXCLUDED_INTEGRATIONS[@]}" | jq -R 'select(length>0)' | jq -s '.')
+            bulk_package=$(echo "$package" | jq '{name: .name, version: .latest_version}' )
-        INUSE_JSON=$(printf '%s\n' "${in_use_integrations[@]}" | jq -R 'select(length>0)' | jq -s 'unique')
 
-        # vnum replicates the previous version_conversion (%d%03d%03d of the first three dotted
+            if [[ ! "${EXCLUDED_INTEGRATIONS[@]}" =~ "$package_name" ]]; then
-        #  fields); needs() replicates the excluded/subscription/installed/upgrade/in-use logic.
+            {% if not SUB %}
-        JQ_DECISION='
+                if [[ "$subscription" != "basic" && "$subscription" != "null" && -n "$subscription" ]]; then
-def vnum:
+                    # pass over integrations that require non-basic elastic license
-  [ (split(".")|.[0:3][] | gsub("[^0-9].*";"") | (if .=="" then "0" else . end) | tonumber) ]
+                    echo "$package_name integration requires an Elastic license of $subscription or greater... skipping"
-  | (.[0]//0)*1000000 + (.[1]//0)*1000 + (.[2]//0);
+                    continue
-def needs($sub;$autoup;$excluded;$inuse):
+                else
-  .name as $n
+                    if [[ "$installed_version" == "null" || -z "$installed_version" ]]; then
-  | ($n | IN($excluded[]) | not)
+                        echo "$package_name is not installed... Adding to next update."
-  and ( $sub or (.subscription==null or .subscription=="basic" or .subscription=="") )
+                        jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
-  and ( (.installed_version==null or .installed_version=="")
-        or ( ((.latest_version|vnum) > (.installed_version|vnum))
-             and ( $autoup or ($n | IN($inuse[]) | not) ) ) );'
 
-        JQ_ARGS=(--argjson sub "$SUB" --argjson autoup "$AUTOUP" --argjson excluded "$EXCLUDED_JSON" --argjson inuse "$INUSE_JSON")
-
-        # (a) Per-package status messages (parity with the previous echo output).
-        jq -r "${JQ_ARGS[@]}" "$JQ_DECISION"'
-          .packages[]
-          | .name as $n
-          | if ($n|IN($excluded[])) then "Skipping \($n)..."
-            elif (($sub|not) and (.subscription!=null and .subscription!="basic" and .subscription!="")) then
-                 "\($n) integration requires an Elastic license of \(.subscription) or greater... skipping"
-            elif (.installed_version==null or .installed_version=="") then
-                 "\($n) is not installed... Adding to next update."
-            elif ((.latest_version|vnum) > (.installed_version|vnum)) then
-                 (if ($autoup or ($n|IN($inuse[])|not))
-                  then "\($n) is at version \(.installed_version) latest version is \(.latest_version)... Adding to next update."
-                  else "skipping available upgrade for in use integration - \($n)." end)
-            else empty end
-        ' "$INSTALLED_PACKAGE_LIST"
-
-        # (b) The bulk install list, built in a single pass.
-        jq "${JQ_ARGS[@]}" "$JQ_DECISION"'
-          {packages: [ .packages[] | select(needs($sub;$autoup;$excluded;$inuse)) | {name, version: .latest_version} ]}
-        ' "$INSTALLED_PACKAGE_LIST" > "$BULK_INSTALL_PACKAGE_LIST"
-
-        if jq -e '.packages | length > 0' "$BULK_INSTALL_PACKAGE_LIST" >/dev/null; then
                         PENDING_UPDATE=true
+                    else
+                        results=$(compare_versions "$latest_version" "$installed_version")
+                        if [ $results == "greater" ]; then
+                            {#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations  #}
+                            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                            if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then
+                            {%- endif %}
+                                echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
+                                jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
+
+                                PENDING_UPDATE=true
+                            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                            else
+                                echo "skipping available upgrade for in use integration - $package_name."
                             fi
+                            {%- endif %}
+                        fi
+                    fi
+                fi
+            {% else %}
+                if [[ "$installed_version" == "null" || -z "$installed_version" ]]; then
+                    echo "$package_name is not installed... Adding to next update."
+                    jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
+                    PENDING_UPDATE=true
+                else
+                    results=$(compare_versions "$latest_version" "$installed_version")
+                    if [ $results == "greater" ]; then
+                        {#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations  #}
+                        {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                        if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then
+                        {%- endif %}
+                            echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
+                            jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
+                            PENDING_UPDATE=true
+                        {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                        else
+                            echo "skipping available upgrade for in use integration - $package_name."
+                        fi
+                        {%- endif %}
+                    fi
+                fi
+            {% endif %}
+            else
+                echo "Skipping $package_name..."
+            fi
+        done <<< "$(jq -c '.packages[]' "$INSTALLED_PACKAGE_LIST")"
 
         if [ "$PENDING_UPDATE" = true ]; then
             # Run chunked install of packages

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade

@@ -12,22 +12,17 @@ PKG_LOAD_FAILURES=0
 PKG_LOAD_FAILURES_NAMES=()
 
 {%- for PACKAGE in SUPPORTED_PACKAGES %}
-if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && LATEST_VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
+echo "Upgrading {{ PACKAGE }} package..."
-
+if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
-    if [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then
+    if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
-        echo "{{ PACKAGE }} integration version $INSTALLED_VERSION is already at the reported latest version $LATEST_VERSION, skipping upgrade."
-    else
-        echo "Upgrading {{ PACKAGE }} package to version $LATEST_VERSION..."
-        if ! elastic_fleet_package_install "{{ PACKAGE }}" "$LATEST_VERSION"; then
         PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
         PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
     fi
-    fi
 else
-    echo "ERROR: Failed to get version information for integration {{ PACKAGE }}"
     PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
     PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
 fi
+echo
 {%- endfor %}
 
 if [ $PKG_LOAD_FAILURES -gt 0 ]; then
@@ -40,3 +35,6 @@ if [ $PKG_LOAD_FAILURES -gt 0 ]; then
 else
     echo "Successfully upgraded all packages."
 fi
+
+echo
+/usr/sbin/so-elasticsearch-templates-load

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -181,9 +181,6 @@ if ! elastic_fleet_policy_create "so-grid-nodes_heavy" "SO Grid Nodes - Heavy No
     exit 1
 fi
 
-# Check for package upgrades
-so-elastic-fleet-package-upgrade
-
 # Load Integrations for default policies
 so-elastic-fleet-integration-policy-load
 
@@ -244,37 +241,11 @@ printf '%s\n'\
     "" >> "$global_pillar_file"
 
 # Call Elastic-Fleet Salt State
-printf "\nApplying elasticfleet state\n"
+printf "\nApplying elasticfleet state"
-for state_attempt in {1..3}; do
+salt-call state.apply elasticfleet queue=True
-    if salt-call state.apply elasticfleet queue=True; then
-        break
-    elif [[ $state_attempt -lt 3 ]]; then
-        printf "\nElasticfleet state did not complete successfully... Attempt (%s/3). Retrying...\n" "$state_attempt"
-        sleep 10
-    else
-        printf "\nFailure(s) in elasticfleet state... Exiting...\n"
-        exit 1
-    fi
-done
 
-printf "\nRunning so-elastic-agent-gen-installers\n"
 # Generate installers & install Elastic Agent on the node
-for agent_gen_attempt in {1..3}; do
+so-elastic-agent-gen-installers
-    if so-elastic-agent-gen-installers; then
+printf "\nApplying elasticfleet.install_agent_grid state"
-        break
+salt-call state.apply elasticfleet.install_agent_grid queue=True
-    elif [[ $agent_gen_attempt -lt 3 ]]; then
+exit 0
-        printf "\nUnable to generate Elastic Agent installers... Attempt (%s/3). Retrying...\n" "$agent_gen_attempt"
-        sleep 10
-    else
-        printf "\nFailed to generate Elastic Agent installers after 3 attempts. Exiting...\n"
-        exit 1
-    fi
-done
-
-printf "\nApplying elasticfleet.install_agent_grid state\n"
-if ! salt-call state.apply elasticfleet.install_agent_grid queue=True; then
-    printf "\nFailure(s) in elasticfleet.install_agent_grid state... Exiting...\n"
-    exit 1
-fi
-
-printf "\nElastic Fleet setup completed successfully\n"

salt/elasticsearch/defaults.yaml

@@ -1,6 +1,6 @@
 elasticsearch:
   enabled: false
-  version: 9.3.7
+  version: 9.3.3
   index_clean: true
   data_retention_method: DLM
   vm:

salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2

@@ -118,70 +118,70 @@
     {
       "pipeline": {
         "tag": "pipeline_e16851a7",
-        "name": "logs-pfsense.log-1.25.4-firewall",
+        "name": "logs-pfsense.log-1.25.2-firewall",
         "if": "ctx.event.provider == 'filterlog'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_828590b5",
-        "name": "logs-pfsense.log-1.25.4-openvpn",
+        "name": "logs-pfsense.log-1.25.2-openvpn",
         "if": "ctx.event.provider == 'openvpn'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_9d37039c",
-        "name": "logs-pfsense.log-1.25.4-ipsec",
+        "name": "logs-pfsense.log-1.25.2-ipsec",
         "if": "ctx.event.provider == 'charon'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_ad56bbca",
-        "name": "logs-pfsense.log-1.25.4-dhcp",
+        "name": "logs-pfsense.log-1.25.2-dhcp",
         "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\", \"dnsmasq-dhcp\"].contains(ctx.event.provider)"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_dd85553d",
-        "name": "logs-pfsense.log-1.25.4-unbound",
+        "name": "logs-pfsense.log-1.25.2-unbound",
         "if": "ctx.event.provider == 'unbound'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_720ed255",
-        "name": "logs-pfsense.log-1.25.4-haproxy",
+        "name": "logs-pfsense.log-1.25.2-haproxy",
         "if": "ctx.event.provider == 'haproxy'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_456beba5",
-        "name": "logs-pfsense.log-1.25.4-php-fpm",
+        "name": "logs-pfsense.log-1.25.2-php-fpm",
         "if": "ctx.event.provider == 'php-fpm'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_a0d89375",
-        "name": "logs-pfsense.log-1.25.4-squid",
+        "name": "logs-pfsense.log-1.25.2-squid",
         "if": "ctx.event.provider == 'squid'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_c2f1ed55",
-        "name": "logs-pfsense.log-1.25.4-snort",
+        "name": "logs-pfsense.log-1.25.2-snort",
         "if": "ctx.event.provider == 'snort'"
       }
     },
     {
       "pipeline": {
         "tag":"pipeline_33db1c9e",
-        "name": "logs-pfsense.log-1.25.4-suricata",
+        "name": "logs-pfsense.log-1.25.2-suricata",
         "if": "ctx.event.provider == 'suricata'"
       }
     },

salt/elasticsearch/files/ingest/zeek.ssl

@@ -5,7 +5,6 @@
     { "remove":   { "field": ["host"],     "ignore_failure": true } },
     { "json":             { "field": "message",                 "target_field": "message2",             "ignore_failure": true  } },
     { "rename":         { "field": "message2.version",          "target_field": "ssl.version",          "ignore_missing": true  } },
-    { "set":      { "description": "Set transport for the community_id processor", "if": "ctx.ssl?.version == null || !ctx.ssl.version.startsWith('DTLS')", "field": "network.transport", "value": "tcp", "ignore_failure": true } },
     { "rename":         { "field": "message2.cipher",           "target_field": "ssl.cipher",           "ignore_missing": true  } },
     { "rename":         { "field": "message2.curve",            "target_field": "ssl.curve",            "ignore_missing": true  } },
     { "rename":         { "field": "message2.server_name",      "target_field": "ssl.server_name",              "ignore_missing": true  } },

salt/elasticsearch/soc_elasticsearch.yaml

@@ -645,7 +645,6 @@ elasticsearch:
             global: True
             advanced: True
             helpLink: elasticsearch
-    so-logs-soc: *dataStreamSettings
     so-logs-system_x_auth: *dataStreamSettings
     so-logs-system_x_syslog: *dataStreamSettings
     so-logs-system_x_system: *dataStreamSettings

salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load

@@ -11,8 +11,10 @@ ADDON_STATEFILE_SUCCESS=/opt/so/state/addon_estemplates.txt
 ELASTICSEARCH_TEMPLATES_DIR="/opt/so/conf/elasticsearch/templates"
 SO_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/index"
 ADDON_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/addon-index"
-FAILED_NAMES=()
+SO_LOAD_FAILURES=0
-FAILED_COUNT=0
+ADDON_LOAD_FAILURES=0
+SO_LOAD_FAILURES_NAMES=()
+ADDON_LOAD_FAILURES_NAMES=()
 IS_HEAVYNODE="false"
 FORCE="false"
 VERBOSE="false"
@@ -44,86 +46,20 @@ while [[ $# -gt 0 ]]; do
     shift
 done
 
-# Max number of concurrent template PUT jobs. Override via env if needed.
-MAX_TEMPLATE_JOBS=${MAX_TEMPLATE_JOBS:-10}
-
-# Block until fewer than MAX_TEMPLATE_JOBS background jobs are running.
-template_throttle() {
-    while (( $(jobs -rp | wc -l) >= MAX_TEMPLATE_JOBS )); do
-        wait -n
-    done
-}
-
-# Per-job failure markers and an output lock for serializing parallel job output.
-# Each failed load drops one file (named after the template) into FAIL_DIR; the
-# output of each job is flushed as a single block under flock so concurrent jobs
-# never interleave their (chatty) retry output.
-FAIL_DIR=$(mktemp -d)
-OUTPUT_LOCK="${FAIL_DIR}/.output.lock"
-: > "$OUTPUT_LOCK"
-trap 'rm -rf "$FAIL_DIR"' EXIT
-
-# Record a failure: $1 = the template name/path to report later. Slashes are
-# encoded so the path becomes a safe single filename.
-record_failure() {
-    local marker="${1//\//__}"
-    : > "${FAIL_DIR}/fail.${marker}"
-}
-
-# Populate FAILED_NAMES and FAILED_COUNT from the current phase's markers.
-# Must run in the current shell (not a command substitution) so the array sticks.
-collect_failures() {
-    FAILED_NAMES=()
-    FAILED_COUNT=0
-    local f name
-    shopt -s nullglob
-    for f in "${FAIL_DIR}"/fail.*; do
-        name="${f##*/fail.}"
-        name="${name//__//}"
-        FAILED_NAMES+=("$name")
-        FAILED_COUNT=$((FAILED_COUNT + 1))
-    done
-    shopt -u nullglob
-}
-
-# Clear markers and names between phases so SO and addon counts stay independent.
-reset_failures() {
-    shopt -s nullglob
-    rm -f "${FAIL_DIR}"/fail.*
-    shopt -u nullglob
-    FAILED_NAMES=()
-    FAILED_COUNT=0
-}
-
-# Print a block of text atomically (under the shared output lock) so the output
-# of concurrent background jobs is not interleaved.
-locked_echo() {
-    { flock 9; printf '%s\n' "$1"; } 9>>"$OUTPUT_LOCK"
-}
-
-# Loads one template file via PUT. Intended to be dispatched as a background job.
-#   $1 uri          - e.g. _component_template/foo or _index_template/foo
-#   $2 file         - path to the template JSON
-#   $3 report_name  - name/path to record if this load fails
 load_template() {
     local uri="$1"
     local file="$2"
-    local report_name="$3"
-    local out rc=0 block
 
-    # Capture everything (including retry's diagnostic chatter) into one block so
+    echo "Loading template file $file"
-    # concurrent jobs never interleave; the whole block is flushed under one flock.
+    if ! output=$(retry 3 3 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"); then
-    block="Loading template file $file"$'\n'
+        echo "$output"
-    if ! out=$(retry 3 3 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}" 2>&1); then
+
-        block+="$out"$'\n'
+        return 1
-        rc=1
+
     elif [[ "$VERBOSE" == "true" ]]; then
-        block+="$out"$'\n'
+        echo "$output"
     fi
 
-    { flock 9; printf '%s' "$block"; } 9>>"$OUTPUT_LOCK"
-
-    (( rc != 0 )) && record_failure "$report_name"
 }
 
 check_required_component_template_exists() {
@@ -174,9 +110,6 @@ load_component_templates() {
         return
     fi
 
-    # Dispatch loads as throttled background jobs. The barrier (wait) happens in
-    # the caller after all component groups have been dispatched, since index
-    # templates must not load until every component template is in place.
     for component in "$pattern"/*.json; do
         tmpl_name=$(basename "${component%.json}")
 
@@ -185,8 +118,10 @@ load_component_templates() {
             tmpl_name="${tmpl_name%-mappings}-mappings"
         fi
 
-        template_throttle
+        if ! load_template "_component_template/${tmpl_name}" "$component"; then
-        load_template "_component_template/${tmpl_name}" "$component" "$component" &
+            SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
+            SO_LOAD_FAILURES_NAMES+=("$component")
+        fi
     done
 }
 
@@ -237,9 +172,6 @@ if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]] && index_templates_e
     load_component_templates "Elastic Agent" "elastic-agent"
     load_component_templates "Security Onion" "so"
 
-    # Barrier: every component template PUT must complete before we snapshot the
-    # component template list and start loading index templates that depend on them.
-    wait
     component_templates=$(so-elasticsearch-component-templates-list)
     echo -e "Loading Security Onion index templates...\n"
     for so_idx_tmpl in "${SO_TEMPLATES_DIR}"/*.json; do
@@ -249,7 +181,7 @@ if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]] && index_templates_e
             # TODO: Better way to load only heavynode specific templates
             if ! check_heavynode_compatiable_index_template "$tmpl_name"; then
                 if [[ "$VERBOSE" == "true" ]]; then
-                    locked_echo "Skipping over $so_idx_tmpl, template is not a heavynode specific index template."
+                    echo "Skipping over $so_idx_tmpl, template is not a heavynode specific index template."
                 fi
 
                 continue
@@ -257,34 +189,32 @@ if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]] && index_templates_e
         fi
 
         if check_required_component_template_exists "$so_idx_tmpl"; then
-            template_throttle
+            if ! load_template "_index_template/$tmpl_name" "$so_idx_tmpl"; then
-            load_template "_index_template/$tmpl_name" "$so_idx_tmpl" "$so_idx_tmpl" &
+                SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
+                SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl")
+            fi
         else
-            locked_echo "Skipping over $so_idx_tmpl due to missing required component template(s)."
+            echo "Skipping over $so_idx_tmpl due to missing required component template(s)."
-            record_failure "$so_idx_tmpl"
+            SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
+            SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl")
 
             continue
         fi
     done
 
-    # Barrier: all SO index template PUTs must finish before tallying failures.
+    if [[ $SO_LOAD_FAILURES -eq 0 ]]; then
-    wait
-
-    collect_failures
-    if [[ $FAILED_COUNT -eq 0 ]]; then
         echo "All Security Onion core templates loaded successfully."
 
         touch "$SO_STATEFILE_SUCCESS"
     else
-        echo "Encountered $FAILED_COUNT failure(s) loading templates:"
+        echo "Encountered $SO_LOAD_FAILURES failure(s) loading templates:"
-        for failed_template in "${FAILED_NAMES[@]}"; do
+        for failed_template in "${SO_LOAD_FAILURES_NAMES[@]}"; do
             echo "  - $failed_template"
         done
         if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then
             fail "Failed to load all Security Onion core templates successfully."
         fi
     fi
-    reset_failures
 elif ! index_templates_exist "$SO_TEMPLATES_DIR"; then
     echo "No Security Onion core index templates found in ${SO_TEMPLATES_DIR}, skipping."
 elif [[ -f "$SO_STATEFILE_SUCCESS" ]]; then
@@ -303,27 +233,26 @@ if should_load_addon_templates; then
         tmpl_name=$(basename "${addon_idx_tmpl%-template.json}")
 
         if check_required_component_template_exists "$addon_idx_tmpl"; then
-            template_throttle
+            if ! load_template "_index_template/${tmpl_name}" "$addon_idx_tmpl"; then
-            load_template "_index_template/${tmpl_name}" "$addon_idx_tmpl" "$addon_idx_tmpl" &
+                ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1))
+                ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl")
+            fi
         else
-            locked_echo "Skipping over $addon_idx_tmpl due to missing required component template(s)."
+            echo "Skipping over $addon_idx_tmpl due to missing required component template(s)."
-            record_failure "$addon_idx_tmpl"
+            ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1))
+            ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl")
 
             continue
         fi
     done
 
-    # Barrier: all addon index template PUTs must finish before tallying failures.
+    if [[ $ADDON_LOAD_FAILURES -eq 0 ]]; then
-    wait
-
-    collect_failures
-    if [[ $FAILED_COUNT -eq 0 ]]; then
         echo "All addon integration templates loaded successfully."
 
         touch "$ADDON_STATEFILE_SUCCESS"
     else
-        echo "Encountered $FAILED_COUNT failure(s) loading addon integration templates:"
+        echo "Encountered $ADDON_LOAD_FAILURES failure(s) loading addon integration templates:"
-        for failed_template in "${FAILED_NAMES[@]}"; do
+        for failed_template in "${ADDON_LOAD_FAILURES_NAMES[@]}"; do
             echo "  - $failed_template"
         done
         if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load

@@ -6,48 +6,6 @@
 
 . /usr/sbin/so-common
 
-MAX_JOBS=${MAX_ILM_JOBS:-10}
-
-# Lock used to serialize block writes so concurrent jobs never interleave their output.
-ILM_OUTPUT_LOCK=$(mktemp)
-ILM_FAIL_FILE=$(mktemp)
-trap 'rm -f "$ILM_OUTPUT_LOCK" "$ILM_FAIL_FILE"' EXIT
-
-# Policies are loaded concurrently (up to MAX_JOBS at a time) for speed. Each policy's block is
-# printed the moment its curl returns, so output appears in COMPLETION ORDER, not the order
-# policies are defined in configuration.
-echo "Loading ILM policies concurrently; output below appears in completion order, not configuration order."
-echo
-
-put_policy() {
-  local desc="$1" policyname="$2" data="$3" result rc=0
-  if ! result=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L --fail \
-    -X PUT "https://localhost:9200/_ilm/policy/${policyname}" \
-    -H 'Content-Type: application/json' -d"${data}" 2>&1); then
-    rc=1
-  elif ! jq -e '.acknowledged == true' <<<"$result" >/dev/null 2>&1; then
-    rc=1
-  fi
-
-  # curl above ran in parallel; serialize just this block write so concurrent jobs never interleave.
-  {
-    flock 200
-    printf 'Setting up %s policy...\n%s\n\n' "${desc}" "${result}"
-    if (( rc != 0 )); then
-      printf '%s\n' "${policyname}" >>"$ILM_FAIL_FILE"
-    fi
-  } 200>>"${ILM_OUTPUT_LOCK}"
-
-  return "$rc"
-}
-
-# Block until fewer than MAX_JOBS background curls are running.
-throttle() {
-  while (( $(jobs -rp | wc -l) >= MAX_JOBS )); do
-    wait -n || true
-  done
-}
-
 {%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
 {%- if GLOBALS.role != "so-heavynode" %}
 {%-   from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
@@ -56,36 +14,35 @@ throttle() {
 {%- for index, settings in ES_INDEX_SETTINGS.items() %}
 {%-   if settings.policy is defined %}
 {%-     if index == 'so-logs-detections.alerts' %}
-  throttle
+  echo
-  put_policy "so-logs-detections.alerts-so" "{{ index }}-so" '{ "policy": {{ settings.policy | tojson(true) }} }' &
+  echo "Setting up so-logs-detections.alerts-so policy..."
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-so" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+  echo
 {%-     elif index == 'so-logs-soc' %}
-  throttle
+  echo
-  put_policy "so-soc-logs" "so-soc-logs" '{ "policy": {{ settings.policy | tojson(true) }} }' &
+  echo "Setting up so-soc-logs policy..."
-  throttle
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/so-soc-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
-  put_policy "{{ index }}-logs" "{{ index }}-logs" '{ "policy": {{ settings.policy | tojson(true) }} }' &
+  echo
+  echo
+  echo "Setting up {{ index }}-logs policy..."
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+  echo
 {%-     else %}
-  throttle
+  echo
-  put_policy "{{ index }}-logs" "{{ index }}-logs" '{ "policy": {{ settings.policy | tojson(true) }} }' &
+  echo "Setting up {{ index }}-logs policy..."
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+  echo
 {%-     endif %}
 {%-   endif %}
 {%- endfor %}
+echo
 {%- if GLOBALS.role != "so-heavynode" %}
 {%-   for index, settings in ALL_ADDON_SETTINGS.items() %}
 {%-     if settings.policy is defined %}
-  throttle
+  echo
-  put_policy "{{ index }}-logs" "{{ index }}-logs" '{ "policy": {{ settings.policy | tojson(true) }} }' &
+  echo "Setting up {{ index }}-logs policy..."
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+  echo
 {%-     endif %}
 {%-   endfor %}
 {%- endif %}
-
-wait || true
-
-if [[ -s "$ILM_FAIL_FILE" ]]; then
-  echo "ERROR: Failed to load ILM policy(s):"
-  while read -r POLICY; do
-    echo " - $POLICY"
-  done < "$ILM_FAIL_FILE"
-  exit 1
-else
-  echo "Successfully loaded all ILM policies."
-fi

salt/global/defaults.yaml

@@ -1,3 +1,10 @@
 global:
   pcapengine: SURICATA
   pipeline: REDIS
+  push:
+    enabled: true
+    highstate_interval_hours: 2
+    debounce_seconds: 30
+    drain_interval: 15
+    batch: '25%'
+    batch_wait: 15

salt/global/soc_global.yaml

@@ -59,4 +59,41 @@ global:
     description: Allows use of Endgame with Security Onion. This feature requires a license from Endgame.
     global: True
     advanced: True
+  push:
+    enabled:
+      description: Master kill-switch for the active push feature. When disabled, rule and pillar changes are picked up at the next scheduled highstate instead of being pushed immediately.
+      forcedType: bool
+      helpLink: push
+      global: True
+    highstate_interval_hours:
+      description: How often every minion in the grid runs a scheduled state.highstate, in hours. Lower values keep minions closer in sync at the cost of more load; higher values reduce load but increase worst-case latency for non-pushed changes. The salt-minion health check restarts a minion if its last highstate is older than this value plus one hour.
+      forcedType: int
+      helpLink: push
+      global: True
+      advanced: True
+    debounce_seconds:
+      description: Trailing-edge debounce window in seconds. A push intent must be quiet for this long before the drainer dispatches. Rapid bursts of edits within this window coalesce into one dispatch.
+      forcedType: int
+      helpLink: push
+      global: True
+      advanced: True
+    drain_interval:
+      description: How often the push drainer checks for ready intents, in seconds. Small values lower dispatch latency at the cost of more background work on the manager.
+      forcedType: int
+      helpLink: push
+      global: True
+      advanced: True
+    batch:
+      description: "Host batch size for push orchestrations. A number (e.g. '10') or a percentage (e.g. '25%'). Limits how many minions run the push state at once so large fleets don't thundering-herd."
+      helpLink: push
+      global: True
+      advanced: True
+      regex: '^([0-9]+%?)$'
+      regexFailureMessage: Enter a whole number or a whole-number percentage (e.g. 10 or 25%).
+    batch_wait:
+      description: Seconds to wait between host batches in a push orchestration. Gives the fleet time to breathe between waves.
+      forcedType: int
+      helpLink: push
+      global: True
+      advanced: True
 

salt/hydra/enabled.sls

@@ -22,7 +22,6 @@ include:
 so-hydra:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-hydra:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: hydra
     - name: so-hydra
     - networks:
@@ -59,6 +58,8 @@ so-hydra:
       - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
     {%   endfor %}
     {% endif %}
+    # Intentionally unless-stopped -- matches the fleet default.
+    - restart_policy: unless-stopped
     - watch:
       - file: hydraconfig
     - require:

salt/kibana/defaults.yaml

@@ -22,7 +22,7 @@ kibana:
           - default
           - file
     migrations:
-      discardCorruptObjects: "9.3.7"
+      discardCorruptObjects: "9.3.3"
     telemetry:
       enabled: False
     xpack:

salt/kibana/enabled.sls

@@ -70,7 +70,7 @@ wait_for_so-kibana:
     - ssl: True
     - verify_ssl: False
     - status: 200
-    - wait_for: 600
+    - wait_for: 300
     - request_interval: 15
     - require:
       - docker_container: so-kibana

salt/kratos/enabled.sls

@@ -15,7 +15,6 @@ include:
 so-kratos:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kratos:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: kratos
     - name: so-kratos
     - networks:
@@ -52,6 +51,8 @@ so-kratos:
       - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
     {%   endfor %}
     {% endif %}
+    # Intentionally unless-stopped -- matches the fleet default.
+    - restart_policy: unless-stopped
     - watch:
       - file: kratosschema
       - file: kratosconfig

salt/libvirt/dev/saltDev.sls

@@ -1,67 +0,0 @@
-# This state is designed to run on a development manager running in a libvirt VM. It will map the default pillar and salt directories
-# from /opt/so/saltstack/default to your local development machine as the source path.
-# The VM requires a filesystem to be added. Only the source path should be changed to your development codebase
-#  Driver: virtio-9p
-#  Source path: ~/project/securityonion
-#  Target path: saltDev
-
-# If you want a directory to be RW, then kvm must have group privileges.
-# ll /home/user/projects/securityonion/salt/hypervisor
-# total 48
-# drwxrwxr-x  3 user kvm 4096 Feb 13 11:18 ./
-# drwxrwxr-x 64 user user 4096 Feb 13 10:32 ../
-# -rw-rw-r--  1 user kvm 2238 Feb 12 15:06 defaults.yaml
-# -rw-rw-r--  1 user kvm 1467 Feb 12 15:06 init.sls
-# -rw-rw-r--  1 user kvm   70 Feb 13 09:37 soc_hypervisor.yaml
-# drwxrwxr-x  3 user kvm 4096 Feb 12 15:06 tools/
-
-# Ensure required kernel modules are configured for loading
-/etc/modules-load.d/virtio-9p.conf:
-  file.managed:
-    - contents: |
-        9pnet_virtio
-        9pnet
-        9p
-    - mode: 644
-    - user: root
-    - group: root
-
-# Load the kernel modules immediately (in the correct order)
-load_9p_modules:
-  cmd.run:
-    - names:
-      - modprobe 9pnet_virtio
-      - modprobe 9pnet
-      - modprobe 9p
-    - unless: lsmod | grep -E '9pnet_virtio|9pnet|9p'
-
-# Ensure mount point exists
-/opt/so/saltstack/default:
-  file.directory:
-    - user: root
-    - group: root
-    - mode: 755
-    - makedirs: True
-
-# Configure fstab entry using mount.fstab_present
-# Configure fstab entry using mount.fstab_present
-saltdev_fstab:
-  mount.fstab_present:
-    - name: saltDev
-    - fs_file: /opt/so/saltstack/default
-    - fs_vfstype: 9p
-    - fs_mntops: _netdev,trans=virtio,version=9p2000.L
-    - fs_freq: 0
-    - fs_passno: 0
-
-# Mount the filesystem if not already mounted
-mount_saltdev:
-  mount.mounted:
-    - name: /opt/so/saltstack/default
-    - device: saltDev
-    - fstype: 9p
-    - opts: _netdev,trans=virtio,version=9p2000.L
-    - require:
-      - file: /opt/so/saltstack/default
-      - mount: saltdev_fstab
-      - cmd: load_9p_modules

salt/logrotate/defaults.yaml

@@ -150,16 +150,6 @@ logrotate:
       - extension .log
       - dateext
       - dateyesterday
-    /opt/so/log/postgres/*_x_log:
-      - daily
-      - rotate 14
-      - missingok
-      - copytruncate
-      - compress
-      - create
-      - extension .log
-      - dateext
-      - dateyesterday
     /opt/so/log/telegraf/*_x_log:
       - daily
       - rotate 14

salt/logrotate/soc_logrotate.yaml

@@ -91,13 +91,6 @@ logrotate:
       multiline: True
       global: True
       forcedType: "[]string"
-    "/opt/so/log/postgres/*_x_log":
-      description: List of logrotate options for this file.
-      title: /opt/so/log/postgres/*.log
-      advanced: True
-      multiline: True
-      global: True
-      forcedType: "[]string"
     "/opt/so/log/telegraf/*_x_log":
       description: List of logrotate options for this file.
       title: /opt/so/log/telegraf/*.log

salt/manager/beacons.sls

@@ -1,10 +1,10 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
-{% from 'salt/auto_apply.map.jinja' import AUTOAPPLY %}
+{% from 'global/map.jinja' import GLOBALMERGED %}
 
 include:
   - salt.minion
 
-{% if GLOBALS.is_manager and AUTOAPPLY.enabled %}
+{% if GLOBALS.is_manager and GLOBALMERGED.push.enabled %}
 salt_beacons_pushstate:
   file.managed:
     - name: /etc/salt/minion.d/beacons_pushstate.conf

salt/manager/files/beacons_pushstate.conf.jinja

@@ -1,11 +1,41 @@
-{% from 'salt/auto_apply.map.jinja' import AUTOAPPLY %}
+{% from 'global/map.jinja' import GLOBALMERGED %}
 beacons:
-  postgres_pillar_beacon:
+  pillar_db:
-    - interval: {{ AUTOAPPLY.drain_interval }}
+    - interval: {{ GLOBALMERGED.push.drain_interval }}
-    - disable_during_state_run: False
+    - disable_during_state_run: True
-  rules_beacon:
+  inotify:
-    - interval: {{ AUTOAPPLY.drain_interval }}
+    - disable_during_state_run: True
-    - disable_during_state_run: False
+    - coalesce: True
-    - paths:
+    - files:
-        /opt/so/saltstack/local/salt/suricata/rules: suricata
+        /opt/so/saltstack/local/salt/suricata/rules:
-        /opt/so/saltstack/local/salt/strelka/rules/compiled: strelka
+          mask:
+            - close_write
+            - moved_to
+            - delete
+          recurse: True
+          auto_add: True
+          exclude:
+            - '\.sw[a-z]$':
+                regex: True
+            - '~$':
+                regex: True
+            - '/4913$':
+                regex: True
+            - '/\.#':
+                regex: True
+        /opt/so/saltstack/local/salt/strelka/rules/compiled:
+          mask:
+            - close_write
+            - moved_to
+            - delete
+          recurse: True
+          auto_add: True
+          exclude:
+            - '\.sw[a-z]$':
+                regex: True
+            - '~$':
+                regex: True
+            - '/4913$':
+                regex: True
+            - '/\.#':
+                regex: True

salt/manager/files/mirror-kernel.txt

@@ -1,2 +0,0 @@
-https://repo.securityonion.net/file/so-repo/prod/3/oracle/9-uek8
-https://repo-alt.securityonion.net/prod/3/oracle/9-uek8

salt/manager/files/repodownload.conf

@@ -11,8 +11,3 @@ name=Security Onion Repo repo
 mirrorlist=file:///opt/so/conf/reposync/mirror.txt
 enabled=1
 gpgcheck=1
-[securityonionkernelsync]
-name=Security Onion Kernel Repo repo
-mirrorlist=file:///opt/so/conf/reposync/mirror-kernel.txt
-enabled=1
-gpgcheck=1

salt/manager/init.sls

@@ -87,28 +87,6 @@ repo_dir:
       - group
     - show_changes: False
 
-kernelrepo_dir:
-  file.directory:
-    - name: /nsm/kernelrepo
-    - user: socore
-    - group: socore
-    - recurse:
-      - user
-      - group
-    - show_changes: False
-
-# Ensure /nsm/kernelrepo is always a valid (if empty) repo before it is ever assigned to
-# a client. Without repodata/repomd.xml an enabled file:///nsm/kernelrepo repo makes every
-# dnf operation fail; so-repo-sync only populates it after the highstate, so seed an empty
-# repo here. Only runs when repodata is missing, so it won't clobber a synced repo.
-kernelrepo_init_empty:
-  cmd.run:
-    - name: createrepo /nsm/kernelrepo
-    - unless: 'test -e /nsm/kernelrepo/repodata/repomd.xml'
-    - require:
-      - file: kernelrepo_dir
-      - pkg: install_createrepo
-
 manager_sbin:
   file.recurse:
     - name: /usr/sbin
@@ -145,13 +123,6 @@ so-repo-mirrorlist:
     - user: socore
     - group: socore
 
-so-repo-kernel-mirrorlist:
-  file.managed:
-    - name: /opt/so/conf/reposync/mirror-kernel.txt
-    - source: salt://manager/files/mirror-kernel.txt
-    - user: socore
-    - group: socore
-
 so-repo-sync:
   {%     if MANAGERMERGED.reposync.enabled %}
   cron.present:

salt/manager/tools/sbin/so-push-drainer

@@ -10,7 +10,7 @@ so-push-drainer
 ===============
 
 Scheduled drainer for the active-push feature. Runs on the manager every
-drain_interval seconds (default 15) via a salt schedule in salt/salt/push_drain_schedule.sls.
+drain_interval seconds (default 15) via a salt schedule in salt/schedule.sls.
 
 For each intent file under /opt/so/state/push_pending/*.json whose last_touch
 is older than debounce_seconds, this script:
@@ -21,7 +21,8 @@ is older than debounce_seconds, this script:
   * deletes the contributed intent files on successful dispatch
 
 Reactor sls files (push_suricata, push_strelka, push_pillar) write intents
-but never dispatch directly
+but never dispatch directly -- see plan
+/home/mreeves/.claude/plans/goofy-marinating-hummingbird.md for the full design.
 """
 
 import fcntl
@@ -59,9 +60,9 @@ def _make_logger():
 
 
 def _load_push_cfg():
-    """Read the salt:auto_apply pillar subtree via salt-call. Returns a dict."""
+    """Read the global:push pillar subtree via salt-call. Returns a dict."""
     caller = salt.client.Caller()
-    cfg = caller.cmd('pillar.get', 'salt:auto_apply', {})
+    cfg = caller.cmd('pillar.get', 'global:push', {})
     return cfg if isinstance(cfg, dict) else {}
 
 
@@ -134,7 +135,7 @@ def main():
     try:
         push = _load_push_cfg()
     except Exception:
-        log.exception('failed to read salt:auto_apply pillar; aborting drain pass')
+        log.exception('failed to read global:push pillar; aborting drain pass')
         return 1
 
     if not push.get('enabled', True):

salt/manager/tools/sbin/so-repo-sync

@@ -10,16 +10,5 @@ NOROOT=1
 set -e
 
 curl --retry 5 --retry-delay 60 -A "reposync/$(sync_options)" https://sigs.securityonion.net/checkup --output /tmp/checkup
-
 dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/
 createrepo /nsm/repo
-
-# The kernel repo section is deployed to repodownload.conf by the manager highstate, which
-# runs AFTER this script during soup. On the first upgrade to a kernel-aware version the
-# on-disk config still predates the section, so guard on its presence to avoid dnf's
-# "Unknown repo: 'securityonionkernelsync'" aborting the sync (set -e). The next sync after the
-# highstate deploys the section will pick it up.
-if grep -q '^\[securityonionkernelsync\]' /opt/so/conf/reposync/repodownload.conf; then
-  dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernelsync --download-metadata -p /nsm/kernelrepo/
-  createrepo /nsm/kernelrepo
-fi

salt/manager/tools/sbin/soup

@@ -12,17 +12,7 @@
 UPDATE_DIR=/tmp/sogh/securityonion
 DEFAULT_SALT_DIR=/opt/so/saltstack/default
 INSTALLEDVERSION=$(cat /etc/soversion)
-# /etc/sopostversion is a soup-owned marker (no salt state manages it) tracking how
+POSTVERSION=$INSTALLEDVERSION
-# far the post-upgrade walk has progressed. Its presence means a prior upgrade did
-# not finish its post-upgrade steps; its contents are the resume point. It is read
-# here before preupgrade_changes mutates INSTALLEDVERSION and before any highstate
-# stamps /etc/soversion from the pillar.
-POSTVERSION_FILE=/etc/sopostversion
-if [ -f "$POSTVERSION_FILE" ]; then
-  POSTVERSION=$(cat "$POSTVERSION_FILE")
-else
-  POSTVERSION=$INSTALLEDVERSION
-fi
 INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
 BATCHSIZE=5
 SOUP_LOG=/root/soup.log
@@ -33,10 +23,6 @@ NOTIFYCUSTOMELASTICCONFIG=false
 TOPFILE=/opt/so/saltstack/default/salt/top.sls
 BACKUPTOPFILE=/opt/so/saltstack/default/salt/top.sls.backup
 SALTUPGRADED=false
-# Set true once soup begins modifying the system (past the pre-flight checks), so the
-# EXIT trap can tell the user the update did not finish and must be re-run. Only the
-# pre-flight gates (ES compatibility, disk, network) fail before this is set.
-SOUP_UPGRADE_STARTED=false
 SALT_CLOUD_INSTALLED=false
 SALT_CLOUD_CONFIGURED=false
 # Check if salt-cloud is installed
@@ -137,28 +123,6 @@ check_err() {
 
     echo "SOUP XTRACE debug log (if enabled) at $SOUP_DEBUG_LOG. Re-run soup with SOUP_DEBUG=1 to create $SOUP_DEBUG_LOG"
 
-    # If soup had already started modifying the system, make it unmistakable that the
-    # update is incomplete and must be re-run. soup is resumable: a version upgrade
-    # picks up from the /etc/sopostversion marker, and a hotfix re-applies because
-    # /etc/sohotfix is only advanced after a successful highstate.
-    if [[ "$SOUP_UPGRADE_STARTED" == "true" ]]; then
-      echo ""
-      echo "=============================================================================="
-      echo " UPGRADE INCOMPLETE"
-      echo "=============================================================================="
-      echo " This soup run did NOT finish. Your Security Onion installation may be in a"
-      echo " partially-updated state and is not yet fully upgraded."
-      echo ""
-      echo " Review the error above and $SOUP_LOG, resolve the underlying problem, then"
-      echo " run soup again to resume and complete the update:"
-      echo ""
-      echo "     sudo soup"
-      echo ""
-      echo " soup is resumable -- re-running it continues from where this run stopped."
-      echo "=============================================================================="
-      echo ""
-    fi
-
     exit $exit_code
   fi
 
@@ -281,7 +245,6 @@ check_airgap() {
       UPDATE_DIR=/tmp/soagupdate/SecurityOnion
       AGDOCKER=/tmp/soagupdate/docker
       AGREPO=/tmp/soagupdate/minimal/Packages
-      AGUEKREPO=/tmp/soagupdate/uek/Packages
   else
       is_airgap=1
   fi
@@ -327,30 +290,6 @@ check_pillar_items() {
   fi
 }
 
-check_cluster_health() {
-  echo "Checking Elasticsearch cluster health."
-  # Require a 'green' cluster before upgrading; anything less (yellow, red, or
-  # unreachable) blocks. Modeled on the wait used in so-elasticsearch-roles-load.
-  if so-elasticsearch-query "_cluster/health?wait_for_status=green&timeout=120s" --fail > /dev/null 2>&1; then
-    printf "\nThe Elasticsearch cluster is healthy (green). We can proceed with SOUP.\n\n"
-  else
-    printf "\nThe Elasticsearch cluster is not green. Please resolve the cluster health issue so the cluster is green before running SOUP again.\n\n"
-    exit 0
-  fi
-}
-
-check_fleet_server() {
-  echo "Checking that Elastic Fleet Server is responding."
-  # Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls,
-  # which waits for HTTP 200 from the Fleet Server status API.
-  if curl -sk --fail --retry 3 --retry-delay 10 --max-time 30 "https://localhost:8220/api/status" > /dev/null 2>&1; then
-    printf "\nElastic Fleet Server is responding. We can proceed with SOUP.\n\n"
-  else
-    printf "\nElastic Fleet Server is not responding at https://localhost:8220/api/status. Please ensure Elastic Fleet is healthy before running SOUP again.\n\n"
-    exit 0
-  fi
-}
-
 check_saltmaster_status() {
   set +e
   echo "Waiting on the Salt Master service to be ready."
@@ -437,8 +376,6 @@ get_soup_script_hashes() {
   GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}')
   CURRENTSOFIREWALL=$(md5sum /usr/sbin/so-firewall | awk '{print $1}')
   GITSOFIREWALL=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-firewall | awk '{print $1}')
-  CURRENTSOYAML=$(md5sum /usr/sbin/so-yaml.py | awk '{print $1}')
-  GITSOYAML=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py | awk '{print $1}')
 }
 
 highstate() {
@@ -476,13 +413,6 @@ preupgrade_changes() {
     true
 }
 
-set_postversion() {
-    # Persist post-upgrade walk progress so an interrupted upgrade can resume the
-    # remaining steps on the next soup run (see /etc/sopostversion handling).
-    POSTVERSION="$1"
-    echo "$POSTVERSION" > "$POSTVERSION_FILE"
-}
-
 postupgrade_changes() {
     # This function is to add any new pillar items if needed.
     echo "Running post upgrade processes."
@@ -490,8 +420,6 @@ postupgrade_changes() {
     [[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0
     [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0
     [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0
-    # All applicable post-upgrade steps completed; clear the resume marker.
-    rm -f "$POSTVERSION_FILE"
     true
 }
 
@@ -584,7 +512,7 @@ post_to_3.0.0() {
   # convert yes/no in suricata pillars to true/false
   convert_suricata_yes_no
 
-  set_postversion 3.0.0
+  POSTVERSION=3.0.0
 }
 
 ### 3.0.0 End ###
@@ -762,21 +690,6 @@ ensure_postgres_local_pillar() {
   chown -R socore:socore "$dir"
 }
 
-ensure_salt_local_pillar() {
-  # The salt.auto_apply settings are a new SOC settings
-  # module, so the new pillar/top.sls references salt.soc_salt / salt.adv_salt
-  # unconditionally. Managers upgrading from before this change have no
-  # /opt/so/saltstack/local/pillar/salt/ (make_some_dirs only runs at install
-  # time), so the stubs must be created here before salt-master restarts against
-  # the new top.sls.
-  echo "Ensuring salt local pillar stubs exist."
-  local dir=/opt/so/saltstack/local/pillar/salt
-  mkdir -p "$dir"
-  [[ -f "$dir/soc_salt.sls" ]] || touch "$dir/soc_salt.sls"
-  [[ -f "$dir/adv_salt.sls" ]] || touch "$dir/adv_salt.sls"
-  chown -R socore:socore "$dir"
-}
-
 ensure_postgres_secret() {
   # On a fresh install, generate_passwords + secrets_pillar seed
   # secrets:postgres_pass in /opt/so/saltstack/local/pillar/secrets.sls. That
@@ -826,6 +739,7 @@ fix_logstash_0013_lumberjack_pipeline_name() {
 up_to_3.1.0() {
   ensure_postgres_local_pillar
   ensure_postgres_secret
+  determine_elastic_agent_upgrade
   elasticsearch_backup_index_templates
   # Clear existing component template state file.
   rm -f /opt/so/state/esfleet_component_templates.json
@@ -862,39 +776,33 @@ post_to_3.1.0() {
   # Check for unhealthy / unauthorized integration transform jobs and attempt reauthorizations
   check_transform_health_and_reauthorize || true
 
-  set_postversion 3.1.0
+  POSTVERSION=3.1.0
 }
 
 ### 3.1.0 End ###
 
 ### 3.2.0 Scripts ###
 
-recollate_postgres() {
+bootstrap_so_soc_database() {
-  echo ""
+  # init-db.sh is mounted into so-postgres at /docker-entrypoint-initdb.d/init-db.sh
-  echo "Recollating PostgreSQL databases. The following output may contain warnings about a version mismatch, followed by a note indicating that the collation version has been changed."
+  # and runs automatically only on a fresh data directory. Hosts upgrading from
-  for db in postgres securityonion so_telegraf; do
+  # 3.1.0 already have /nsm/postgres populated, so the so_soc bootstrap block
-    docker exec so-postgres psql -U postgres $db -c "reindex database $db"
+  # added in 3.2 never fires. Re-run the script explicitly; it's idempotent.
-    docker exec so-postgres psql -U postgres $db -c "alter database $db refresh collation version"
+  echo "Bootstrapping so_soc database via init-db.sh."
-  done
+  # The postgres image has no USER directive, so `docker exec` defaults to
-  echo "Recollating PostgreSQL databases complete."
+  # root, and the container env intentionally omits POSTGRES_USER (the upstream
-  echo ""
+  # entrypoint defaults it transiently during first-init only). Recreate both
-}
+  # so psql inside init-db.sh resolves the connect user correctly.
-
+  local exec_cmd="docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh"
-scrub_postgres_log_passwords() {
+  if ! /usr/sbin/so-postgres-wait; then
-  # Purge plaintext passwords a pre-3.2 postgres could log on DDL errors.
+    FINAL_MESSAGE_QUEUE+=("WARNING: so-postgres was not ready during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd")
-  local log=/opt/so/log/postgres/postgres.log
-  [[ -f "$log" ]] || return 0
-  if ! grep -qai "PASSWORD '" "$log" 2>/dev/null; then
-    echo "No leaked passwords found in $log."
     return 0
   fi
-  echo "Removing leaked password statements from $log."
+  if ! $exec_cmd; then
-  local tmp
+    FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd")
-  tmp=$(mktemp)
+    return 0
-  # Rewrite in place (cat >) to keep the inode postgres is writing to.
+  fi
-  grep -avi "PASSWORD '" "$log" > "$tmp" 2>/dev/null || true
+  echo "so_soc bootstrap complete."
-  cat "$tmp" > "$log"
-  rm -f "$tmp"
 }
 
 # Existing grids should keep ILM unless an admin explicitly opts in to DLM.
@@ -942,34 +850,7 @@ kibana_backport_streams_index_template() {
 
 }
 
-# Runs kafka-features.sh upgrade --release-version $1
-#   Upgrades Kafka KRaft cluster metadata
-update_kafka_metadata() {
-    metadata_version="$1"
-    global_pillar="/opt/so/saltstack/local/pillar/global/soc_global.sls"
-    if PIPELINE=$(so-yaml.py get -r "$global_pillar" global.pipeline 2> /dev/null) && [[ "$PIPELINE" == "KAFKA" ]]; then
-        kafka_nodes_raw=$(salt-call pillar.get kafka:nodes --out=json)
-        if kafka_nodes=$(jq -er '.local | select(type == "object" and length > 0)' <<< "$kafka_nodes_raw"); then
-            bootstrap_servers=$(jq -r '[to_entries[] | select(.value.role | contains("broker")) | "\(.value.ip):9092"] | join(",")' <<< "$kafka_nodes")
-            echo "Upgrading Kafka KRaft cluster version"
-            so-kafka-cli kafka-features.sh --bootstrap-server "$bootstrap_servers" --command-config /opt/kafka/config/kraft/client.properties upgrade --release-version "$metadata_version" 2>/dev/null || true
-
-            return 0
-        else
-            FINAL_MESSAGE_QUEUE+=("WARNING: Unable to automatically perform Kafka KRaft cluster metadata update. This step can be performed manually using the following command (replacing \$BROKER_IP with the ip of atleast 1 available Kafka broker):")
-            FINAL_MESSAGE_QUEUE+=("    - so-kafka-cli kafka-features.sh --bootstrap-server \$BROKER_IP:9092 --command-config /opt/kafka/config/kraft/client.properties upgrade --release-version $metadata_version")
-        fi
-    else
-        echo "Nothing to do!"
-    fi
-}
-
 up_to_3.2.0() {
-  ensure_salt_local_pillar
-
-  # download 9.3.7 elastic agent packages
-  determine_elastic_agent_upgrade
-
   fix_logstash_0013_lumberjack_pipeline_name
 
   pin_elasticsearch_data_retention_method
@@ -978,21 +859,15 @@ up_to_3.2.0() {
 }
 
 post_to_3.2.0() {
-  # Recollate due to image OS rebase
+  bootstrap_so_soc_database
-  recollate_postgres
 
-  # SOC database bootstrap is handled by the postgres.enabled highstate.
+  # Including agent regen script here since it was missed in post_to_3.1.0
-  scrub_postgres_log_passwords
-
-  # Generate 9.3.7 elastic agent installers
   echo "Regenerating Elastic Agent Installers"
   /sbin/so-elastic-agent-gen-installers
 
   kibana_backport_streams_index_template
 
-  update_kafka_metadata "4.3"
+  POSTVERSION=3.2.0
-
-  set_postversion 3.2.0
 }
 
 ### 3.2.0 End ###
@@ -1105,19 +980,13 @@ update_airgap_rules() {
   rsync -a $UPDATE_DIR/agrules/securityonion-resources/* /nsm/securityonion-resources/
 }
 
-update_airgap_repos() {
+update_airgap_repo() {
   # Update the files in the repo
-  echo "Syncing new updates to /nsm/repo & /nsm/kernelrepo"
+  echo "Syncing new updates to /nsm/repo"
-  # Airgap soup copies new files into the local repo, but doesn't remove old packages. Retaining the ability to rollback package updates
+  rsync -a $AGREPO/* /nsm/repo/
-  rsync -a "$AGREPO"/ /nsm/repo/
+  echo "Creating repo"
-  rsync -a "$AGUEKREPO"/ /nsm/kernelrepo/
-
   dnf -y install yum-utils createrepo_c
-
-  echo "Running createrepo for /nsm/repo"
   createrepo /nsm/repo
-  echo "Running createrepo for /nsm/kernelrepo"
-  createrepo /nsm/kernelrepo
 }
 
 update_salt_mine() {
@@ -1144,20 +1013,8 @@ upgrade_check() {
   fi
   [[ -f /etc/sohotfix ]] && CURRENTHOTFIX=$(cat /etc/sohotfix)
   if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then
-    # A leftover post-version marker means a previous upgrade to this version
-    # advanced /etc/soversion (the highstate stamps it from the pillar) but did not
-    # finish its post-upgrade steps. Resume the upgrade instead of reporting "latest".
-    if [ -f "$POSTVERSION_FILE" ] && [ "$(cat "$POSTVERSION_FILE")" != "$NEWVERSION" ]; then
-      echo "A previous upgrade to $NEWVERSION did not complete its post-upgrade steps; resuming."
-      is_hotfix=false
-      return 0
-    fi
     echo "Checking to see if there are hotfixes needed"
     if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then
-      # Reaching here means we are at the target version and NOT resuming (the resume
-      # check above returned otherwise). Clear any stale resume marker so a completed
-      # upgrade is never mistaken for a partial one and re-run on a later invocation.
-      rm -f "$POSTVERSION_FILE"
       echo "You are already running the latest version of Security Onion."
       exit 0
     else
@@ -1236,7 +1093,7 @@ upgrade_salt() {
 
 verify_latest_update_script() {
   get_soup_script_hashes
-  if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" && "$CURRENTSOYAML" == "$GITSOYAML" ]]; then
+  if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then
     echo "This version of the soup script is up to date. Proceeding."
   else
     echo "You are not running the latest soup version. Updating soup and its components. This might take multiple runs to complete."
@@ -1245,7 +1102,7 @@ verify_latest_update_script() {
 
     # Verify that soup scripts updated as expected
     get_soup_script_hashes
-    if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" && "$CURRENTSOYAML" == "$GITSOYAML" ]]; then
+    if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then
       echo "Succesfully updated soup scripts."
     else
       echo "There was a problem updating soup scripts. Trying to rerun script update."
@@ -1269,8 +1126,7 @@ verify_es_version_compatibility() {
         ["8.18.4"]="8.18.6 8.18.8 9.0.8"
 	    ["8.18.6"]="8.18.8 9.0.8"
 	    ["8.18.8"]="9.0.8"
-        ["9.0.8"]="9.3.3 9.3.7"
+        ["9.0.8"]="9.3.3"
-        ["9.3.3"]="9.3.7"
     )
 
     # Elasticsearch MUST upgrade through these versions
@@ -1556,13 +1412,18 @@ verify_es_version_compatibility() {
 }
 
 wait_for_salt_minion_with_restart() {
-    /usr/sbin/so-salt-minion-wait
+    local minion="$1"
+    local max_wait="${2:-60}"
+    local interval="${3:-3}"
+    local logfile="$4"
+    
+    wait_for_salt_minion "$minion" "$max_wait" "$interval" "$logfile"
     local result=$?
     
     if [[ $result -ne 0 ]]; then
         echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion not ready, attempting restart..."
         systemctl_func "restart" "salt-minion"
-        /usr/sbin/so-salt-minion-wait
+        wait_for_salt_minion "$minion" "$max_wait" "$interval" "$logfile"
         result=$?
     fi
     
@@ -1848,15 +1709,6 @@ main() {
   set_minionid
   MINION_ROLE=$(lookup_role)
   echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
-  # /etc/soversion is stamped to the target version before the upgrade fully
-  # completes, so a lingering resume marker means this grid is only partially
-  # upgraded even though the line above shows the target version. Make that explicit
-  # so it is not mistaken for a finished upgrade.
-  if [ -f "$POSTVERSION_FILE" ] && [ "$(cat "$POSTVERSION_FILE")" != "$INSTALLEDVERSION" ]; then
-    echo ""
-    echo "NOTE: A previous upgrade to $INSTALLEDVERSION did not finish. This grid is"
-    echo "      partially upgraded and this soup run will resume and complete it."
-  fi
   echo ""
   check_minimum_version
 
@@ -1885,18 +1737,12 @@ main() {
   echo "Verifying Elasticsearch version compatibility across the grid before upgrading."
   verify_es_version_compatibility
 
-  # Pre-flight health checks: confirm the grid is in a good state before we change
-  # anything. These run before any modifications, so a failure exits cleanly and the
-  # operator can fix the issue and re-run soup.
-  check_cluster_health
-  check_fleet_server
-
   echo "Checking for Salt Master and Minion updates."
   upgrade_check_salt
   set -e
 
   if [[ $is_airgap -eq 0 ]]; then
-    update_airgap_repos
+    update_airgap_repo
     dnf clean all
     check_os_updates
   elif [[ $OS == 'oracle' ]]; then
@@ -1907,7 +1753,6 @@ main() {
   fi
 
   if [ "$is_hotfix" == "true" ]; then
-    SOUP_UPGRADE_STARTED=true
     echo "Applying $HOTFIXVERSION hotfix"
     # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars
     if [[ ! "$MINION_ROLE" == "import" ]]; then
@@ -1918,16 +1763,10 @@ main() {
     create_local_directories "/opt/so/saltstack/default"
     apply_hotfix
     echo "Hotfix applied"
+    update_version
     enable_highstate
     highstate
-    # Record the hotfix only after the highstate succeeds. /etc/sohotfix is written
-    # solely by soup (no salt state manages it), so deferring the write means a failed
-    # hotfix highstate leaves the old hotfix value and re-running soup re-applies it,
-    # rather than reporting "already latest". The soversion/pillar writes in
-    # update_version are no-ops here since the version is unchanged for a hotfix.
-    update_version
   else
-    SOUP_UPGRADE_STARTED=true
     echo ""
     echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
     echo ""
@@ -1983,10 +1822,6 @@ main() {
     copy_new_files
     echo ""
     create_local_directories "/opt/so/saltstack/default"
-    # Seed the resume marker before the highstate stamps /etc/soversion to the new
-    # version, so an interrupted upgrade is detectable as "not finished" on re-run.
-    # POSTVERSION still holds the pre-upgrade (or prior resume) version here.
-    [ -f "$POSTVERSION_FILE" ] || echo "$POSTVERSION" > "$POSTVERSION_FILE"
     update_version
 
     echo ""
@@ -2015,13 +1850,19 @@ main() {
 
     enable_highstate
 
+    echo "salt-call state.show_top"
+    salt-call state.show_top
+
     echo ""
-    echo "Running a highstate at $(date +"%T.%6N"). This could take several minutes."
+    echo "Running a highstate. This could take several minutes."
     set +e
-    wait_for_salt_minion_with_restart || fail "Salt minion was not running or ready."
+    wait_for_salt_minion_with_restart "$MINIONID" "60" "3" "$SOUP_LOG" || fail "Salt minion was not running or ready."
     highstate
     set -e
 
+    echo "salt-call saltutil.running"
+    salt-call saltutil.running
+
     stop_salt_master
 
     masterunlock
@@ -2030,8 +1871,8 @@ main() {
 
     check_saltmaster_status
 
-    echo "Running a highstate at $(date +"%T.%6N") to complete the Security Onion upgrade on this manager. This could take several minutes."
+    echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
-    wait_for_salt_minion_with_restart || fail "Salt minion was not running or ready."
+    wait_for_salt_minion_with_restart "$MINIONID" "60" "3" "$SOUP_LOG" || fail "Salt minion was not running or ready."
 
     # Stop long-running scripts to allow potentially updated scripts to load on the next execution.
     if pgrep salt-relay.sh > /dev/null 2>&1; then
@@ -2044,6 +1885,9 @@ main() {
     # ensure the mine is updated and populated before highstates run, following the salt-master restart
     update_salt_mine
 
+    echo "salt-call state.show_top"
+    salt-call state.show_top
+
     highstate
     check_saltmaster_status
     postupgrade_changes

salt/nginx/enabled.sls

@@ -60,7 +60,6 @@ so-nginx:
       - /opt/so/conf/navigator/layers/:/opt/socore/html/navigator/assets/so:ro
       - /opt/so/conf/navigator/config.json:/opt/socore/html/navigator/assets/config.json:ro
       - /nsm/repo:/opt/socore/html/repo:ro
-      - /nsm/kernelrepo:/opt/socore/html/kernelrepo:ro
       - /nsm/rules:/nsm/rules:ro
       {%   if NGINXMERGED.external_suricata %}
       - /opt/so/rules/nids/suri:/surirules:ro

salt/nginx/etc/nginx.conf

@@ -323,16 +323,6 @@ http {
 			autoindex_localtime on;
 		}
 
-		location /kernelrepo/ {
-			allow all;
-			sendfile on;
-			sendfile_max_chunk 1m;
-			autoindex on;
-			autoindex_exact_size off;
-			autoindex_format html;
-			autoindex_localtime on;
-		}
-
 		location /influxdb/ {
 			auth_request          /auth/sessions/whoami;
 			rewrite               /influxdb/api/(.*) /api/$1 break;
@@ -399,7 +389,7 @@ http {
 		error_page 429 = @error429;
 
 		location @error401 {
-			if ($request_uri ~* (^.*/api/.*|^/connect/.*|^/oauth2/.*|^/.*\.map$)) {
+			if ($request_uri ~* (^/api/.*|^/connect/.*|^/oauth2/.*|^/.*\.map$)) {
 				return    401;
 			}
 

salt/orch/push_batch.sls

@@ -1,7 +1,7 @@
-{% from 'salt/auto_apply.map.jinja' import AUTOAPPLY %}
+{% from 'global/map.jinja' import GLOBALMERGED %}
 {% set actions = salt['pillar.get']('actions', []) %}
-{% set BATCH = AUTOAPPLY.batch %}
+{% set BATCH = GLOBALMERGED.push.batch %}
-{% set BATCH_WAIT = AUTOAPPLY.batch_wait %}
+{% set BATCH_WAIT = GLOBALMERGED.push.batch_wait %}
 
 {% for action in actions %}
 {%   if action.get('highstate') %}

salt/postgres/enabled.sls

@@ -19,7 +19,6 @@ include:
 so-postgres:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-postgres:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: so-postgres
     - networks:
       - sobridge:
@@ -85,23 +84,6 @@ so-postgres:
       - x509: postgres_crt
       - x509: postgres_key
 
-postgres_wait_ready:
-  cmd.run:
-    - name: /usr/sbin/so-postgres-wait
-    - require:
-      - docker_container: so-postgres
-      - file: postgres_sbin
-
-# Reconcile the SOC database (role, grants, so_telegraf) every highstate so a
-# partially-initialized cluster self-heals. POSTGRES_USER is injected because
-# the container env omits it.
-postgres_bootstrap_soc_db:
-  cmd.run:
-    - name: docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh
-    - require:
-      - cmd: postgres_wait_ready
-      - file: postgresinitdb
-
 delete_so-postgres_so-status.disabled:
   file.uncomment:
     - name: /opt/so/conf/so-status/so-status.conf

salt/postgres/files/init-db.sh

@@ -8,17 +8,13 @@ if [ -z "${SO_POSTGRES_PASS:-}" ] && [ -n "${SO_POSTGRES_PASS_FILE:-}" ] && [ -r
     SO_POSTGRES_PASS="$(< "$SO_POSTGRES_PASS_FILE")"
 fi
 psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
-    -- Keep the password out of postgres.log if this DDL errors.
-    SET log_min_error_statement = panic;
-    -- Idempotent, race-safe upsert: CREATE, falling back to ALTER if the role
-    -- already exists or is created concurrently.
     DO \$\$
     BEGIN
-        BEGIN
+        IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${SO_POSTGRES_USER}') THEN
             EXECUTE format('CREATE ROLE %I WITH LOGIN PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}');
-        EXCEPTION WHEN duplicate_object OR unique_violation THEN
+        ELSE
-            EXECUTE format('ALTER ROLE %I WITH LOGIN PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}');
+            EXECUTE format('ALTER ROLE %I WITH PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}');
-        END;
+        END IF;
     END
     \$\$;
     GRANT ALL ON SCHEMA public TO "$SO_POSTGRES_USER";

salt/postgres/telegraf_users.sls

@@ -8,14 +8,23 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'telegraf/map.jinja' import TELEGRAFMERGED %}
 
-{# postgres.enabled declares the so-postgres container and postgres_wait_ready
+{# postgres_wait_ready below requires `docker_container: so-postgres`, which is
-   that the requires below reference. Salt de-duplicates the circular include. #}
+   declared in postgres.enabled. Include it here so state.apply postgres.telegraf_users
+   on its own (e.g. from orch.deploy_newnode) still has that ID in scope. Salt
+   de-duplicates the circular include. #}
 include:
   - postgres.enabled
 
 {% set TG_OUT = TELEGRAFMERGED.output | upper %}
 {% if TG_OUT in ['POSTGRES', 'BOTH'] %}
 
+postgres_wait_ready:
+  cmd.run:
+    - name: /usr/sbin/so-postgres-wait
+    - require:
+      - docker_container: so-postgres
+      - file: postgres_sbin
+
 # Ensure the shared Telegraf database exists. init-db.sh only runs on a
 # fresh data dir, so hosts upgraded onto an existing /nsm/postgres volume
 # would otherwise never get so_telegraf.

salt/postgres/tools/sbin/so-telegraf-postgres

@@ -71,8 +71,6 @@ EOSQL
       -v role_user="$ROLE_USER" \
       -v role_pass="$ROLE_PASS" \
       -U postgres -d so_telegraf <<'EOSQL'
--- Keep the password out of postgres.log if this DDL errors.
-SET log_min_error_statement = panic;
 SELECT format(
   CASE WHEN EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'role_user')
        THEN 'ALTER ROLE %I WITH LOGIN PASSWORD %L'

salt/reactor/pillar_push_map.yaml

@@ -185,17 +185,6 @@ registry:
   - state: registry
     tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone'
 
-# salt: fanout to a fleetwide highstate. The salt.auto_apply settings tune the
-# push pipeline itself (enabled, debounce/drain intervals, batch sizing) and
-# salt.schedule sets the per-minion highstate interval; they are consumed by the
-# manager's schedule, beacons, and master reactor config as well as every
-# minion's highstate schedule, so a targeted re-apply isn't meaningful. A salt
-# audit row only fires for SOC-driven salt.auto_apply / salt.schedule edits --
-# salt version bumps go through soup, not SOC, so they never reach this map.
-salt:
-  - highstate: True
-    tgt: '*'
-
 # sensoroni: universal.
 sensoroni:
   - state: sensoroni

salt/reactor/push_pillar.sls

@@ -1,7 +1,7 @@
 #!py
 
-# Reactor invoked by the postgres_pillar_beacon when SOC records settings changes in
+# Reactor invoked by the pillar_db beacon when SOC records settings changes in
-# the securityonion.audit_settings table (see salt/_beacons/postgres_pillar_beacon.py). The beacon
+# the so_soc.audit_settings table (see salt/_beacons/pillar_db.py). The beacon
 # emits one event per new row carrying setting_id and node_id.
 #
 # Two branches, keyed on node_id:
@@ -59,9 +59,9 @@ def _load_push_map():
 def _push_enabled():
     try:
         caller = Caller()
-        return bool(caller.cmd('pillar.get', 'salt:auto_apply:enabled', True))
+        return bool(caller.cmd('pillar.get', 'global:push:enabled', True))
     except Exception:
-        LOG.exception('push_pillar: pillar.get salt:auto_apply:enabled failed, assuming enabled')
+        LOG.exception('push_pillar: pillar.get global:push:enabled failed, assuming enabled')
         return True
 
 
@@ -134,7 +134,7 @@ def run():
         LOG.info('push_pillar: push disabled, skipping')
         return {}
 
-    # The postgres_pillar_beacon nests its payload under data['data']; fall back to the
+    # The pillar_db beacon nests its payload under data['data']; fall back to the
     # top level so the reactor is robust to either shape.
     event = data.get('data', data)  # noqa: F821 -- data provided by reactor
     setting_id = event.get('setting_id', '')

salt/reactor/push_strelka.sls

@@ -1,12 +1,12 @@
 #!py
 
-# Reactor invoked by the rules_beacon poll beacon (salt/_beacons/rules_beacon.py) on rule
+# Reactor invoked by the inotify beacon on rule file changes under
-# file changes under /opt/so/saltstack/local/salt/strelka/rules/compiled/.
+# /opt/so/saltstack/local/salt/strelka/rules/compiled/.
 #
 # Writes (or updates) a push intent at /opt/so/state/push_pending/rules_strelka.json
 # and returns {}. The so-push-drainer schedule picks up ready intents, dedupes
 # across pending files, and dispatches orch.push_batch. Reactors never dispatch
-# directly
+# directly -- see plan /home/mreeves/.claude/plans/goofy-marinating-hummingbird.md.
 
 import fcntl
 import json
@@ -35,9 +35,9 @@ def _sensor_compound():
 def _push_enabled():
     try:
         caller = Caller()
-        return bool(caller.cmd('pillar.get', 'salt:auto_apply:enabled', True))
+        return bool(caller.cmd('pillar.get', 'global:push:enabled', True))
     except Exception:
-        LOG.exception('push_strelka: pillar.get salt:auto_apply:enabled failed, assuming enabled')
+        LOG.exception('push_strelka: pillar.get global:push:enabled failed, assuming enabled')
         return True
 
 

salt/reactor/push_suricata.sls

@@ -1,12 +1,12 @@
 #!py
 
-# Reactor invoked by the rules_beacon poll beacon (salt/_beacons/rules_beacon.py) on rule
+# Reactor invoked by the inotify beacon on rule file changes under
-# file changes under /opt/so/saltstack/local/salt/suricata/rules/.
+# /opt/so/saltstack/local/salt/suricata/rules/.
 #
 # Writes (or updates) a push intent at /opt/so/state/push_pending/rules_suricata.json
 # and returns {}. The so-push-drainer schedule picks up ready intents, dedupes
 # across pending files, and dispatches orch.push_batch. Reactors never dispatch
-# directly
+# directly -- see plan /home/mreeves/.claude/plans/goofy-marinating-hummingbird.md.
 
 import fcntl
 import json
@@ -34,9 +34,9 @@ def _sensor_compound_plus_import():
 def _push_enabled():
     try:
         caller = Caller()
-        return bool(caller.cmd('pillar.get', 'salt:auto_apply:enabled', True))
+        return bool(caller.cmd('pillar.get', 'global:push:enabled', True))
     except Exception:
-        LOG.exception('push_suricata: pillar.get salt:auto_apply:enabled failed, assuming enabled')
+        LOG.exception('push_suricata: pillar.get global:push:enabled failed, assuming enabled')
         return True
 
 

salt/registry/enabled.sls

@@ -16,14 +16,15 @@ include:
 # Install the registry container
 so-dockerregistry:
   docker_container.running:
-    - image: ghcr.io/security-onion-solutions/registry:3.1.1
+    - image: ghcr.io/security-onion-solutions/registry:3.0.0
-    # Intentionally `always`-- registry is critical and must
-    # come back up even if it was manually stopped.
-    - restart_policy: always
     - hostname: so-registry
     - networks:
       - sobridge:
         - ipv4_address: {{ DOCKERMERGED.containers['so-dockerregistry'].ip }}
+    # Intentionally `always` (not unless-stopped) -- registry is critical infra
+    # and must come back up even if it was manually stopped. Do not homogenize
+    # to unless-stopped; see the container auto-restart section of the plan.
+    - restart_policy: always
     - port_bindings:
       {% for BINDING in DOCKERMERGED.containers['so-dockerregistry'].port_bindings %}
       - {{ BINDING }}

salt/repo/client/oracle.sls

@@ -6,10 +6,6 @@
 {% from 'repo/client/map.jinja' import REPOPATH with context %}
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 
-{% import_yaml 'salt/minion.defaults.yaml' as saltversion %}
-{% set saltversion = saltversion.salt.minion.version %}
-{% set INSTALLEDSALTVERSION = grains.saltversion %}
-
 {% set role = grains.id.split('_') | last %}
 {% set MANAGER = salt['grains.get']('master') %}
 {% if grains['os'] == 'OEL' %}
@@ -61,32 +57,6 @@ so_repo:
     - enabled: 1
     - gpgcheck: 1
 
-# Only assign the kernel repo once this node's running salt matches the version this
-# SO release ships. During a soup the grid is mid-salt-upgrade; gating here keeps the
-# UEK8 kernel repo (and the kernel update it enables) from activating until the node is
-# fully on the target salt, the same way other states defer across the upgrade window.
-{% if saltversion | string == INSTALLEDSALTVERSION | string %}
-so_kernel_repo:
-  pkgrepo.managed:
-    - name: securityonionkernel
-    - humanname: Security Onion Kernel Repo
-  {% if GLOBALS.is_manager %}
-    - baseurl: file:///nsm/kernelrepo/
-  {% else %}
-    - baseurl: https://{{ GLOBALS.repo_host }}/kernelrepo
-  {% endif %}
-    - enabled: 1
-    - gpgcheck: 1
-    # Supplementary kernel repo: tolerate it being empty/unreachable (e.g. before the
-    # manager has populated /nsm/kernelrepo) so a missing repomd.xml can't make every
-    # dnf/pkg operation on the grid fail.
-    - skip_if_unavailable: 1
-    # Only assign the kernel repo once physical NIC names are pinned by MAC, so the
-    # UEK8 kernel update can't renumber interfaces SO binds by name (see pin_nic_names
-    # in salt/common/init.sls, which drops this marker via /usr/sbin/so-nic-pin).
-    - onlyif: 'test -e /opt/so/state/nic_names_pinned'
-{% endif %}
-
 {% endif %}
   
 # TODO: Add a pillar entry for custom repos

salt/salt/auto_apply.map.jinja

@@ -1,2 +0,0 @@
-{% import_yaml 'salt/defaults.yaml' as SALT_DEFAULTS %}
-{% set AUTOAPPLY = salt['pillar.get']('salt:auto_apply', SALT_DEFAULTS.salt.auto_apply, merge=True) %}

salt/salt/defaults.yaml

@@ -1,9 +0,0 @@
-salt:
-  auto_apply:
-    enabled: true
-    debounce_seconds: 30
-    drain_interval: 15
-    batch: '25%'
-    batch_wait: 15
-  schedule:
-    highstate_interval_hours: 2

salt/salt/files/reactor_pushstate.conf

@@ -1,7 +1,11 @@
 reactor:
-  - 'salt/beacon/*/rules_beacon/suricata':
+  - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/suricata/rules':
     - salt://reactor/push_suricata.sls
-  - 'salt/beacon/*/rules_beacon/strelka':
+  - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/suricata/rules/*':
+    - salt://reactor/push_suricata.sls
+  - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/strelka/rules/compiled':
     - salt://reactor/push_strelka.sls
-  - 'salt/beacon/*/postgres_pillar_beacon/audit_settings':
+  - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/strelka/rules/compiled/*':
+    - salt://reactor/push_strelka.sls
+  - 'salt/beacon/*/pillar_db/audit_settings':
     - salt://reactor/push_pillar.sls

salt/salt/highstate_schedule.sls

@@ -1,11 +0,0 @@
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-{% from 'salt/schedule.map.jinja' import SCHEDULEMERGED %}
-
-highstate_schedule:
-  schedule.present:
-    - function: state.highstate
-    - hours: {{ SCHEDULEMERGED.highstate_interval_hours }}
-    - maxrunning: 1
-{% if not GLOBALS.is_manager %}
-    - splay: 1800
-{% endif %}

salt/salt/master.sls

@@ -10,11 +10,12 @@
 #    software that is protected by the license key."
 
 {% from 'allowed_states.map.jinja' import allowed_states %}
-{% from 'salt/auto_apply.map.jinja' import AUTOAPPLY %}
+{% from 'global/map.jinja' import GLOBALMERGED %}
 {% if sls in allowed_states %}
 
 include:
   - salt.minion
+  - salt.master.pyinotify
   - salt.master.boot_mine_update
 {%   if 'vrt' in salt['pillar.get']('features', []) %}
   - salt.cloud
@@ -64,7 +65,7 @@ engines_config:
     - name: /etc/salt/master.d/engines.conf
     - source: salt://salt/files/engines.conf
 
-{% if AUTOAPPLY.enabled %}
+{% if GLOBALMERGED.push.enabled %}
 reactor_pushstate_config:
   file.managed:
     - name: /etc/salt/master.d/reactor_pushstate.conf

salt/salt/master/pyinotify.sls

@@ -0,0 +1,20 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+pyinotify_module_package:
+  file.recurse:
+    - name: /opt/so/conf/salt/module_packages/pyinotify
+    - source: salt://salt/module_packages/pyinotify
+    - clean: True
+    - makedirs: True
+
+pyinotify_python_module_install:
+  cmd.run:
+    - name: /opt/saltstack/salt/bin/python3.10 -m pip install pyinotify --no-index --find-links=/opt/so/conf/salt/module_packages/pyinotify/ --upgrade
+    - onchanges:
+      - file: pyinotify_module_package
+    - failhard: True
+    - watch_in:
+      - service: salt_minion_service

salt/salt/minion/init.sls

@@ -131,18 +131,11 @@ salt_minion_service:
 {% endif %}
     - order: last
 
-# block until the salt-minion daemon is ready for the current instance, so follow-on jobs and the
+# block until the just-restarted salt-minion is back and can execute modules locally, so
-# next highstate iteration do not race the restart. onchanges + require on salt_minion_service
+# follow-on jobs and the next highstate iteration do not race the restart. onchanges +
-# catches every restart trigger uniformly because watch mod_watch results replace the service
+# require on salt_minion_service catches every restart trigger uniformly because watch
-# state's running entry. wait logic lives in /usr/sbin/so-salt-minion-wait (deployed by salt_sbin
+# mod_watch results replace the service state's running entry. wait logic lives in
-# from salt/tools/sbin/); its steady-state authority is the master req/publish sockets for the
+# /usr/sbin/so-salt-minion-wait (deployed by common_sbin from common/tools/sbin/).
-# current daemon pid (resolved via systemd, not the pidfile), and it corroborates a just-restarted
-# instance with the pid-tagged "Minion is ready to receive requests!" log line only within a short
-# window of startup. Because that socket signal does not require a recent restart, the wait also
-# succeeds cleanly when salt_minion_service reports a non-restart change (e.g. an enable toggle)
-# rather than false-timing-out. set_log_levels above enforces the log_level_logfile: info that the
-# ready line depends on. salt restarts this unit with --no-block, so mod_watch returns while the old
-# daemon is still up; the script waits for systemd's restart job to drain before it reads MainPID.
 wait_for_salt_minion_ready:
   cmd.run:
     - name: /usr/sbin/so-salt-minion-wait

salt/salt/push_drain_schedule.sls

@@ -1,17 +0,0 @@
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-{% from 'salt/auto_apply.map.jinja' import AUTOAPPLY %}
-
-{% if GLOBALS.is_manager and AUTOAPPLY.enabled %}
-push_drain_schedule:
-  schedule.present:
-    - function: cmd.run
-    - job_args:
-      - /usr/sbin/so-push-drainer
-    - seconds: {{ AUTOAPPLY.drain_interval }}
-    - maxrunning: 1
-    - return_job: False
-{% elif GLOBALS.is_manager %}
-push_drain_schedule:
-  schedule.absent:
-    - name: push_drain_schedule
-{% endif %}

salt/salt/schedule.map.jinja

@@ -1,2 +0,0 @@
-{% import_yaml 'salt/defaults.yaml' as SALT_DEFAULTS %}
-{% set SCHEDULEMERGED = salt['pillar.get']('salt:schedule', SALT_DEFAULTS.salt.schedule, merge=True) %}

salt/salt/soc_salt.yaml

@@ -1,39 +0,0 @@
-salt:
-  auto_apply:
-    enabled:
-      description: Master kill-switch for the active push feature. When disabled, rule and pillar changes are picked up at the next scheduled highstate instead of being pushed immediately.
-      forcedType: bool
-      helpLink: push
-      global: True
-    debounce_seconds:
-      description: Trailing-edge debounce window in seconds. A push intent must be quiet for this long before the drainer dispatches. Rapid bursts of edits within this window coalesce into one dispatch.
-      forcedType: int
-      helpLink: push
-      global: True
-      advanced: True
-    drain_interval:
-      description: How often the push drainer checks for ready intents, in seconds. Small values lower dispatch latency at the cost of more background work on the manager.
-      forcedType: int
-      helpLink: push
-      global: True
-      advanced: True
-    batch:
-      description: "Host batch size for push orchestrations. A number (e.g. '10') or a percentage (e.g. '25%'). Limits how many minions run the push state at once so large fleets don't thundering-herd."
-      helpLink: push
-      global: True
-      advanced: True
-      regex: '^([0-9]+%?)$'
-      regexFailureMessage: Enter a whole number or a whole-number percentage (e.g. 10 or 25%).
-    batch_wait:
-      description: Seconds to wait between host batches in a push orchestration. Gives the fleet time to breathe between waves.
-      forcedType: int
-      helpLink: push
-      global: True
-      advanced: True
-  schedule:
-    highstate_interval_hours:
-      description: How often every minion in the grid runs a scheduled state.highstate, in hours. Lower values keep minions closer in sync at the cost of more load; higher values reduce load but increase worst-case latency for non-pushed changes. The salt-minion health check restarts a minion if its last highstate is older than this value plus one hour.
-      forcedType: int
-      helpLink: push
-      global: True
-      advanced: True

salt/salt/tools/sbin/so-salt-minion-wait

@@ -5,203 +5,31 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-# Block until the current salt-minion daemon is ready to receive requests. Invoked from the
+# Block until the local salt-minion service is back up and can execute modules locally.
-# wait_for_salt_minion_ready state in salt/minion/init.sls after salt_minion_service fires its
+# Invoked from the wait_for_salt_minion_ready state in salt/minion/init.sls after
-# watch-driven restart, so follow-on jobs and the next highstate iteration do not race it. It is
+# salt_minion_service fires its watch-driven mod_watch (a non-blocking systemctl restart),
-# also correct on an already-running minion (no recent restart): the steady-state readiness signal
+# so follow-on jobs and the next highstate iteration do not race the in-flight restart.
-# is the live master sockets, so it does not depend on a restart having just happened.
-#
-# Salt logs "Minion is ready to receive requests!" from Minion.tune_in() only after
-# sync_connect_master() returns, which means the pub channel authenticated, the long-running req
-# channel connected, and _post_master_init() finished loading modules and compiling pillar. Two
-# signals reproduce that:
-#
-#   1. Steady state  the pid holds an ESTABLISHED req connection to a master on 4506 plus a second
-#                    (publish) connection to that same master IP on another port. The publish port is
-#                    learned from the master's auth reply and is absent from minion config, so it is
-#                    derived from the connection rather than read from config. This is the always-on
-#                    authority: it reflects whatever daemon is running now, restart or not.
-#   2. Startup only  the pid-tagged ready line in the minion log. Salt's log_fmt_logfile embeds
-#                    [%(process)d] just before the message, so this is keyed to one daemon instance.
-#                    Salt logs it exactly once per start, so it exists only to close a ~2.8s window
-#                    after the sockets come up where they are established but _post_master_init() is
-#                    still finishing. It is therefore required only within READY_LINE_WINDOW seconds
-#                    of (re)start (by pid uptime); past that the line has scrolled out of the log and
-#                    the socket gate alone decides. See instance_ready().
-#
-# The daemon pid is resolved from systemd, never from /var/run/salt-minion.pid. salt_minion() runs
-# the real minion in a multiprocessing child; that child writes the pidfile, owns the sockets and
-# logs the ready line, while systemd's MainPID is the parent. During a restart the pidfile can still
-# name the OLD child, whose own ready line is already in the log -- matching it would report ready
-# instantly. Children of the current MainPID structurally exclude the old instance.
-#
-# That is only true once systemd has actually swapped MainPID. Salt restarts this unit with
-# --no-block (salt/modules/systemd_service.py:_no_block_default returns True for salt-minion), so
-# service.restart returns as soon as the job is enqueued and the state proceeds to run this script
-# while the OLD daemon is still up -- observed at ~7s before MainPID flips. Reading MainPID in that
-# window names the outgoing instance, which is still fully connected and has its own ready line, so
-# every gate below would pass on the daemon that is about to die. Wait for systemd's job queue for
-# the unit to drain first; that is the deterministic "the swap has happened" signal.
 
 . /usr/sbin/so-common
 
-set -u
+# Initial sleep gives the systemctl restart (--no-block by default for salt-minion on
-
+# >=3006.15) time to begin tearing down the old process before we probe for readiness.
+INITIAL_SLEEP=3
 TIMEOUT=120
-MASTER_PORT=4506
+PING_TIMEOUT=5
-LOG_TAIL_LINES=10000
-# Seconds after a (re)start during which the pid-tagged ready line is still required. Past this the
-# daemon is clearly beyond the ~2.8s post-connect race and the socket gate is authoritative -- the
-# one-time ready line has scrolled out of the log tail on a long-running minion. Kept under TIMEOUT
-# so a fresh minion that connects but never logs the line still falls back to socket-only near the
-# end instead of false-timing-out.
-READY_LINE_WINDOW=90
-DEFAULT_LOG_FILE="/opt/so/log/salt/minion"
-LOG_FILE="$DEFAULT_LOG_FILE"
 
-# Decide whether the ready line can ever appear. salt-call --local sets file_client=local, so this
+sleep "$INITIAL_SLEEP"
-# reads the merged config (honoring minion.d overrides) without contacting the master. salt defaults
-# log_level_logfile to None, meaning it inherits log_level, so resolve that before deciding.
-LOG_LEVEL_LOGFILE=$(salt-call --local --out=newline_values_only config.get log_level_logfile 2>/dev/null | head -n1)
-case "${LOG_LEVEL_LOGFILE,,}" in
-  ""|none) LOG_LEVEL_LOGFILE=$(salt-call --local --out=newline_values_only config.get log_level 2>/dev/null | head -n1) ;;
-esac
 
-case "${LOG_LEVEL_LOGFILE,,}" in
+elapsed="$INITIAL_SLEEP"
-  all|garbage|trace|debug|profile|info) USE_LOG_GATE=1 ;;
-  *)                                    USE_LOG_GATE=0 ;;
-esac
-
-if [ "$USE_LOG_GATE" -eq 1 ]; then
-  LOG_FILE=$(salt-call --local --out=newline_values_only config.get log_file 2>/dev/null | head -n1)
-  [ -z "$LOG_FILE" ] && LOG_FILE="$DEFAULT_LOG_FILE"
-  [ -d "$(dirname "$LOG_FILE")" ] || USE_LOG_GATE=0
-fi
-
-if command -v ss >/dev/null 2>&1; then
-  USE_SOCKET_GATE=1
-else
-  USE_SOCKET_GATE=0
-fi
-
-if [ "$USE_LOG_GATE" -eq 0 ] && [ "$USE_SOCKET_GATE" -eq 0 ]; then
-  echo "so-salt-minion-wait: no usable readiness signal (log_level_logfile='${LOG_LEVEL_LOGFILE:-unset}', ss not found)" >&2
-  exit 1
-fi
-
-if [ "$USE_LOG_GATE" -eq 1 ] && [ "$USE_SOCKET_GATE" -eq 1 ]; then
-  echo "so-salt-minion-wait: gating on pid-tagged ready line in ${LOG_FILE} plus master sockets"
-elif [ "$USE_LOG_GATE" -eq 1 ]; then
-  echo "so-salt-minion-wait: ss not found; gating on pid-tagged ready line in ${LOG_FILE} only"
-else
-  echo "so-salt-minion-wait: INFO file logging unavailable (log_level_logfile='${LOG_LEVEL_LOGFILE:-unset}'); gating on master sockets only"
-fi
-
-# True while systemd still has a queued or running job for the unit, i.e. an in-flight --no-block
-# restart. MainPID still names the outgoing daemon until this drains. The unit name is passed as a
-# filter and grepped as well, so this stays correct if an older systemctl ignores the filter.
-restart_pending() {
-  systemctl list-jobs --no-legend salt-minion.service 2>/dev/null | grep -q 'salt-minion\.service'
-}
-
-# Emit the pid(s) of the current daemon instance. systemd's MainPID is the parent keepalive process;
-# its child runs tune_in. Fall back to MainPID when there is no child (--disable-keepalive path).
-resolve_daemon_pids() {
-  local mainpid children
-  mainpid=$(systemctl show -p MainPID --value salt-minion 2>/dev/null)
-  if [ -z "$mainpid" ] || [ "$mainpid" = "0" ]; then
-    return 1
-  fi
-  children=$(pgrep -P "$mainpid" 2>/dev/null)
-  printf '%s\n' "${children:-$mainpid}"
-}
-
-# Elapsed seconds since this pid started (Linux procps etimes). Empty/non-numeric -> failure, so the
-# caller can fall back to the strict (log-gate-enforced) behavior when uptime cannot be read.
-pid_uptime() {
-  local pid=$1 secs
-  secs=$(ps -o etimes= -p "$pid" 2>/dev/null | tr -d ' ')
-  case "$secs" in ''|*[!0-9]*) return 1 ;; esac
-  printf '%s\n' "$secs"
-}
-
-# True iff the ready line tagged with this pid is in the current or most recently rotated log.
-ready_logged() {
-  local pid=$1 f
-  for f in "$LOG_FILE" "$LOG_FILE.1"; do
-    [ -r "$f" ] || continue
-    if tail -n "$LOG_TAIL_LINES" "$f" 2>/dev/null | grep -Fq "[$pid] Minion is ready to receive requests!"; then
-      return 0
-    fi
-  done
-  return 1
-}
-
-# True iff this pid holds an ESTABLISHED req connection to a master on MASTER_PORT and a second
-# ESTABLISHED connection to that same master IP on another port. The trailing comma in "pid=N,"
-# keeps pid=123 from matching pid=1234. Grid comms are IPv4 (the unit's ExecStartPre gates on ip -4).
-socket_ready() {
-  local pid=$1 mip master_ips
-  master_ips=$(ss -tnp state established "dport = :${MASTER_PORT}" 2>/dev/null \
-    | grep -F "pid=${pid}," \
-    | grep -oE "[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+:${MASTER_PORT}" \
-    | sed "s/:${MASTER_PORT}\$//" \
-    | sort -u)
-  [ -z "$master_ips" ] && return 1
-  for mip in $master_ips; do
-    if ss -tnp state established "dst ${mip} and dport != :${MASTER_PORT}" 2>/dev/null | grep -qF "pid=${pid},"; then
-      return 0
-    fi
-  done
-  return 1
-}
-
-instance_ready() {
-  local pid=$1 uptime
-  # The log gate only closes the ~2.8s window right after the master sockets come up where they are
-  # established but _post_master_init() is still loading modules/compiling pillar. Salt logs the
-  # pid-tagged ready line exactly once at startup, so on a daemon that started long ago the line has
-  # scrolled out of the log tail and the gate could never pass -- making the wait require a recent
-  # restart. Enforce it only while the daemon is young enough that the race could still be open; past
-  # READY_LINE_WINDOW the socket gate is authoritative. If uptime can't be read, keep the strict
-  # behavior (uptime=0 -> gate enforced) so the fresh-restart path never regresses.
-  if [ "$USE_LOG_GATE" -eq 1 ]; then
-    uptime=$(pid_uptime "$pid") || uptime=0
-    if [ "$uptime" -lt "$READY_LINE_WINDOW" ] && ! ready_logged "$pid"; then
-      return 1
-    fi
-  fi
-  if [ "$USE_SOCKET_GATE" -eq 1 ] && ! socket_ready "$pid"; then
-    return 1
-  fi
-  return 0
-}
-
-elapsed=0
-pids=""
-announced_pending=0
 while [ "$elapsed" -lt "$TIMEOUT" ]; do
-  if restart_pending; then
+  if systemctl is-active --quiet salt-minion \
-    # An in-flight --no-block restart: MainPID still names the outgoing daemon. Evaluating now
+     && salt-call --local --timeout="$PING_TIMEOUT" --out=quiet test.ping >/dev/null 2>&1; then
-    # would bless the instance that is about to be torn down.
+    echo "salt-minion ready after ${elapsed}s"
-    if [ "$announced_pending" -eq 0 ]; then
-      echo "so-salt-minion-wait: systemd restart job in flight; waiting for it to drain"
-      announced_pending=1
-    fi
-  elif pids=$(resolve_daemon_pids); then
-    # shellcheck disable=SC2086
-    for pid in $pids; do
-      if instance_ready "$pid"; then
-        echo "salt-minion (pid ${pid}) ready after ${elapsed}s"
     exit 0
   fi
-    done
-  fi
   sleep 1
   elapsed=$((elapsed + 1))
 done
 
-mainpid=$(systemctl show -p MainPID --value salt-minion 2>/dev/null)
+echo "salt-minion did not become ready within ${TIMEOUT}s" >&2
-restart_pending && pending=yes || pending=no
-echo "salt-minion did not become ready within ${TIMEOUT}s (MainPID=${mainpid:-unknown}, candidate pids='${pids:-none}', restart_job_pending=${pending}, log_gate=${USE_LOG_GATE}, socket_gate=${USE_SOCKET_GATE}, log_file=${LOG_FILE})" >&2
 exit 1

salt/schedule.sls

@@ -0,0 +1,26 @@
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+{% from 'global/map.jinja' import GLOBALMERGED %}
+
+highstate_schedule:
+  schedule.present:
+    - function: state.highstate
+    - hours: {{ GLOBALMERGED.push.highstate_interval_hours }}
+    - maxrunning: 1
+{% if not GLOBALS.is_manager %}
+    - splay: 1800
+{% endif %}
+
+{% if GLOBALS.is_manager and GLOBALMERGED.push.enabled %}
+push_drain_schedule:
+  schedule.present:
+    - function: cmd.run
+    - job_args:
+      - /usr/sbin/so-push-drainer
+    - seconds: {{ GLOBALMERGED.push.drain_interval }}
+    - maxrunning: 1
+    - return_job: False
+{% elif GLOBALS.is_manager %}
+push_drain_schedule:
+  schedule.absent:
+    - name: push_drain_schedule
+{% endif %}

salt/soc/config.sls

@@ -134,30 +134,6 @@ socsigmasopipeline:
     - group: 939
     - mode: 600
 
-socsigmaplaybookpipeline:
-  file.managed:
-    - name: /opt/so/conf/soc/sigma_playbook_pipeline.yaml
-    - source: salt://soc/files/soc/sigma_playbook_pipeline.yaml
-    - user: 939
-    - group: 939
-    - mode: 600
-
-socplaybookplaceholdermap:
-  file.managed:
-    - name: /opt/so/conf/soc/playbook_placeholder_map.yaml
-    - source: salt://soc/files/soc/playbook_placeholder_map.yaml
-    - user: 939
-    - group: 939
-    - mode: 600
-
-socplaybookplaceholdermapcustom:
-  file.managed:
-    - name: /opt/so/conf/soc/playbook_placeholder_map_custom.yaml
-    - source: salt://soc/files/soc/playbook_placeholder_map_custom.yaml
-    - user: 939
-    - group: 939
-    - mode: 600
-
 socbanner:
   file.managed:
     - name: /opt/so/conf/soc/banner.md

salt/soc/defaults.map.jinja

@@ -8,7 +8,6 @@
 {% from 'docker/docker.map.jinja' import DOCKERMERGED -%}
 {% set INFLUXDB_TOKEN = salt['pillar.get']('influxdb:token') %}
 {% import_text 'influxdb/metrics_link.txt' as METRICS_LINK %}
-{% from 'telegraf/map.jinja' import TELEGRAFMERGED %}
 
 {% for module, application_url in GLOBALS.application_urls.items() %}
 {%   do SOCDEFAULTS.soc.config.server.modules[module].update({'hostUrl': application_url}) %}
@@ -25,14 +24,6 @@
 
 {% do SOCDEFAULTS.soc.config.server.modules.elastic.update({'username': GLOBALS.elasticsearch.auth.users.so_elastic_user.user, 'password': GLOBALS.elasticsearch.auth.users.so_elastic_user.pass}) %}
 
-{% if TELEGRAFMERGED.output == 'POSTGRES' %}
-{%   for tool in SOCDEFAULTS.soc.config.server.client.tools %}
-{%     if tool.name == "toolInfluxDb" %}
-{%       do SOCDEFAULTS.soc.config.server.client.tools.remove(tool) %}
-{%     endif %}
-{%   endfor %}
-
-{% else %}
 {% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %}
 {% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'token': INFLUXDB_TOKEN}) %}
 {% for tool in SOCDEFAULTS.soc.config.server.client.tools %}
@@ -40,7 +31,6 @@
 {%     do tool.update({'link': METRICS_LINK}) %}
 {%   endif %}
 {% endfor %}
-{% endif %}
 
 {% do SOCDEFAULTS.soc.config.server.modules.statickeyauth.update({'anonymousCidr': DOCKERMERGED.range, 'apiKey': pillar.sensoroni.config.sensoronikey}) %}
 

salt/soc/defaults.yaml

@@ -1500,23 +1500,15 @@ soc:
           playbookRepos:
             default:
               - repo: https://github.com/Security-Onion-Solutions/securityonion-resources-playbooks
-                rulesetName: sos-playbook-resources
                 branch: main
                 folder: securityonion-normalized
-              - repo: https://github.com/Security-Onion-Solutions/securityonion-resources-playbooks
-                rulesetName: sos-published
-                branch: published
-                folder: sigma
             airgap:
               - repo: file:///nsm/airgap-resources/playbooks/securityonion-resources-playbooks
-                rulesetName: sos-resources-ag
                 branch: main
                 folder: securityonion-normalized
         assistant:
           systemPromptAddendum: ""
           systemPromptAddendumMaxLength: 50000
-          maxSubSessionTokens: 0
-          maxDelegationDepth: 5
           adapters:
             - name: SOAI
               protocol: securityonion_ai_cloud
@@ -1528,11 +1520,6 @@ soc:
               serviceAccountJSON: ""
               serviceAccountLocation: ""
               healthTimeoutSeconds: 5
-          agentic: false
-          agentMapping:
-            Orchestrator: Claude Sonnet
-            Investigator: Claude Sonnet
-            Detection Engineer: Claude Sonnet
         onionconfig:
           saltstackDir: /opt/so/saltstack
           bypassEnabled: false
@@ -1784,13 +1771,13 @@ soc:
               enabled: true
           queries:
             - name: Default Query
-              description: Show all events grouped by module and dataset
-              query: '* | groupby event.module* event.dataset'
-              showSubtitle: true
-            - name: Observer
               description: Show all events grouped by the observer host
               query: '* | groupby observer.name'
               showSubtitle: true
+            - name: Log Type
+              description: Show all events grouped by module and dataset
+              query: '* | groupby event.module* event.dataset'
+              showSubtitle: true
             - name: SOC - Auth
               description: Users authenticated to SOC grouped by IP address and identity
               query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby http.request.headers.x-real-ip user.name'
@@ -2702,8 +2689,6 @@ soc:
           thresholdColorRatioLow: 0.5
           thresholdColorRatioMed: 0.75
           thresholdColorRatioMax: 1
-          toolBusyMaxRetries: 30
-          toolBusyRetryDelayMs: 1000
           availableModels:
             - id: sonnet
               displayName: Claude Sonnet

salt/soc/enabled.sls

@@ -46,10 +46,7 @@ so-soc:
       - /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro
       - /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro
       - /opt/so/conf/soc/sigma_so_pipeline.yaml:/opt/sensoroni/sigma_so_pipeline.yaml:ro
-      - /opt/so/conf/soc/sigma_playbook_pipeline.yaml:/opt/sensoroni/sigma_playbook_pipeline.yaml:ro
+      - /opt/so/conf/soc/sigma_final_pipeline.yaml:/opt/sensoroni/sigma_final_pipeline.yaml:rw
-      - /opt/so/conf/soc/sigma_final_pipeline.yaml:/opt/sensoroni/sigma_final_pipeline.yaml:ro
-      - /opt/so/conf/soc/playbook_placeholder_map.yaml:/opt/sensoroni/playbook_placeholder_map.yaml:ro
-      - /opt/so/conf/soc/playbook_placeholder_map_custom.yaml:/opt/sensoroni/playbook_placeholder_map_custom.yaml:ro
       - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro
       - /opt/so/conf/soc/custom_roles:/opt/sensoroni/rbac/custom_roles:ro
       - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw
@@ -103,8 +100,6 @@ so-soc:
       - file: soccustomroles
       - file: socusersroles
       - file: socclientsroles
-      - file: socplaybookplaceholdermap
-      - file: socplaybookplaceholdermapcustom
 
 delete_so-soc_so-status.disabled:
   file.uncomment:

salt/soc/files/soc/playbook_placeholder_map.yaml

@@ -1,49 +0,0 @@
-# Global Playbook placeholder map: %token% -> event field path.
-#
-# Loaded by the SOC Playbook module and used to resolve `field|expand:%placeholder%` values 
-# from an alert when converting playbook questions to OQL. 
-# Left: the %token% used in a question
-# Right: the event field its value is read from (event_data.-nested or bare; the module
-# tries both). 
-#
-# Example: with `src_ip: source.ip` (below), a question that writes
-# `source.ip|expand: '%src_ip%'` resolves %src_ip% to the alert's source.ip at convert time.
-#
-# This is the global base layer. To add or override tokens edit playbook_placeholder_map_custom.yaml.
-# those entries overlay this map and win on conflict.
-
-CommandLine: process.command_line
-CurrentDirectory: process.working_directory
-Image: process.executable
-ImageLoaded: dll.name
-ParentImage: process.parent.executable
-ParentName: process.parent.name
-ParentProcessGuid: process.parent.entity_id
-ProcessGuid: process.entity_id
-TargetFilename: file.name
-TargetObject: registry.path
-TargetUserName: user.target.name
-User: user.name
-community_id: network.community_id
-dns_resolved_ip: dns.resolved_ip
-document_id: soc_id
-dst_ip: destination.ip
-dst_port: destination.port
-event_data_source_ip: source.ip
-file_path: file.path
-file_dirs: process.file_dirs
-file_name: process.name
-file_paths: process.file_paths
-hostname: host.name
-private_ip: network.private_ip
-public_ip: network.public_ip
-related_hosts: related.hosts
-related_ip: related.ip
-src_ip: source.ip
-dns_query_name: dns.query_name
-flow_id: log.id.uid
-payload: network.data.decoded
-rule_category: rule.category
-rule_name: rule.name
-rule_uuid: rule.uuid
-src_port: source.port

salt/soc/files/soc/playbook_placeholder_map_custom.yaml

@@ -1,14 +0,0 @@
-# Custom Playbook placeholder map: %token% -> event field path.
-#
-#
-# Left: the %token% used in a playbook question.
-# Right: the event field its value is read from (event_data.-nested or bare; the module tries
-# both). Note: a token that is simply named after a flat event field resolves automatically
-# without an entry here - only add a mapping when the token name differs from the field name.
-#
-# Example:
-#
-# account_id: cloudflare.account_id
-#
-# A question that writes
-# `account_id|expand: '%account_id%'` resolves %account_id% from the alert at convert time.

salt/soc/files/soc/sigma_playbook_pipeline.yaml

@@ -1,12 +0,0 @@
-name: Security Onion - Playbook Pipeline
-priority: 97
-transformations:
-    # Route string fields to their lowercase-normalized .caseless subfield so wildcard
-    # matches are case-insensitive.
-    - id: case_insensitive_string_fields
-      type: field_name_mapping
-      mapping:
-        process.executable: process.executable.caseless
-        process.parent.executable: process.parent.executable.caseless
-        process.command_line: process.command_line.caseless
-        process.parent.command_line: process.parent.command_line.caseless

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -63,14 +63,6 @@ transformations:
       rule_conditions:
       - type: logsource
         category: antivirus
-    # OS-agnostic process_creation scoping for product-less (NIDS/host-pivot) rules.
-    - id: process_creation_os_agnostic
-      type: add_condition
-      conditions:
-        event.category: process
-      rule_conditions:
-      - type: logsource
-        category: process_creation
     # Transforms the `Hashes` field to ECS fields
     # ECS fields are used by the hash fields emitted by Elastic Defend
     # If shipped with Elastic Agent, sysmon logs will also have hashes mapped to ECS fields    
@@ -116,40 +108,6 @@ transformations:
       - type: logsource
         product: windows
         category: driver_load
-    - id: ecs_fix_process_creation
-      type: field_name_mapping
-      mapping:
-        # bare `Hashes` (the combined-string case is broken out above)
-        winlog.event_data.Hashes: process.hash.sha256
-        winlog.event_data.IntegrityLevel: process.Ext.token.integrity_level_name
-        winlog.event_data.ParentName: process.parent.name
-      rule_conditions:
-      - type: logsource
-        product: windows
-        category: process_creation
-    - id: ecs_fix_registry_set
-      type: field_name_mapping
-      mapping:
-        winlog.event_data.Details: registry.data.strings
-        # field rename only; EventType values (SetValue/CreateKey) still differ from
-        # event.action values (modification/creation)
-        winlog.event_data.EventType: event.action
-      rule_conditions:
-      - type: logsource
-        product: windows
-        category: registry_set
-    - id: ecs_fix_image_load
-      type: field_name_mapping
-      mapping:
-        file.path: dll.path
-        file.code_signature.signed: dll.code_signature.exists
-        winlog.event_data.Signature: dll.code_signature.subject_name
-        file.code_signature.status: dll.code_signature.status
-        winlog.event_data.Hashes: dll.hash.sha256
-      rule_conditions:
-      - type: logsource
-        product: windows
-        category: image_load
     - id: linux_security_add-fields
       type: add_condition
       conditions:
@@ -323,15 +281,6 @@ transformations:
       rule_conditions:
       - type: logsource
         category: file_event
-    # Scope image_load rules to Elastic Endpoint library events (event.category:library, dll.*
-    # populated).
-    - id: endpoint_image_load_add-fields
-      type: add_condition
-      conditions:
-        event.category: 'library'
-      rule_conditions:
-      - type: logsource
-        category: image_load
     # Maps network rules to all network logs
     # This targets all network logs, all services, generated from endpoints and network
     - id: network_add-fields

salt/soc/merged.map.jinja

@@ -7,11 +7,6 @@
 {% from 'soc/defaults.map.jinja' import SOCDEFAULTS with context %}
 {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
 {% from 'manager/map.jinja' import MANAGERMERGED %}
-{% from 'telegraf/map.jinja' import TELEGRAFMERGED %}
-{%- set PG_ENTRY = salt['pillar.get']('telegraf:postgres_creds:' ~ grains.id, {}) %}
-{%- set PG_USER = PG_ENTRY.get('user', '') %}
-{%- set PG_PASS = PG_ENTRY.get('pass', '') %}
-
 {% set DOCKER_EXTRA_HOSTS = ELASTICSEARCH_NODES %}
 {% do DOCKER_EXTRA_HOSTS.append({GLOBALS.influxdb_host:pillar.node_data[GLOBALS.influxdb_host].ip}) %}
 
@@ -80,20 +75,6 @@
 {%   do SOCMERGED.config.server.update({'airgapEnabled': false}) %}
 {% endif %}
 
-{# Define the postgresmetrics module if telegraf is setup to only use Postgres #}
-{% if TELEGRAFMERGED.output != 'INFLUXDB' and PG_USER and PG_PASS %}
-{%   do SOCMERGED.config.server.modules.update({
-        'postgresmetrics': {
-          'database': 'so_telegraf',
-          'host': GLOBALS.manager_ip,
-          'password': PG_PASS,
-          'port': 5432,
-          'sslMode': 'allow',
-          'user': PG_USER,
-        }
-      }) %}
-{%   do SOCMERGED.config.server.modules.pop('influxdb') %}
-{% endif %}
 
 {# Define the Detections custom ruleset that should always be present #}
 {% set CUSTOM_RULESET = {

salt/soc/soc_soc.yaml

@@ -46,15 +46,7 @@ soc:
         syntax: yaml
         file: True
         global: True
-        advanced: False
+        advanced: True
-        helpLink: security-onion-console-customization
-      playbook_placeholder_map_custom__yaml:
-        title: Playbook Placeholder Map
-        description: Custom mappings of Playbook %placeholder% tokens to event fields.
-        syntax: yaml
-        file: True
-        global: True
-        advanced: False
         helpLink: security-onion-console-customization
   config:
     licenseKey:
@@ -727,16 +719,6 @@ soc:
             description: Maximum length of the system prompt addendum. Longer prompts will be truncated.
             global: True
             advanced: True
-          maxSubSessionTokens:
-            description: Maximum number of output tokens a delegated sub-session may generate across all of its turns. When the budget is reached, the sub-agent is halted and its result is returned to the parent agent. Set to 0 to disable the limit.
-            global: True
-            advanced: True
-            forcedType: int
-          maxDelegationDepth:
-            description: Maximum delegation nesting depth for sub-agents. For example, a value of 2 lets the main agent delegate to a sub-agent that may itself delegate one level deeper. Any deeper delegation is refused and the requesting agent continues without it. Set to 0 to disable the limit.
-            global: True
-            advanced: True
-            forcedType: int
           adapters:
             description: Configuration for AI adapters used by the Onion AI assistant. Please see documentation for help on which fields are required for which protocols.
             global: True
@@ -775,32 +757,12 @@ soc:
               label: Health Timeout Seconds
               required: False
               forcedType: int
-          agentic:
-            description: Indicates if the Assistant Module should operate in agentic mode or not. If true, agents can work together to solve tasks.
-            global: True
-            forcedType: bool
-          agentMapping:
-            Orchestrator:
-              description: The initial agent in most agentic conversations. This agent will delegate requests to specialized agents.
-              global: True
-            Investigator:
-              description: This agent investigates alerts, explains events and records, and hunts through event data. It can also acknowledge alerts and escalate to cases.
-              global: True
-            Detection Engineer:
-              description: This agent manages detections and their overrides, including tuning noisy rules and authoring rule content.
-              global: True
       client:
         assistant:
           enabled:
             description: Set to true to enable the Onion AI assistant in SOC.
             global: True
             forcedType: bool
-          toolBusyMaxRetries:
-            description: How many times to retry auto approving a tool while a tool is already running.
-            global: True
-          toolBusyRetryDelayMs:
-            description: How long in milliseconds to wait between each retry when auto approving a tool.
-            global: True
           investigationPrompt: 
             description: Prompt given to Onion AI when beginning an investigation.
             global: True

salt/strelka/backend/enabled.sls

@@ -15,7 +15,6 @@ include:
 strelka_backend:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-strelka-backend:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - binds:
       - /opt/so/conf/strelka/backend/:/etc/strelka/:ro
       - /opt/so/conf/strelka/rules/compiled/:/etc/yara/:ro
@@ -48,6 +47,11 @@ strelka_backend:
       - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
     {%   endfor %}
     {% endif %}
+    # Intentionally `on-failure` (not unless-stopped) -- strelka backend shuts
+    # down cleanly during rule reloads and we do not want those clean exits to
+    # trigger an auto-restart. Do not homogenize; see the container
+    # auto-restart section of the plan.
+    - restart_policy: on-failure
     - watch:
       - file: strelkasensorcompiledrules
       - file: backend_backend_config

salt/suricata/enabled.sls

@@ -17,8 +17,8 @@ include:
 so-suricata:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-suricata:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - privileged: True
+    - restart_policy: unless-stopped
     - environment:
       - INTERFACE={{ GLOBALS.sensor.interface }}
       {% if DOCKERMERGED.containers['so-suricata'].extra_env %}
@@ -70,7 +70,6 @@ surirulereload:
     - name: /usr/sbin/so-suricata-reload-rules >> /opt/so/log/suricata/reload.log 2>&1
     - onchanges: 
         - file: surirulesync
-    - onlyif: test -f /opt/so/rules/suricata/all-rulesets.rules
     - require:
         - docker_container: so-suricata
     

salt/suricata/soc_suricata.yaml

@@ -153,12 +153,12 @@ suricata:
       cpu-affinity:
         management-cpu-set:
           cpu:
-            description: Bind management threads to a core or range of cores. This can be a single core, list of cores, or list of range of cores. set-cpu-affinity must be set to true for this to be used.
+            description: Bind management threads to a core or range of cores. This can be a sigle core, list of cores, or list of range of cores. set-cpu-affinity must be set to true for this to be used.
             forcedType: "[]string"
             helpLink: suricata
         worker-cpu-set:
           cpu:
-            description: Bind worker threads to a core or range of cores. This can be a single core, list of cores, or list of range of cores. set-cpu-affinity must be set to true for this to be used.
+            description: Bind worker threads to a core or range of cores. This can be a sigle core, list of cores, or list of range of cores. set-cpu-affinity must be set to true for this to be used.
             forcedType: "[]string"
             helpLink: suricata
     vars:

salt/suricata/tools/sbin/so-suricata-reload-rules

@@ -7,59 +7,5 @@
 
 . /usr/sbin/so-common
 
-RULES_FILE="/opt/so/rules/suricata/all-rulesets.rules"
+retry 60 3 'docker exec so-suricata /opt/suricata/bin/suricatasc -c reload-rules /var/run/suricata/suricata-command.socket' '{"message":"done","return":"OK"}' || fail "The Suricata container was not ready in time."
-SOCKET="/var/run/suricata/suricata-command.socket"
+retry 60 3 'docker exec so-suricata /opt/suricata/bin/suricatasc -c ruleset-reload-nonblocking /var/run/suricata/suricata-command.socket' '{"message":"done","return":"OK"}' || fail "The Suricata container was not ready in time."
-SURICATASC="docker exec so-suricata /opt/suricata/bin/suricatasc"
-
-# Format an epoch as a human-readable local timestamp for log messages.
-fmt_time() { date -d "@$1" '+%Y-%m-%d %H:%M:%S %Z' 2>/dev/null; }
-
-# Prefix each input line with the current timestamp.
-timestamp_lines() { while IFS= read -r line; do printf '%s %s\n' "$(date '+%Y-%m-%d %H:%M:%S %Z')" "$line"; done; }
-
-# Epoch of Suricata's last *completed* ruleset reload; non-zero return on failure.
-suricata_reload_epoch() {
-  local out ts
-  out=$($SURICATASC -c ruleset-reload-time "$SOCKET" 2>/dev/null)
-  ts=$(echo "$out" | jq -r '.message[0].last_reload // empty' 2>/dev/null)
-  [ -n "$ts" ] || return 1
-  date -d "$ts" +%s 2>/dev/null
-}
-
-# Trigger a fresh reload and confirm Suricata is running a ruleset at least as new
-# as the rules file. Returns 0 only when both hold, so retry keeps going until an
-# in-progress reload clears and our own reload completes.
-reload_and_verify() {
-  local out reload_epoch
-  out=$($SURICATASC -c reload-rules "$SOCKET")
-  echo "reload-rules: $out"
-
-  if [[ "$out" =~ "Reload already in progress" ]]; then
-    echo "A reload is already in progress; waiting for it to clear so a fresh reload can load the current ruleset."
-    return 1
-  fi
-  if [[ ! "$out" =~ '{"message":"done","return":"OK"}' ]]; then
-    echo "Suricata not ready or unexpected reload output; will retry."
-    return 1
-  fi
-
-  reload_epoch=$(suricata_reload_epoch) || { echo "Could not read ruleset-reload-time; will retry."; return 1; }
-  if [ "$reload_epoch" -ge "$target_mtime" ]; then
-    echo "Loaded ruleset is current: last reload ($(fmt_time "$reload_epoch")) is newer than rules file ($(fmt_time "$target_mtime"))."
-    return 0
-  fi
-  echo "Loaded ruleset is stale: last reload ($(fmt_time "$reload_epoch")) is older than rules file ($(fmt_time "$target_mtime")); retrying."
-  return 1
-}
-
-# Run the reload/verify, timestamping every line of output (ours and the
-# retry/fail helpers') so reload.log shows when each step ran. The pipeline is
-# synchronous, so the log is fully flushed and ordered before we exit; the
-# script's real exit code is preserved via PIPESTATUS.
-{
-  # Epoch mtime of the ruleset we need Suricata to have loaded. Captured once so
-  # a file update mid-reload does not move the goalpost.
-  target_mtime=$(stat -c %Y "$RULES_FILE") || fail "Could not stat the Suricata rules file: $RULES_FILE"
-  retry 60 3 'reload_and_verify' || fail "Suricata did not load the current ruleset in time."
-} 2>&1 | timestamp_lines
-exit "${PIPESTATUS[0]}"

salt/telegraf/etc/telegraf.conf

@@ -244,7 +244,6 @@
   username = "{{ ES_USER }}"
   password = "{{ ES_PASS }}"
   insecure_skip_verify = true
-  cluster_health = true
 {%- elif grains['role'] in ['so-searchnode']  %}
 [[inputs.elasticsearch]]
   servers = ["https://{{ NODEIP }}:9200"]