ft

.github/DISCUSSION_TEMPLATE/3-0.yml

@@ -13,7 +13,6 @@ body:
         - 3.1.0
         - 3.2.0
         - 3.3.0
-        - 3.4.0
         - Other (please provide detail below)
     validations:
       required: true

.github/workflows/pythontest.yml

@@ -6,9 +6,6 @@ on:
       - "salt/sensoroni/files/analyzers/**"
       - "salt/manager/tools/sbin/**"
       - "salt/_beacons/**"
-      - "salt/telegraf/tools/sbin_jinja/**"
-      - "salt/telegraf/defaults.yaml"
-      - "salt/telegraf/soc_telegraf.yaml"
 
 jobs:
   build:
@@ -37,25 +34,3 @@ jobs:
     - name: Test with pytest
       run: |
         PYTHONPATH=${{ matrix.python-code-path }} pytest ${{ matrix.python-code-path }} --cov=${{ matrix.python-code-path }} --doctest-modules --cov-report=term --cov-fail-under=100 --cov-config=pytest.ini
-
-  telegraf-collector:
-    # so-container-stats is a jinja template rather than an importable module, so it gets its
-    # own job: the test renders it the way salt does, then drives it with a faked docker engine
-    # and cgroup tree. No container runtime is needed.
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python
-      uses: actions/setup-python@v3
-      with:
-        python-version: "3.14"
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install flake8 pytest jinja2 pyyaml
-    - name: Lint with flake8
-      run: |
-        flake8 salt/telegraf/tools/sbin_jinja/so-container-stats_test.py --config=pytest.ini
-    - name: Test with pytest
-      run: |
-        pytest salt/telegraf/tools/sbin_jinja/so-container-stats_test.py -v

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 3.3.0-20260911 ISO image released on 2026/09/11
+### 3.2.0-20260729 ISO image released on 2026/07/29
 
 
 ### Download and Verify
 
-3.3.0-20260911 ISO image:  
+3.2.0-20260729 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-3.3.0-20260911.iso
+https://download.securityonion.net/file/securityonion/securityonion-3.2.0-20260729.iso
  
-MD5: 12B18433D3A2198A185892FF79CF638F  
+MD5: B1E10F46DF872B655C29325DF965A4DB  
-SHA1: 2B3C2E1FA7A78ED1F956E7EDCC12E32593C14EEE  
+SHA1: 0F3C7ED80F6D326B7A993C2F899B986320C01BF9  
-SHA256: 0938C73B76CE30EC9E4394D312C79EA7CAC721B6818541697279A6221F7D870D  
+SHA256: 7465163C1D1ADFCDC3935530EAFB312E987C016941ADC11841B214553314D1FF  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.3.0-20260911.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.2.0-20260729.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.3.0-20260911.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.2.0-20260729.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-3.3.0-20260911.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-3.2.0-20260729.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-3.3.0-20260911.iso.sig securityonion-3.3.0-20260911.iso
+gpg --verify securityonion-3.2.0-20260729.iso.sig securityonion-3.2.0-20260729.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Fri 11 Sep 2026 11:23:56 AM EDT using RSA key ID FE507013
+gpg: Signature made Tue 28 Jul 2026 06:17:34 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

VERSION

@@ -1 +1 @@
-3.4.0
+3.0.0-foxtrot

salt/_beacons/local_files_beacon_test.py

@@ -1,221 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-import hashlib
-import os
-import shutil
-import tempfile
-import unittest
-from unittest.mock import patch
-
-import local_files_beacon
-
-
-class TestRulesBeacon(unittest.TestCase):
-
-    def setUp(self):
-        # Isolate all on-disk state (watermarks and the dirs we fingerprint) in a
-        # throwaway tree, and point WATERMARK_DIR at it so the real read/write
-        # helpers run against actual files.
-        self.tmpdir = tempfile.mkdtemp()
-        self.state = os.path.join(self.tmpdir, 'state')
-        patcher = patch.object(local_files_beacon, 'WATERMARK_DIR', self.state)
-        patcher.start()
-        self.addCleanup(patcher.stop)
-
-    def tearDown(self):
-        shutil.rmtree(self.tmpdir, ignore_errors=True)
-
-    def _make_dir(self, name, files=None):
-        path = os.path.join(self.tmpdir, name)
-        os.makedirs(path, exist_ok=True)
-        for fname, content in (files or {}).items():
-            with open(os.path.join(path, fname), 'w') as f:
-                f.write(content)
-        return path
-
-    # -- trivial contract -------------------------------------------------
-
-    def test_virtual_returns_true(self):
-        self.assertTrue(local_files_beacon.__virtual__())
-
-    def test_validate_returns_valid(self):
-        self.assertEqual(local_files_beacon.validate({}), (True, 'valid'))
-
-    # -- _paths_from_config -----------------------------------------------
-
-    def test_paths_from_config_list_of_dicts(self):
-        config = [{'interval': 10}, {'paths': {'/a': 'suricata', '/b': 'strelka'}}]
-        self.assertEqual(
-            local_files_beacon._paths_from_config(config),
-            {'/a': 'suricata', '/b': 'strelka'},
-        )
-
-    def test_paths_from_config_plain_dict(self):
-        self.assertEqual(
-            local_files_beacon._paths_from_config({'paths': {'/a': 'suricata'}}),
-            {'/a': 'suricata'},
-        )
-
-    def test_paths_from_config_skips_non_dict_items(self):
-        self.assertEqual(local_files_beacon._paths_from_config(['bogus', 42]), {})
-
-    def test_paths_from_config_paths_not_a_dict(self):
-        self.assertEqual(local_files_beacon._paths_from_config({'paths': 'nope'}), {})
-
-    def test_paths_from_config_unexpected_type(self):
-        self.assertEqual(local_files_beacon._paths_from_config('nonsense'), {})
-
-    # -- _excluded --------------------------------------------------------
-
-    def test_excluded_matches_temp_and_editor_files(self):
-        for pathname in ('/rules/foo.swp', '/rules/foo~', '/rules/4913', '/rules/.#foo'):
-            self.assertTrue(local_files_beacon._excluded(pathname), pathname)
-
-    def test_excluded_allows_real_rule_files(self):
-        self.assertFalse(local_files_beacon._excluded('/rules/suricata.rules'))
-
-    # -- _fingerprint -----------------------------------------------------
-
-    def test_fingerprint_missing_dir_is_empty_tree_digest(self):
-        missing = os.path.join(self.tmpdir, 'does-not-exist')
-        self.assertEqual(local_files_beacon._fingerprint(missing), hashlib.sha1().hexdigest())
-
-    def test_fingerprint_changes_when_content_changes(self):
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        before = local_files_beacon._fingerprint(d)
-        with open(os.path.join(d, 'a.rules'), 'w') as f:
-            f.write('alert tcp any any -> any any')  # different size
-        self.assertNotEqual(local_files_beacon._fingerprint(d), before)
-
-    def test_fingerprint_ignores_excluded_files(self):
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        before = local_files_beacon._fingerprint(d)
-        with open(os.path.join(d, 'a.rules.swp'), 'w') as f:
-            f.write('editor swap')
-        self.assertEqual(local_files_beacon._fingerprint(d), before)
-
-    def test_fingerprint_skips_unstatable_entries(self):
-        # A dangling symlink appears in os.walk's file list but os.stat raises
-        # OSError, exercising the except-continue path.
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        good = local_files_beacon._fingerprint(d)
-        os.symlink(os.path.join(d, 'missing-target'), os.path.join(d, 'broken.link'))
-        self.assertEqual(local_files_beacon._fingerprint(d), good)
-
-    def test_fingerprint_prunes_git_metadata(self):
-        # zkg packages are git clones, so the watched tree carries .git.
-        d = self._make_dir('zkg', {'pkg.zeek': 'print 1;'})
-        before = local_files_beacon._fingerprint(d)
-        git_dir = os.path.join(d, 'pkg', '.git', 'refs', 'heads')
-        os.makedirs(git_dir)
-        with open(os.path.join(git_dir, 'main'), 'w') as f:
-            f.write('0' * 40)
-        self.assertEqual(local_files_beacon._fingerprint(d), before)
-
-    def test_fingerprint_still_sees_worktree_next_to_git(self):
-        d = self._make_dir('zkg', {'pkg.zeek': 'print 1;'})
-        os.makedirs(os.path.join(d, 'pkg', '.git'))
-        before = local_files_beacon._fingerprint(d)
-        with open(os.path.join(d, 'pkg', 'scripts.zeek'), 'w') as f:
-            f.write('print 2;')
-        self.assertNotEqual(local_files_beacon._fingerprint(d), before)
-
-    # -- _read_watermark / _write_watermark -------------------------------
-
-    def test_watermark_round_trip(self):
-        local_files_beacon._write_watermark('suricata', '/rules/suricata', 'deadbeef')
-        self.assertEqual(
-            local_files_beacon._read_watermark('suricata', '/rules/suricata'), 'deadbeef')
-
-    def test_read_watermark_missing_returns_none(self):
-        self.assertIsNone(local_files_beacon._read_watermark('suricata', '/rules/suricata'))
-
-    def test_read_watermark_empty_file_returns_none(self):
-        os.makedirs(self.state, exist_ok=True)
-        with open(local_files_beacon._watermark_file('suricata', '/rules/suricata'), 'w') as f:
-            f.write('')
-        self.assertIsNone(local_files_beacon._read_watermark('suricata', '/rules/suricata'))
-
-    def test_write_watermark_swallows_oserror(self):
-        with patch.object(local_files_beacon.os, 'makedirs', side_effect=OSError):
-            local_files_beacon._write_watermark('suricata', '/rules/suricata', 'deadbeef')
-        self.assertIsNone(local_files_beacon._read_watermark('suricata', '/rules/suricata'))
-
-    def test_watermark_file_differs_per_directory_within_one_tag(self):
-        # zeek/policy and zeek/zkg share the tag 'zeek'.
-        self.assertNotEqual(
-            local_files_beacon._watermark_file('zeek', '/local/zeek/policy'),
-            local_files_beacon._watermark_file('zeek', '/local/zeek/zkg'),
-        )
-
-    def test_watermarks_are_independent_within_one_tag(self):
-        local_files_beacon._write_watermark('zeek', '/local/zeek/policy', 'policyhash')
-        local_files_beacon._write_watermark('zeek', '/local/zeek/zkg', 'zkghash')
-        self.assertEqual(
-            local_files_beacon._read_watermark('zeek', '/local/zeek/policy'), 'policyhash')
-        self.assertEqual(
-            local_files_beacon._read_watermark('zeek', '/local/zeek/zkg'), 'zkghash')
-
-    # -- beacon -----------------------------------------------------------
-
-    def _config(self, mapping):
-        return [{'paths': mapping}]
-
-    def test_beacon_seeds_first_run_and_emits_nothing(self):
-        with patch.object(local_files_beacon, '_fingerprint', return_value='hash1'), \
-             patch.object(local_files_beacon, '_read_watermark', return_value=None), \
-             patch.object(local_files_beacon, '_write_watermark') as mock_write:
-            result = local_files_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
-        self.assertEqual(result, [])
-        mock_write.assert_called_once_with('suricata', '/rules/suricata', 'hash1')
-
-    def test_beacon_emits_on_change(self):
-        with patch.object(local_files_beacon, '_fingerprint', return_value='newhash'), \
-             patch.object(local_files_beacon, '_read_watermark', return_value='oldhash'), \
-             patch.object(local_files_beacon, '_write_watermark') as mock_write:
-            result = local_files_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
-        self.assertEqual(result, [{'tag': 'suricata', 'path': '/rules/suricata'}])
-        mock_write.assert_called_once_with('suricata', '/rules/suricata', 'newhash')
-
-    def test_beacon_no_change_emits_nothing(self):
-        with patch.object(local_files_beacon, '_fingerprint', return_value='samehash'), \
-             patch.object(local_files_beacon, '_read_watermark', return_value='samehash'), \
-             patch.object(local_files_beacon, '_write_watermark') as mock_write:
-            result = local_files_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
-        self.assertEqual(result, [])
-        mock_write.assert_not_called()
-
-    def test_beacon_end_to_end_with_real_files(self):
-        # Exercise the full stack (real fingerprint + real watermark files) across
-        # two poll passes: first seeds silently, second fires after a write.
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        config = self._config({d: 'suricata'})
-
-        self.assertEqual(local_files_beacon.beacon(config), [])  # seed pass
-        self.assertEqual(local_files_beacon.beacon(config), [])  # unchanged pass
-
-        with open(os.path.join(d, 'b.rules'), 'w') as f:
-            f.write('alert tcp any any -> any any')
-        self.assertEqual(local_files_beacon.beacon(config), [{'tag': 'suricata', 'path': d}])
-
-    def test_beacon_two_dirs_one_tag_do_not_flap(self):
-        # Tag-keyed watermarks would clobber each other and emit on every pass.
-        policy = self._make_dir('zeek/policy', {'intel.dat': '#fields\tindicator'})
-        zkg = self._make_dir('zeek/zkg', {'README': 'place packages here'})
-        config = self._config({policy: 'zeek', zkg: 'zeek'})
-
-        self.assertEqual(local_files_beacon.beacon(config), [])  # seed pass
-        self.assertEqual(local_files_beacon.beacon(config), [])  # idle
-        self.assertEqual(local_files_beacon.beacon(config), [])  # still idle
-
-        with open(os.path.join(policy, 'intel.dat'), 'a') as f:
-            f.write('\nevil.com\tIntel::DOMAIN\tsource\n')
-        self.assertEqual(local_files_beacon.beacon(config), [{'tag': 'zeek', 'path': policy}])
-        self.assertEqual(local_files_beacon.beacon(config), [])  # quiet again
-
-
-if __name__ == '__main__':
-    unittest.main()

salt/_beacons/rules_beacon.py

@@ -3,14 +3,13 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-# Custom salt beacon that watches hand-edited directories under
+# Custom salt beacon that watches the suricata/strelka rule directories for changes
-# /opt/so/saltstack/local/salt/ for changes and emits a beacon event per changed
+# and emits a beacon event per changed directory. This replaces the stock salt
-# directory. This replaces the stock salt `inotify` beacon, which leaks a kernel
+# `inotify` beacon, which leaks a kernel inotify instance every time the minion
-# inotify instance every time the minion rebuilds the beacon loader's __context__
+# rebuilds the beacon loader's __context__ (orphaning the old pyinotify.Notifier
-# (orphaning the old pyinotify.Notifier without closing it) until
+# without closing it) until fs.inotify.max_user_instances is exhausted and the
-# fs.inotify.max_user_instances is exhausted and the beacon dies with EMFILE.
+# beacon dies with EMFILE. Polling holds zero inotify instances, so the leak is
-# Polling holds zero inotify instances, so the leak is impossible, and it keeps
+# impossible, and it keeps firing during state runs (no blackout).
-# firing during state runs (no blackout).
 #
 # Detection is poll-based with a per-directory fingerprint persisted to
 # WATERMARK_DIR: each pass walks the directory and hashes every file's
@@ -20,10 +19,9 @@
 # up on the next one).
 #
 # Each emitted event carries the watched directory path under the configured tag
-# (e.g. salt/beacon/<minion>/local_files_beacon/zeek); the push_files reactor
+# (e.g. salt/beacon/<minion>/rules_beacon/suricata); the push_suricata / push_strelka
-# looks the tag up in salt/reactor/pillar_push_map.yaml and writes a push intent,
+# reactors write a push intent, after which the existing so-push-drainer /
-# after which the existing so-push-drainer / orch.push_batch pipeline takes over
+# orch.push_batch pipeline takes over unchanged.
-# unchanged.
 
 import hashlib
 import logging
@@ -79,9 +77,7 @@ def _fingerprint(directory):
     h = hashlib.sha1()
     if os.path.isdir(directory):
         entries = []
-        for root, dirs, files in os.walk(directory):
+        for root, _dirs, files in os.walk(directory):
-            # zkg packages are git clones; .git churn would fire a state apply on its own.
-            dirs[:] = [d for d in dirs if d != '.git']
             for name in files:
                 full = os.path.join(root, name)
                 if _excluded(full):
@@ -98,22 +94,20 @@ def _fingerprint(directory):
     return h.hexdigest()
 
 
-def _watermark_file(tag, directory):
+def _watermark_file(tag):
-    # Keyed by directory: zeek/policy and zeek/zkg share the tag `zeek`.
+    return os.path.join(WATERMARK_DIR, 'rules_beacon_%s.hash' % tag)
-    scope = hashlib.sha1(directory.encode('utf-8', 'surrogateescape')).hexdigest()[:12]
-    return os.path.join(WATERMARK_DIR, 'local_files_beacon_%s_%s.hash' % (tag, scope))
 
 
-def _read_watermark(tag, directory):
+def _read_watermark(tag):
     try:
-        with open(_watermark_file(tag, directory), 'r') as f:
+        with open(_watermark_file(tag), 'r') as f:
             return (f.read() or '').strip() or None
     except IOError:
         return None
 
 
-def _write_watermark(tag, directory, digest):
+def _write_watermark(tag, digest):
-    path = _watermark_file(tag, directory)
+    path = _watermark_file(tag)
     try:
         os.makedirs(WATERMARK_DIR, exist_ok=True)
         tmp = path + '.tmp'
@@ -121,7 +115,7 @@ def _write_watermark(tag, directory, digest):
             f.write(digest)
         os.rename(tmp, path)
     except OSError:
-        log.exception('local_files_beacon: failed to persist watermark to %s', path)
+        log.exception('rules_beacon: failed to persist watermark to %s', path)
 
 
 def beacon(config):
@@ -129,17 +123,17 @@ def beacon(config):
 
     for directory, tag in _paths_from_config(config).items():
         digest = _fingerprint(directory)
-        previous = _read_watermark(tag, directory)
+        previous = _read_watermark(tag)
 
         # First run / missing watermark: seed the digest and emit nothing so a
         # fresh host does not fire a spurious fleetwide push.
         if previous is None:
-            _write_watermark(tag, directory, digest)
+            _write_watermark(tag, digest)
             continue
 
         if digest != previous:
-            _write_watermark(tag, directory, digest)
+            _write_watermark(tag, digest)
             retval.append({'tag': tag, 'path': directory})
-            log.info('local_files_beacon: change detected in %s, emitting %s', directory, tag)
+            log.info('rules_beacon: change detected in %s, emitting %s', directory, tag)
 
     return retval

salt/_beacons/rules_beacon_test.py

@@ -0,0 +1,172 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+import hashlib
+import os
+import shutil
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import rules_beacon
+
+
+class TestRulesBeacon(unittest.TestCase):
+
+    def setUp(self):
+        # Isolate all on-disk state (watermarks and the dirs we fingerprint) in a
+        # throwaway tree, and point WATERMARK_DIR at it so the real read/write
+        # helpers run against actual files.
+        self.tmpdir = tempfile.mkdtemp()
+        self.state = os.path.join(self.tmpdir, 'state')
+        patcher = patch.object(rules_beacon, 'WATERMARK_DIR', self.state)
+        patcher.start()
+        self.addCleanup(patcher.stop)
+
+    def tearDown(self):
+        shutil.rmtree(self.tmpdir, ignore_errors=True)
+
+    def _make_dir(self, name, files=None):
+        path = os.path.join(self.tmpdir, name)
+        os.makedirs(path, exist_ok=True)
+        for fname, content in (files or {}).items():
+            with open(os.path.join(path, fname), 'w') as f:
+                f.write(content)
+        return path
+
+    # -- trivial contract -------------------------------------------------
+
+    def test_virtual_returns_true(self):
+        self.assertTrue(rules_beacon.__virtual__())
+
+    def test_validate_returns_valid(self):
+        self.assertEqual(rules_beacon.validate({}), (True, 'valid'))
+
+    # -- _paths_from_config -----------------------------------------------
+
+    def test_paths_from_config_list_of_dicts(self):
+        config = [{'interval': 10}, {'paths': {'/a': 'suricata', '/b': 'strelka'}}]
+        self.assertEqual(
+            rules_beacon._paths_from_config(config),
+            {'/a': 'suricata', '/b': 'strelka'},
+        )
+
+    def test_paths_from_config_plain_dict(self):
+        self.assertEqual(
+            rules_beacon._paths_from_config({'paths': {'/a': 'suricata'}}),
+            {'/a': 'suricata'},
+        )
+
+    def test_paths_from_config_skips_non_dict_items(self):
+        self.assertEqual(rules_beacon._paths_from_config(['bogus', 42]), {})
+
+    def test_paths_from_config_paths_not_a_dict(self):
+        self.assertEqual(rules_beacon._paths_from_config({'paths': 'nope'}), {})
+
+    def test_paths_from_config_unexpected_type(self):
+        self.assertEqual(rules_beacon._paths_from_config('nonsense'), {})
+
+    # -- _excluded --------------------------------------------------------
+
+    def test_excluded_matches_temp_and_editor_files(self):
+        for pathname in ('/rules/foo.swp', '/rules/foo~', '/rules/4913', '/rules/.#foo'):
+            self.assertTrue(rules_beacon._excluded(pathname), pathname)
+
+    def test_excluded_allows_real_rule_files(self):
+        self.assertFalse(rules_beacon._excluded('/rules/suricata.rules'))
+
+    # -- _fingerprint -----------------------------------------------------
+
+    def test_fingerprint_missing_dir_is_empty_tree_digest(self):
+        missing = os.path.join(self.tmpdir, 'does-not-exist')
+        self.assertEqual(rules_beacon._fingerprint(missing), hashlib.sha1().hexdigest())
+
+    def test_fingerprint_changes_when_content_changes(self):
+        d = self._make_dir('rules', {'a.rules': 'alert'})
+        before = rules_beacon._fingerprint(d)
+        with open(os.path.join(d, 'a.rules'), 'w') as f:
+            f.write('alert tcp any any -> any any')  # different size
+        self.assertNotEqual(rules_beacon._fingerprint(d), before)
+
+    def test_fingerprint_ignores_excluded_files(self):
+        d = self._make_dir('rules', {'a.rules': 'alert'})
+        before = rules_beacon._fingerprint(d)
+        with open(os.path.join(d, 'a.rules.swp'), 'w') as f:
+            f.write('editor swap')
+        self.assertEqual(rules_beacon._fingerprint(d), before)
+
+    def test_fingerprint_skips_unstatable_entries(self):
+        # A dangling symlink appears in os.walk's file list but os.stat raises
+        # OSError, exercising the except-continue path.
+        d = self._make_dir('rules', {'a.rules': 'alert'})
+        good = rules_beacon._fingerprint(d)
+        os.symlink(os.path.join(d, 'missing-target'), os.path.join(d, 'broken.link'))
+        self.assertEqual(rules_beacon._fingerprint(d), good)
+
+    # -- _read_watermark / _write_watermark -------------------------------
+
+    def test_watermark_round_trip(self):
+        rules_beacon._write_watermark('suricata', 'deadbeef')
+        self.assertEqual(rules_beacon._read_watermark('suricata'), 'deadbeef')
+
+    def test_read_watermark_missing_returns_none(self):
+        self.assertIsNone(rules_beacon._read_watermark('suricata'))
+
+    def test_read_watermark_empty_file_returns_none(self):
+        os.makedirs(self.state, exist_ok=True)
+        with open(rules_beacon._watermark_file('suricata'), 'w') as f:
+            f.write('')
+        self.assertIsNone(rules_beacon._read_watermark('suricata'))
+
+    def test_write_watermark_swallows_oserror(self):
+        with patch.object(rules_beacon.os, 'makedirs', side_effect=OSError):
+            rules_beacon._write_watermark('suricata', 'deadbeef')
+        self.assertIsNone(rules_beacon._read_watermark('suricata'))
+
+    # -- beacon -----------------------------------------------------------
+
+    def _config(self, mapping):
+        return [{'paths': mapping}]
+
+    def test_beacon_seeds_first_run_and_emits_nothing(self):
+        with patch.object(rules_beacon, '_fingerprint', return_value='hash1'), \
+             patch.object(rules_beacon, '_read_watermark', return_value=None), \
+             patch.object(rules_beacon, '_write_watermark') as mock_write:
+            result = rules_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
+        self.assertEqual(result, [])
+        mock_write.assert_called_once_with('suricata', 'hash1')
+
+    def test_beacon_emits_on_change(self):
+        with patch.object(rules_beacon, '_fingerprint', return_value='newhash'), \
+             patch.object(rules_beacon, '_read_watermark', return_value='oldhash'), \
+             patch.object(rules_beacon, '_write_watermark') as mock_write:
+            result = rules_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
+        self.assertEqual(result, [{'tag': 'suricata', 'path': '/rules/suricata'}])
+        mock_write.assert_called_once_with('suricata', 'newhash')
+
+    def test_beacon_no_change_emits_nothing(self):
+        with patch.object(rules_beacon, '_fingerprint', return_value='samehash'), \
+             patch.object(rules_beacon, '_read_watermark', return_value='samehash'), \
+             patch.object(rules_beacon, '_write_watermark') as mock_write:
+            result = rules_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
+        self.assertEqual(result, [])
+        mock_write.assert_not_called()
+
+    def test_beacon_end_to_end_with_real_files(self):
+        # Exercise the full stack (real fingerprint + real watermark files) across
+        # two poll passes: first seeds silently, second fires after a write.
+        d = self._make_dir('rules', {'a.rules': 'alert'})
+        config = self._config({d: 'suricata'})
+
+        self.assertEqual(rules_beacon.beacon(config), [])  # seed pass
+        self.assertEqual(rules_beacon.beacon(config), [])  # unchanged pass
+
+        with open(os.path.join(d, 'b.rules'), 'w') as f:
+            f.write('alert tcp any any -> any any')
+        self.assertEqual(rules_beacon.beacon(config), [{'tag': 'suricata', 'path': d}])
+
+
+if __name__ == '__main__':
+    unittest.main()

salt/common/init.sls

@@ -117,25 +117,14 @@ elastic_curl_config:
 {% endif %}
 
 
-# A non-root owner here can chmod the directory and replace any script in it, including
-# the root-owned ones. 555 is the mode the filesystem RPM ships; root ignores it anyway.
-usr_sbin_perms:
-  file.directory:
-    - name: /usr/sbin
-    - user: root
-    - group: root
-    - mode: 555
-
 common_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://common/tools/sbin
-    - user: root
+    - user: 939
-    - group: root
+    - group: 939
     - file_mode: 755
     - show_changes: False
-    - require:
-      - file: usr_sbin_perms
 {% if GLOBALS.role == 'so-heavynode' %}
     - exclude_pat:
       - so-pcap-import
@@ -152,26 +141,12 @@ pin_nic_names:
       - file: common_sbin
       - file: statedir
 
-# Once a node is actually running UEK8, the stock EL9 (RHCK) kernel packages are dead weight.
-# They can't be removed any earlier -- dnf protects the running kernel -- so the cleanup waits
-# for the reboot, which makes the highstate the natural place to catch it: fresh installs
-# reboot at the end of setup, and upgraded nodes reboot whenever the admin schedules it.
-# so-kernel-upgrade --cleanup checks rpm before touching dnf, so this costs an rpm query on
-# every highstate after the first pass. The package list lives in the script only, so there
-# is nothing here to drift out of sync with it.
-remove_stock_kernel:
-  cmd.run:
-    - name: /usr/sbin/so-kernel-upgrade --cleanup
-    - onlyif: 'uname -r | grep -qE "^6\.[0-9]+.*uek"'
-    - require:
-      - file: common_sbin
-
 common_sbin_jinja:
   file.recurse:
     - name: /usr/sbin
     - source: salt://common/tools/sbin_jinja
-    - user: root
+    - user: 939
-    - group: root
+    - group: 939 
     - file_mode: 755
     - template: jinja
     - show_changes: False
@@ -184,8 +159,6 @@ so-status_script:
   file.managed:
     - name: /usr/sbin/so-status
     - source: salt://common/tools/sbin/so-status
-    - user: root
-    - group: root
     - mode: 755
 
 {% if GLOBALS.is_sensor %}
@@ -217,11 +190,9 @@ sostatus_log:
     - replace: False
 
 # Install sostatus check cron. This is used to populate Grid.
-# telegraf reads status.log on the same minute boundary this runs, so write aside and rename
-# rather than truncating the file it is reading
 so-status_check_cron:
   cron.present:
-    - name: '/usr/sbin/so-status -j > /opt/so/log/sostatus/status.log.tmp 2>&1; mv -f /opt/so/log/sostatus/status.log.tmp /opt/so/log/sostatus/status.log'
+    - name: '/usr/sbin/so-status -j > /opt/so/log/sostatus/status.log 2>&1'
     - identifier: so-status_check_cron
     - user: root
     - minute: '*/1'

salt/common/soup_scripts.sls

@@ -18,61 +18,47 @@ copy_so-common_common_tools_sbin:
     - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-common
     - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-common
     - force: True
-    - user: root
+    - preserve: True
-    - group: root
-    - mode: 755
 
 copy_so-image-common_common_tools_sbin:
   file.copy:
     - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-image-common
     - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-image-common
     - force: True
-    - user: root
+    - preserve: True
-    - group: root
-    - mode: 755
 
 copy_soup_manager_tools_sbin:
   file.copy:
     - name: /opt/so/saltstack/default/salt/manager/tools/sbin/soup
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/soup
     - force: True
-    - user: root
+    - preserve: True
-    - group: root
-    - mode: 755
 
 copy_so-firewall_manager_tools_sbin:
   file.copy:
     - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-firewall
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall
     - force: True
-    - user: root
+    - preserve: True
-    - group: root
-    - mode: 755
 
 copy_so-yaml_manager_tools_sbin:
   file.copy:
     - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-yaml.py
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-yaml.py
     - force: True
-    - user: root
+    - preserve: True
-    - group: root
-    - mode: 755
 
 copy_so-repo-sync_manager_tools_sbin:
   file.copy:
     - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-repo-sync
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
-    - user: root
+    - preserve: True
-    - group: root
-    - mode: 755
 
 copy_bootstrap-salt_manager_tools_sbin:
   file.copy:
     - name: /opt/so/saltstack/default/salt/salt/scripts/bootstrap-salt.sh
     - source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh
-    - user: root
+    - preserve: True
-    - group: root
-    - mode: 644
 
 # This section is used to put the new script in place so that it can be called during soup.
 # It is faster than calling the states that normally manage them to put them in place.
@@ -81,60 +67,46 @@ copy_so-common_sbin:
     - name: /usr/sbin/so-common
     - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-common
     - force: True
-    - user: root
+    - preserve: True
-    - group: root
-    - mode: 755
 
 copy_so-image-common_sbin:
   file.copy:
     - name: /usr/sbin/so-image-common
     - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-image-common
     - force: True
-    - user: root
+    - preserve: True
-    - group: root
-    - mode: 755
 
 copy_soup_sbin:
   file.copy:
     - name: /usr/sbin/soup
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/soup
     - force: True
-    - user: root
+    - preserve: True
-    - group: root
-    - mode: 755
 
 copy_so-firewall_sbin:
   file.copy:
     - name: /usr/sbin/so-firewall
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall
     - force: True
-    - user: root
+    - preserve: True
-    - group: root
-    - mode: 755
 
 copy_so-yaml_sbin:
   file.copy:
     - name: /usr/sbin/so-yaml.py
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-yaml.py
     - force: True
-    - user: root
+    - preserve: True
-    - group: root
-    - mode: 755
 
 copy_so-repo-sync_sbin:
   file.copy:
     - name: /usr/sbin/so-repo-sync
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
     - force: True
-    - user: root
+    - preserve: True
-    - group: root
-    - mode: 755
 
 copy_bootstrap-salt_sbin:
   file.copy:
     - name: /usr/sbin/bootstrap-salt.sh
     - source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh
     - force: True
-    - user: root
+    - preserve: True
-    - group: root
-    - mode: 755

salt/common/tools/sbin/so-common

@@ -240,8 +240,7 @@ copy_new_files() {
   cd $UPDATE_DIR
   rsync -a salt $DEFAULT_SALT_DIR/ --delete "${EXCLUDE_ARGS[@]}"
   rsync -a pillar $DEFAULT_SALT_DIR/ --delete "${EXCLUDE_ARGS[@]}"
-  # Root-executed code; SOC only needs to read it. Local dirs stay socore-owned.
+  chown -R socore:socore $DEFAULT_SALT_DIR/
-  chown -R root:root $DEFAULT_SALT_DIR/
   cd /tmp
 }
 
@@ -419,19 +418,6 @@ is_single_node_grid() {
 	grep "role: so-" /etc/salt/grains | grep -E "eval|standalone|import" &> /dev/null
 }
 
-remove_elasticsearch_index_template() {
-  local template_name=$1
-  local reason=${2:-"Removing index template for upgrade"}
-
-  # check if template exists
-  if so-elasticsearch-query "_index_template/$template_name" --fail --retry 3 --retry-delay 5 >/dev/null 2>&1; then
-    echo "Removing Elasticsearch index template: $template_name ($reason)"
-    if ! so-elasticsearch-query "_index_template/$template_name" -XDELETE --fail --retry 3 --retry-delay 5; then
-      return 1
-    fi
-  fi
-}
-
 initialize_elasticsearch_indices() {
     local index_names=$1
     local default_entry=${2:-'{"@timestamp":"0"}'}

salt/common/tools/sbin/so-common-status-check

@@ -9,7 +9,6 @@ import sys
 import subprocess
 import os
 import json
-import tempfile
 
 sys.path.append('/opt/saltstack/salt/lib/python3.10/site-packages/')
 import salt.config
@@ -18,21 +17,6 @@ import salt.loader
 __opts__ = salt.config.minion_config('/etc/salt/minion')
 __grains__ = salt.loader.grains(__opts__)
 
-def write_atomic(path, value):
-  # telegraf reads these files on its own schedule; replacing them by rename means it never
-  # reads a truncated file and reports an empty value as if it were real
-  directory = os.path.dirname(path)
-  handle, temp = tempfile.mkstemp(dir=directory)
-  try:
-    with os.fdopen(handle, 'w') as f:
-      f.write(str(value))
-    os.chmod(temp, 0o644)
-    os.replace(temp, path)
-  except Exception:
-    os.path.exists(temp) and os.unlink(temp)
-    raise
-
-
 def check_needs_restarted():
   osfam = __grains__['os_family']
   val = '0'
@@ -50,7 +34,8 @@ def check_needs_restarted():
   else:
     fail("Unsupported OS")
 
-  write_atomic(outfile, val)
+  with open(outfile, 'w') as f:
+    f.write(val)
 
 def check_for_fps():
     feat = 'fps'
@@ -71,7 +56,8 @@ def check_for_fps():
           # Unknown, so assume 0
           fps = 0
 
-    write_atomic('/opt/so/log/sostatus/fps_enabled', fps)
+    with open('/opt/so/log/sostatus/fps_enabled', 'w') as f:
+       f.write(str(fps))
 
 def check_for_lks():
     feat = 'Lks'
@@ -94,7 +80,8 @@ def check_for_lks():
                                 lks = 1
         if lks:
             break
-    write_atomic('/opt/so/log/sostatus/lks_enabled', lks)
+    with open('/opt/so/log/sostatus/lks_enabled', 'w') as f:
+       f.write(str(lks))
 
 def fail(msg):
   print(msg, file=sys.stderr)

salt/common/tools/sbin/so-kernel-upgrade

@@ -5,11 +5,10 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 #
-# so-kernel-upgrade — install the UEK8 (6.x) kernel, make it the boot default, and once the
+# so-kernel-upgrade — install the UEK8 (6.x) kernel and make it the boot default.
-# node is running it, remove the stock EL9 kernel.
 #
 # Security Onion is moving off the EL9 stock kernel (RHCK, 5.14) and UEK7 (5.15) onto UEK8
-# (6.x). Four things have to happen, and the tool has to drive each one:
+# (6.x). Three things have to happen, and the tool has to drive each one:
 #
 #   1. Populate. The manager mirrors the UEK8 packages into /nsm/kernelrepo via so-repo-sync,
 #      and serves them to the grid over https://<manager>/kernelrepo. Until that sync runs the
@@ -27,21 +26,10 @@
 #        - From the stock EL9 kernel (RHCK, 5.14, no UEK) it is a flavor CROSS that is NOT
 #          auto-promoted, so the box keeps booting RHCK until grubby is told otherwise.
 #      This tool inspects the running kernel and only runs 'grubby --set-default' for RHCK.
-#   4. Clean up. Once the node is actually RUNNING UEK8 the stock kernel packages are dead
-#      weight -- disk in /boot and a stale GRUB entry. They cannot come off any earlier:
-#      dnf's protect_running_kernel refuses to erase the booted kernel-core, so the removal
-#      has to wait for the reboot. Waiting is also the safer sequencing on its own terms --
-#      the node has proven it comes up on UEK8 before its fallback is deleted. That is why
-#      the removal does not happen in the uek7 branch either, where dnf would allow it.
 #
 # Every one of those failure modes is silent by default. This tool handles each case and fails
 # loudly when it cannot, rather than reporting success while changing nothing.
 #
-# Invocation: with no arguments it drives the whole sequence for whatever kernel the node is
-# on. With --cleanup it does the step 4 removal ONLY, and no-ops on a node that isn't running
-# UEK8 yet -- that is the form the common highstate calls (remove_stock_kernel in
-# salt/common/init.sls) so the cleanup lands grid-wide after each node reboots.
-#
 # Manager vs minion: only the manager owns /nsm/kernelrepo, so only the manager can populate
 # it. If the repo is empty here, a manager runs so-repo-sync itself; a minion has no way to
 # fix it and exits non-zero telling the admin to sync the manager first.
@@ -61,11 +49,6 @@ KERNEL_REPO_DIR="/nsm/kernelrepo"
 REPOSYNC_CONF="/opt/so/conf/reposync/repodownload.conf"
 GLOBAL_PILLAR="/opt/so/saltstack/local/pillar/global/soc_global.sls"
 
-# Stock EL9 (RHCK) kernel packages, removed only once the node is running UEK8 (see step 4
-# in the header). Left deliberately narrow: UEK7 kernel-uek builds age out on their own via
-# installonly_limit=3, and kernel-devel/kernel-headers are not touched.
-RHCK_PKGS="kernel kernel-core kernel-modules kernel-modules-core kernel-tools kernel-tools-libs"
-
 log() { echo "[so-kernel-upgrade] $*"; }
 die() { echo "[so-kernel-upgrade] ERROR: $*" >&2; exit 1; }
 
@@ -166,13 +149,8 @@ ensure_kernel_repo() {
 }
 
 reboot_notice() {
-    [ "$(uname -r)" = "$(basename "$1" | sed 's/^vmlinuz-//')" ] && return 0
+    [ "$(uname -r)" = "$(basename "$1" | sed 's/^vmlinuz-//')" ] \
-    log "REBOOT REQUIRED to start using the UEK8 kernel (currently running $(uname -r))."
+        || log "REBOOT REQUIRED to start using the UEK8 kernel (currently running $(uname -r))."
-    # The stock kernel can't be removed until it stops being the running one, so say when
-    # that will happen rather than leaving the admin to wonder if it was missed.
-    [ -n "$(rhck_installed)" ] \
-        && log "The stock EL9 kernel is left in place until then; it is removed by the next highstate after the reboot."
-    return 0
 }
 
 # Keep future kernel updates on the UEK line rather than falling back to RHCK. Oracle ships
@@ -184,32 +162,6 @@ set_default_kernel_conf() {
     fi
 }
 
-# Which of RHCK_PKGS are actually installed, one per line. rpm -qa treats each argument as a
-# name glob and prints only what it finds, so a package that was never installed (or is
-# already gone) simply doesn't appear -- no "not installed" noise and no non-zero exit.
-rhck_installed() {
-    rpm -qa $RHCK_PKGS 2>/dev/null
-}
-
-# Remove the stock EL9 kernel. Only ever called once the running kernel is UEK8. The rpm
-# check above is the idempotency guard, so this is a cheap no-op on every highstate after
-# the first one -- it costs an rpm query, not a dnf transaction.
-remove_rhck() {
-    local installed; installed="$(rhck_installed)"
-    if [ -z "$installed" ]; then
-        log "no stock EL9 (RHCK) kernel packages installed; nothing to remove."
-        return 0
-    fi
-
-    log "running UEK8; removing the stock EL9 (RHCK) kernel packages:"
-    echo "$installed" | sed 's/^/[so-kernel-upgrade]   /'
-    dnf -y remove $RHCK_PKGS || die "failed to remove the stock EL9 kernel packages"
-
-    installed="$(rhck_installed)"
-    [ -z "$installed" ] || die "dnf reported success but these remain: $(echo $installed)"
-    log "stock EL9 kernel packages removed."
-}
-
 # Make sure a UEK8 kernel is installed, leaving its boot entry in INSTALLED_UEK8. If one is
 # already present we leave the repo alone -- it may be disabled or empty and we don't need it
 # just to flip the boot default. Otherwise install the explicit NEVRA, not the bare package
@@ -232,38 +184,12 @@ ensure_uek8_installed() {
     log "installed UEK8 kernel: $INSTALLED_UEK8"
 }
 
-# --cleanup does step 4 and nothing else. It exits 0 rather than failing on a node that
-# isn't on UEK8 yet: the highstate gates on 'uname -r' before calling this, and a state that
-# fails whenever that gate races would be worse than one that says what it's waiting for.
-case "$1" in
-"")
-    ;;
---cleanup)
-    if [ "$(running_flavor)" != uek8 ]; then
-        log "not running a UEK8 kernel yet (currently $(uname -r)); leaving the stock EL9 kernel in place."
-        log "Run so-kernel-upgrade with no arguments to install UEK8, then reboot."
-        exit 0
-    fi
-    set_default_kernel_conf
-    remove_rhck
-    exit 0
-    ;;
-*)
-    echo "Usage: so-kernel-upgrade [--cleanup]" >&2
-    echo "  (no arguments)  install UEK8, make it the boot default, clean up once it's running" >&2
-    echo "  --cleanup       remove the stock EL9 kernel; no-op unless already running UEK8" >&2
-    exit 1
-    ;;
-esac
-
 case "$(running_flavor)" in
 uek8)
     # Already on the 6.x UEK line. A plain 'dnf update' keeps this node current within the
-    # lineage and auto-promotes newer builds, so there is no install or grubby work left --
+    # lineage and auto-promotes newer builds, so there is nothing for this tool to do.
-    # only the step 4 cleanup, which this is the first point in the sequence that can run it.
+    log "already running a UEK8 kernel ($(uname -r)); nothing to do."
-    log "already running a UEK8 kernel ($(uname -r)); no kernel install needed."
+    exit 0
-    set_default_kernel_conf
-    remove_rhck
     ;;
 
 uek7)

salt/common/tools/sbin/so-log-check

@@ -177,7 +177,6 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unexpected authorization header" # expected WARN log lines indicating invalid auth header
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Missing ory_kratos_session cookie" # expected WARN log lines indicating invalid auth header
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Static assets preprocessor only supports GET and HEAD requests" # expected WARN log lines indicating invalid auth header
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|respondError"                 # respondError is a function name, output via http middleware as standard request logging
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
@@ -241,7 +240,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal"           # docker container getting recycled
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown"             # so-dockerregistry logs a tag lookup miss during image copy; not tied to one docker version
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown"             # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Could not index event to Elasticsearch.*\"version\" => \"9.0.8\"" # Expected during Elastic upgrade temporarily, as policies referencing older pipelines are updated
 fi
 

salt/common/tools/sbin/so-sensor-clean

@@ -8,37 +8,21 @@
 # Elastic License 2.0.
 
 
-SENSOR_DIR="${SENSOR_DIR:-/nsm}"
+SENSOR_DIR='/nsm'
 CRIT_DISK_USAGE=90
-LOG="${LOG:-/opt/so/log/sensor_clean.log}"
+CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
-LOCK="${LOCK:-/var/tmp/so-sensor-clean.lock}"
+LOG="/opt/so/log/sensor_clean.log"
-MAX_PASSES=100
+TODAY=$(date -u "+%Y-%m-%d")
 
-ZEEK_LOGS="$SENSOR_DIR/zeek/logs"
-STRELKA_FILES="$SENSOR_DIR/strelka/processed"
-SURICATA_LOGS="$SENSOR_DIR/suricata"
-PCAPS="$SENSOR_DIR/pcapout"
-
-log() {
-	echo "$(date) - $*" >>"$LOG"
-}
-
-disk_usage() {
-	df -P "$SENSOR_DIR" | tail -1 | awk '{print $5}' | tr -d %
-}
-
-disk_avail() {
-	df -P "$SENSOR_DIR" | tail -1 | awk '{print $4}'
-}
-
-# sets REMOVED=1 if anything was actually deleted
 clean() {
 	## find the oldest Zeek logs directory
-	OLDEST_DIR=$(ls "$ZEEK_LOGS" 2>/dev/null | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
+	OLDEST_DIR=$(ls /nsm/zeek/logs/ | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
-	if [ -n "$OLDEST_DIR" ]; then
+	if [ -z "$OLDEST_DIR" -o "$OLDEST_DIR" == ".." -o "$OLDEST_DIR" == "." ]; then
-		log "Removing directory: $ZEEK_LOGS/$OLDEST_DIR"
+		echo "$(date) - No old Zeek logs available to clean up in /nsm/zeek/logs/" >>$LOG
-		rm -rf "$ZEEK_LOGS/$OLDEST_DIR"
+		#exit 0
-		REMOVED=1
+	else
+		echo "$(date) - Removing directory: /nsm/zeek/logs/$OLDEST_DIR" >>$LOG
+		rm -rf /nsm/zeek/logs/"$OLDEST_DIR"
 	fi
 
 	## Remarking for now, as we are moving extracted files to /nsm/strelka/processed
@@ -59,73 +43,58 @@ clean() {
 	#fi
 
 	## Clean up Zeek extracted files processed by Strelka
-	OLDEST_STRELKA=$(find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
+	STRELKA_FILES='/nsm/strelka/processed'
-	if [ -n "$OLDEST_STRELKA" ]; then
+	OLDEST_STRELKA=$(find $STRELKA_FILES -type f -printf '%T+ %p\n' | sort -n | head -n 1)
+	if [ -z "$OLDEST_STRELKA" -o "$OLDEST_STRELKA" == ".." -o "$OLDEST_STRELKA" == "." ]; then
+		echo "$(date) - No old files available to clean up in $STRELKA_FILES" >>$LOG
+	else
 		OLDEST_STRELKA_DATE=$(echo $OLDEST_STRELKA | awk '{print $1}' | cut -d+ -f1)
-		log "Removing extracted files for $OLDEST_STRELKA_DATE"
+		OLDEST_STRELKA_FILE=$(echo $OLDEST_STRELKA | awk '{print $2}')
-		REMOVED=1
+		echo "$(date) - Removing extracted files for $OLDEST_STRELKA_DATE" >>$LOG
-		find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
+		find $STRELKA_FILES -type f -printf '%T+ %p\n' | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
-			log "Removing file: $FILE"
+			echo "$(date) - Removing file: $FILE" >>$LOG
 			rm -f "$FILE"
 		done
 	fi
 
 	## Clean up Suricata log files
-	OLDEST_SURICATA=$(find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
+	SURICATA_LOGS='/nsm/suricata'
-	if [ -n "$OLDEST_SURICATA" ]; then
+	OLDEST_SURICATA=$(find $SURICATA_LOGS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
+	if [[ -z "$OLDEST_SURICATA" ]] || [[ "$OLDEST_SURICATA" == ".." ]] || [[ "$OLDEST_SURICATA" == "." ]]; then
+		echo "$(date) - No old files available to clean up in $SURICATA_LOGS" >>$LOG
+	else
 		OLDEST_SURICATA_DATE=$(echo $OLDEST_SURICATA | awk '{print $1}' | cut -d+ -f1)
-		log "Removing logs for $OLDEST_SURICATA_DATE"
+		OLDEST_SURICATA_FILE=$(echo $OLDEST_SURICATA | awk '{print $2}')
-		REMOVED=1
+		echo "$(date) - Removing logs for $OLDEST_SURICATA_DATE" >>$LOG
-		find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
+		find $SURICATA_LOGS -type f -printf '%T+ %p\n' | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
-			log "Removing file: $FILE"
+			echo "$(date) - Removing file: $FILE" >>$LOG
 			rm -f "$FILE"
 		done
 	fi
 
 	## Clean up extracted pcaps
-	OLDEST_PCAP=$(find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
+	PCAPS='/nsm/pcapout'
-	if [ -n "$OLDEST_PCAP" ]; then
+	OLDEST_PCAP=$(find $PCAPS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
+	if [ -z "$OLDEST_PCAP" -o "$OLDEST_PCAP" == ".." -o "$OLDEST_PCAP" == "." ]; then
+		echo "$(date) - No old files available to clean up in $PCAPS" >>$LOG
+	else
 		OLDEST_PCAP_DATE=$(echo $OLDEST_PCAP | awk '{print $1}' | cut -d+ -f1)
-		log "Removing extracted files for $OLDEST_PCAP_DATE"
+		OLDEST_PCAP_FILE=$(echo $OLDEST_PCAP | awk '{print $2}')
-		REMOVED=1
+		echo "$(date) - Removing extracted files for $OLDEST_PCAP_DATE" >>$LOG
-		find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
+		find $PCAPS -type f -printf '%T+ %p\n' | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
-			log "Removing file: $FILE"
+			echo "$(date) - Removing file: $FILE" >>$LOG
 			rm -f "$FILE"
 		done
 	fi
 }
 
-# Only one instance at a time; the lock is the fd, so it releases on any exit
+# Check to see if we are already running
-exec 9>"$LOCK" || exit 1
+NUM_RUNNING=$(pgrep -cf "/bin/bash /usr/sbin/so-sensor-clean")
-if ! flock -n 9; then
+[ "$NUM_RUNNING" -gt 1 ] && echo "$(date) - $NUM_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0
-	log "another so-sensor-clean is already running (lock $LOCK held); exiting"
+
-	exit 0
+if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then
+	while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
+		clean
+		CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
+	done
 fi
-
-CUR_USAGE=$(disk_usage)
-[ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ] || exit 0
-
-log "$SENSOR_DIR at ${CUR_USAGE}% (threshold ${CRIT_DISK_USAGE}%); starting cleanup"
-
-PASS=0
-while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
-	PASS=$((PASS + 1))
-	if [ "$PASS" -gt "$MAX_PASSES" ]; then
-		log "stopping after $MAX_PASSES passes; $SENSOR_DIR still at ${CUR_USAGE}%"
-		break
-	fi
-
-	REMOVED=0
-	BEFORE=$(disk_avail)
-	clean
-	CUR_USAGE=$(disk_usage)
-
-	if [ "$REMOVED" -eq 0 ]; then
-		log "nothing left to remove in $ZEEK_LOGS, $STRELKA_FILES, $SURICATA_LOGS, $PCAPS; $SENSOR_DIR still at ${CUR_USAGE}% - space is consumed outside of NSM cleanup scope"
-		break
-	fi
-	if [ "$(disk_avail)" -le "$BEFORE" ]; then
-		log "pass $PASS freed no space; $SENSOR_DIR still at ${CUR_USAGE}% - stopping until next run"
-		break
-	fi
-done

salt/common/tools/sbin_jinja/so-raid-status

@@ -125,6 +125,4 @@ else
   RAIDSTATUS=1
 fi
 
-# telegraf reads this file; write aside and rename so it never sees a half-written file
+echo "nsmraid=$RAIDSTATUS" > /opt/so/log/raid/status.log
-echo "nsmraid=$RAIDSTATUS" > /opt/so/log/raid/status.log.tmp
-mv -f /opt/so/log/raid/status.log.tmp /opt/so/log/raid/status.log

salt/docker/defaults.yaml

@@ -1,12 +1,6 @@
 docker:
   range: '172.17.1.0/24'
   gateway: '172.17.1.1'
-  networks:
-    sobridge: {}
-    soauth:
-      range: '172.17.2.0/24'
-      gateway: '172.17.2.1'
-      manager_only: True
   ulimits:
     - name: nofile
       soft: 1048576
@@ -64,18 +58,18 @@ docker:
       ulimits: []
     'so-kratos':
       final_octet: 28
-      networks: ['soauth']
       port_bindings:
         - 0.0.0.0:4433:4433
+        - 0.0.0.0:4434:4434
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
       ulimits: []
     'so-hydra':
       final_octet: 30
-      networks: ['soauth']
       port_bindings:
         - 0.0.0.0:4444:4444
+        - 0.0.0.0:4445:4445
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
@@ -134,7 +128,6 @@ docker:
       ulimits: []
     'so-soc':
       final_octet: 34
-      networks: ['sobridge', 'soauth']
       port_bindings:
         - 0.0.0.0:9822:9822
       custom_bind_mounts: []

salt/docker/docker.map.jinja

@@ -1,26 +1,8 @@
 {% import_yaml 'docker/defaults.yaml' as DOCKERDEFAULTS %}
 {% set DOCKERMERGED = salt['pillar.get']('docker', DOCKERDEFAULTS.docker, merge=True) %}
-
+{% set RANGESPLIT = DOCKERMERGED.range.split('.') %}
-{% if DOCKERMERGED.networks.sobridge is not mapping %}
+{% set FIRSTTHREE = RANGESPLIT[0] ~ '.' ~ RANGESPLIT[1] ~ '.' ~ RANGESPLIT[2] ~ '.' %}
-{%   do DOCKERMERGED.networks.update({'sobridge': {}}) %}
-{% endif %}
-{% do DOCKERMERGED.networks['sobridge'].update({'range': DOCKERMERGED.range, 'gateway': DOCKERMERGED.gateway}) %}
-
-{% for netname, net in DOCKERMERGED.networks.items() %}
-{%   set RANGESPLIT = net.range.split('.') %}
-{%   do net.update({'prefix': RANGESPLIT[0] ~ '.' ~ RANGESPLIT[1] ~ '.' ~ RANGESPLIT[2] ~ '.'}) %}
-{% endfor %}
 
 {% for container, vals in DOCKERMERGED.containers.items() %}
-{%   set CONTAINER_NETS = vals.get('networks', ['sobridge']) %}
+{%   do DOCKERMERGED.containers[container].update({'ip': FIRSTTHREE ~ DOCKERMERGED.containers[container].final_octet}) %}
-{%   set IPS = {} %}
-{%   for netname in CONTAINER_NETS %}
-{%     do IPS.update({netname: DOCKERMERGED.networks[netname].prefix ~ vals.final_octet}) %}
-{%   endfor %}
-{%   do DOCKERMERGED.containers[container].update({
-         'networks': CONTAINER_NETS,
-         'ips': IPS,
-         'network': CONTAINER_NETS[0],
-         'ip': IPS[CONTAINER_NETS[0]]
-       }) %}
 {% endfor %}

salt/docker/init.sls

@@ -18,10 +18,10 @@ dockergroup:
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 2.3.6-1.el9
+      - containerd.io: 2.2.1-1.el9
-      - docker-ce: 3:29.8.1-1.el9
+      - docker-ce: 3:29.2.1-1.el9
-      - docker-ce-cli: 1:29.8.1-1.el9
+      - docker-ce-cli: 1:29.2.1-1.el9
-      - docker-ce-rootless-extras: 29.8.1-1.el9
+      - docker-ce-rootless-extras: 29.2.1-1.el9
     - hold: True
     - update_holds: True
 
@@ -71,19 +71,15 @@ dockerreserveports:
     - source: salt://common/files/99-reserved-ports.conf
     - name: /etc/sysctl.d/99-reserved-ports.conf
 
-{% for NETNAME, NETWORK in DOCKERMERGED.networks.items() %}
+sos_docker_net:
-{% if not NETWORK.get('manager_only') or GLOBALS.get('is_manager', False) %}
-sos_docker_net_{{ NETNAME }}:
   docker_network.present:
-    - name: {{ NETNAME }}
+    - name: sobridge
-    - subnet: {{ NETWORK.range }}
+    - subnet: {{ DOCKERMERGED.range }}
-    - gateway: {{ NETWORK.gateway }}
+    - gateway: {{ DOCKERMERGED.gateway }}
     - options:
-        com.docker.network.bridge.name: '{{ NETNAME }}'
+        com.docker.network.bridge.name: 'sobridge'
         com.docker.network.driver.mtu: '1500'
         com.docker.network.bridge.enable_ip_masquerade: 'true'
         com.docker.network.bridge.enable_icc: 'true'
         com.docker.network.bridge.host_binding_ipv4: '0.0.0.0'
-    - unless: ip l | grep {{ NETNAME }}
+    - unless: ip l | grep sobridge
-{% endif %}
-{% endfor %}

salt/docker/soc_docker.yaml

@@ -7,40 +7,6 @@ docker:
     description: Default docker IP range for containers.
     helpLink: docker
     advanced: True
-  networks:
-    sobridge:
-      description: |
-        The default docker network, carrying most containers. Its range and gateway are taken
-        from the docker.range and docker.gateway settings above rather than set here.
-      helpLink: docker
-      readonly: True
-      advanced: True
-      global: True
-    soauth:
-      range:
-        description: |
-          IP range for the soauth docker network, an isolated network for the authentication
-          services, so that the Kratos and Hydra admin APIs are only reachable from the
-          containers placed on it.
-        helpLink: docker
-        readonly: True
-        advanced: True
-        global: True
-      gateway:
-        description: Gateway for the soauth docker network.
-        helpLink: docker
-        readonly: True
-        advanced: True
-        global: True
-      manager_only:
-        description: |
-          Limits the soauth network to grid members running the authentication containers,
-          instead of creating it on every node.
-        helpLink: docker
-        readonly: True
-        advanced: True
-        global: True
-        forcedType: bool
   ulimits:
     description: |
       Default ulimit settings applied to all containers via the Docker daemon. Each entry specifies a resource name (e.g. nofile, memlock, core, nproc) with soft and hard limits. Individual container ulimits override these defaults. Valid resource names include: cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime.
@@ -68,16 +34,6 @@ docker:
         readonly: True
         advanced: True
         global: True
-      networks:
-        description: |
-          Docker networks this container is attached to. The first entry is the container's
-          primary network and determines the address its published ports are forwarded to.
-          Defaults to sobridge when unset.
-        helpLink: docker
-        readonly: True
-        advanced: True
-        global: True
-        forcedType: "[]string"
       port_bindings:
         description: List of port bindings for the container.
         helpLink: docker

salt/elastalert/config.sls

@@ -21,7 +21,6 @@ elastalert:
     - gid: 933
     - home: /opt/so/conf/elastalert
     - createhome: False
-    - shell: /sbin/nologin
 
 elastalogdir:
   file.directory:
@@ -34,8 +33,8 @@ elastalert_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://elastalert/tools/sbin
-    - user: root
+    - user: 933
-    - group: root
+    - group: 939
     - file_mode: 755
 
 #elastalert_sbin_jinja:

salt/elastic-fleet-package-registry/config.sls

@@ -19,7 +19,6 @@ elastic-agent-pr:
     - gid: 948
     - home: /opt/so/conf/elastic-fleet-pr
     - createhome: False
-    - shell: /sbin/nologin
 
 {% else %}
 

salt/elasticagent/config.sls

@@ -20,7 +20,6 @@ elastic-agent:
     - gid: 949
     - home: /opt/so/conf/elastic-agent
     - createhome: False
-    - shell: /sbin/nologin
 
 elasticagentconfdir:
   file.directory:
@@ -40,8 +39,8 @@ elasticagent_sbin_jinja:
   file.recurse:
     - name: /usr/sbin
     - source: salt://elasticagent/tools/sbin_jinja
-    - user: root
+    - user: 949
-    - group: root
+    - group: 939
     - file_mode: 755
     - template: jinja
 

salt/elasticfleet/config.sls

@@ -26,14 +26,13 @@ elastic-fleet:
     - gid: 947
     - home: /opt/so/conf/elastic-fleet
     - createhome: False
-    - shell: /sbin/nologin
 
 elasticfleet_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://elasticfleet/tools/sbin
-    - user: root
+    - user: 947
-    - group: root
+    - group: 939
     - file_mode: 755
     - show_changes: False
 
@@ -41,8 +40,8 @@ elasticfleet_sbin_jinja:
   file.recurse:
     - name: /usr/sbin
     - source: salt://elasticfleet/tools/sbin_jinja
-    - user: root
+    - user: 947
-    - group: root
+    - group: 939 
     - file_mode: 755
     - template: jinja
     - exclude_pat:
@@ -82,8 +81,8 @@ eapackageupgrade:
   file.managed:
     - name: /usr/sbin/so-elastic-fleet-package-upgrade
     - source: salt://elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
-    - user: root
+    - user: 947
-    - group: root
+    - group: 939
     - mode: 755
     - template: jinja
 

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -29,7 +29,7 @@
               "\\.gz$"
             ],
             "include_files": [],
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- add_fields:\n    target: event\n    fields:\n      dataset: windows.forwarded\n      module: windows\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: import.evtx\n- if:\n    equals:\n      winlog.channel: 'Security'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: system.security\n          module: system\n- if:\n    equals:\n      winlog.channel: 'Windows PowerShell'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell\n          module: windows\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n          module: system\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n          module: system\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n- add_fields:\n    target: data_stream\n    fields:\n        type: logs\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.22.3\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.9.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.22.3\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.22.3\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.9.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
             ],

salt/elasticfleet/install_agent_grid.sls

@@ -14,8 +14,8 @@ so-elastic-agent-install:
   file.managed:
     - name: /usr/sbin/so-elastic-agent-install
     - source: salt://elasticfleet/tools/sbin/so-elastic-agent-install
-    - user: root
+    - user: 947
-    - group: root
+    - group: 939
     - mode: 755
     - show_changes: False
 

salt/elasticfleet/integration-defaults.map.jinja

@@ -30,14 +30,17 @@
     'azure_metrics.monitor': 'azure.monitor',
     'azure_metrics.storage_account': 'azure.storage_account',
     'azure_openai.metrics': 'azure.open_ai',
+    'beat.state': 'beats.stack_monitoring.state',
+    'beat.stats': 'beats.stack_monitoring.stats',
+    'enterprisesearch.health': 'enterprisesearch.stack_monitoring.health',
+    'enterprisesearch.stats': 'enterprisesearch.stack_monitoring.stats',
     'kibana.cluster_actions': 'kibana.stack_monitoring.cluster_actions',
     'kibana.cluster_rules': 'kibana.stack_monitoring.cluster_rules',
     'kibana.node_actions': 'kibana.stack_monitoring.node_actions',
     'kibana.node_rules': 'kibana.stack_monitoring.node_rules',
     'kibana.stats': 'kibana.stack_monitoring.stats',
     'kibana.status': 'kibana.stack_monitoring.status',
-    'logstash.node': 'logstash.stack_monitoring.node',
+    'logstash.node_cel': 'logstash.stack_monitoring.node',
-    'logstash.node_cel': 'logstash.node',
     'logstash.node_stats': 'logstash.stack_monitoring.node_stats',
     'synthetics.browser': 'synthetics-browser',
     'synthetics.browser_network': 'synthetics-browser.network',

salt/elasticfleet/tools/sbin/so-elastic-fleet-common

@@ -30,56 +30,6 @@ fleet_api() {
     curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/${QUERYPATH}" "$@" --retry 3 --retry-delay 10 --fail 2>/dev/null
 }
 
-elastic_fleet_require_agent_policy() {
-    local AGENT_POLICY=$1
-    local POLICY_JSON
-
-    if ! POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY") || [ -z "$POLICY_JSON" ]; then
-        echo "Error: Agent policy '$AGENT_POLICY' was not found or is not visible in the current Kibana space." >&2
-        return 1
-    fi
-
-    if ! jq -e '.item.package_policies | type == "array"' <<<"$POLICY_JSON" >/dev/null 2>&1; then
-        echo "Error: Agent policy '$AGENT_POLICY' was not found or is not visible in the current Kibana space." >&2
-        return 1
-    fi
-
-    echo "$POLICY_JSON"
-}
-
-# Print the single active enrollment token for POLICY_ID.
-# Exit 1: retryable (API failure, invalid response, no active token)
-# Exit 2: multiple active tokens - Shouldn't get into this state without manual intervention
-elastic_fleet_active_enrollment_token() {
-    local POLICY_ID=$1
-    local RESP TOKEN_COUNT API_KEY
-
-    if ! RESP=$(fleet_api "enrollment_api_keys?perPage=100" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
-        echo "Error: Failed to retrieve enrollment tokens for agent policy '$POLICY_ID'." >&2
-        return 1
-    fi
-
-    if ! jq -e '.list' <<<"$RESP" >/dev/null 2>&1; then
-        echo "Error: Invalid enrollment token response for agent policy '$POLICY_ID'." >&2
-        return 1
-    fi
-
-    TOKEN_COUNT=$(jq --arg pid "$POLICY_ID" '[.list[] | select(.policy_id == $pid and .active == true)] | length' <<<"$RESP")
-
-    if [ "${TOKEN_COUNT:-0}" -eq 0 ]; then
-        echo "Error: No active enrollment token found for agent policy '$POLICY_ID'." >&2
-        return 1
-    fi
-
-    if [ "$TOKEN_COUNT" -gt 1 ]; then
-        echo "Error: Found $TOKEN_COUNT active enrollment tokens for agent policy '$POLICY_ID'; expected exactly one." >&2
-        return 2
-    fi
-
-    API_KEY=$(jq -r --arg pid "$POLICY_ID" '.list[] | select(.policy_id == $pid and .active == true) | .api_key' <<<"$RESP")
-    echo "$API_KEY"
-}
-
 # Max number of concurrent Fleet write jobs (create/update). Override via env if needed.
 MAX_FLEET_JOBS=${MAX_FLEET_JOBS:-10}
 
@@ -112,7 +62,15 @@ elastic_fleet_load_integrations_dir() {
     i=0
 
     # Fetch the agent policy a single time; we look up integration ids locally below.
-    if ! POLICY_JSON=$(elastic_fleet_require_agent_policy "$AGENT_POLICY"); then
+    if ! POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY"); then
+        echo "Error: Failed to retrieve agent policy '$AGENT_POLICY'."
+        rm -f "$FAIL_FILE"
+        rm -rf "$OUT_DIR"
+        return 1
+    fi
+
+    if ! jq -e '.item.package_policies' <<<"$POLICY_JSON" >/dev/null 2>&1; then
+        echo "Error: Invalid agent policy response for '$AGENT_POLICY'."
         rm -f "$FAIL_FILE"
         rm -rf "$OUT_DIR"
         return 1
@@ -166,15 +124,9 @@ elastic_fleet_integration_check() {
 
     JSON_STRING=$2
 
-    NAME=$(jq -r .name "$JSON_STRING")
+    NAME=$(jq -r .name $JSON_STRING)
-    INTEGRATION_ID=""
 
-    local POLICY_JSON
+    INTEGRATION_ID=$(/usr/sbin/so-elastic-fleet-agent-policy-view "$AGENT_POLICY" | jq -r '.item.package_policies[] | select(.name=="'"$NAME"'") | .id')
-    if ! POLICY_JSON=$(elastic_fleet_require_agent_policy "$AGENT_POLICY"); then
-        return 1
-    fi
-
-    INTEGRATION_ID=$(jq -r --arg n "$NAME" '.item.package_policies[]? | select(.name==$n) | .id' <<<"$POLICY_JSON")
 
 }
 
@@ -196,16 +148,7 @@ elastic_fleet_integration_remove() {
 
     NAME=$2
 
-    local POLICY_JSON
+    INTEGRATION_ID=$(/usr/sbin/so-elastic-fleet-agent-policy-view "$AGENT_POLICY" | jq -r '.item.package_policies[] | select(.name=="'"$NAME"'") | .id')
-    if ! POLICY_JSON=$(elastic_fleet_require_agent_policy "$AGENT_POLICY"); then
-        return 1
-    fi
-
-    INTEGRATION_ID=$(jq -r --arg n "$NAME" '.item.package_policies[]? | select(.name==$n) | .id' <<<"$POLICY_JSON")
-    if [ -z "$INTEGRATION_ID" ]; then
-        echo "Error: Integration '$NAME' was not found in agent policy '$AGENT_POLICY'." >&2
-        return 1
-    fi
 
     JSON_STRING=$( jq -n \
                     --arg INTEGRATIONID "$INTEGRATION_ID" \

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-defend

@@ -13,10 +13,7 @@ ERROR=false
 for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/elastic-defend/*.json
 do
   printf "\n\nInitial Endpoints Policy - Loading $INTEGRATION\n"
-  if ! elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"; then
+  elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
-    ERROR=true
-    continue
-  fi
   if [ -n "$INTEGRATION_ID" ]; then
       printf "\n\nIntegration $NAME exists - Upgrading integration policy\n"
       if ! elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"; then

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server

@@ -7,35 +7,20 @@
 . /usr/sbin/so-elastic-fleet-common
 
 # Get all the fleet policies
-if ! json_output=$(fleet_api "agent_policies" -H 'kbn-xsrf: true'); then
+json_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -L -X GET "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true')
-    echo "Error: Failed to retrieve Fleet agent policies." >&2
-    exit 1
-fi
-
-if ! jq -e '.items' <<<"$json_output" >/dev/null 2>&1; then
-    echo "Error: Invalid Fleet agent policies response." >&2
-    exit 1
-fi
 
 # Extract the IDs that start with "FleetServer_"
-POLICY=$(jq -r '.items[] | select(.id | startswith("FleetServer_")) | .id' <<<"$json_output")
+POLICY=$(echo "$json_output" | jq -r '.items[] | select(.id | startswith("FleetServer_")) | .id')
 
 # Iterate over each ID in the POLICY variable
 for POLICYNAME in $POLICY; do
     printf "\nUpdating Policy: $POLICYNAME\n"
 
-    if ! POLICY_JSON=$(elastic_fleet_require_agent_policy "$POLICYNAME"); then
+    # First get the Integration ID
-        exit 1
+    INTEGRATION_ID=$(/usr/sbin/so-elastic-fleet-agent-policy-view "$POLICYNAME" | jq -r '.item.package_policies[] | select(.package.name == "fleet_server") | .id')  
-    fi
-
-    INTEGRATION_ID=$(jq -r '.item.package_policies[]? | select(.package.name == "fleet_server") | .id' <<<"$POLICY_JSON")
-    if [ -z "$INTEGRATION_ID" ]; then
-        echo "Error: fleet_server integration was not found in agent policy '$POLICYNAME'." >&2
-        exit 1
-    fi
 
     # Modify the default integration policy to update the policy_id and an with the correct naming
-    UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "$POLICYNAME" --arg name "fleet_server-$POLICYNAME" '
+	UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "$POLICYNAME" --arg name "fleet_server-$POLICYNAME" '
     .policy_id = $policy_id |
     .name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json)
 

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers

@@ -22,19 +22,12 @@ NUM_RUNNING=$(pgrep -cf "/bin/bash /sbin/so-elastic-agent-gen-installers")
 
 for i in {1..30}
 do
-   ENROLLMENTOKEN=$(elastic_fleet_active_enrollment_token "endpoints-initial")
+   ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys?perPage=100" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
-   TOKEN_RC=$?
-   if [ "$TOKEN_RC" -eq 2 ]; then
-     exit 1
-   fi
    FLEETHOST=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts/grid-default' | jq -r  '.item.host_urls[]' | paste -sd ',')
-   if [[ -n "$FLEETHOST" ]] && [[ -n "$ENROLLMENTOKEN" ]]; then
+if [[ $FLEETHOST ]] && [[  $ENROLLMENTOKEN ]]; then break; else sleep 10; fi
-     break
-   fi
-   sleep 10
 done
 
-if [[ -z "$FLEETHOST" ]] || [[ -z "$ENROLLMENTOKEN" ]]; then
+if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then 
   printf "\nFleet Host URL, Enrollment Token or Elastic Version empty - exiting..."
   printf "\nFleet Host: $FLEETHOST, Enrollment Token: $ENROLLMENTOKEN\n"
   exit 1
@@ -74,25 +67,19 @@ for GOOS in "${GOTARGETOS[@]}"; do
     GOARCH="amd64"
     if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi
     printf "\n\n### Generating $GOOS/$GOARCH Installer...\n"
-    if ! docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \
+    docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \
     --mount type=bind,source=/etc/pki/tls/certs/,target=/workspace/files/cert/ \
     --mount type=bind,source=/nsm/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \
     --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ \
-    {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN"  -o /output/so-elastic-agent_${GOOS}_${GOARCH}; then
+    {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN"  -o /output/so-elastic-agent_${GOOS}_${GOARCH}
-        printf "\n### ERROR: Failed to generate $GOOS/$GOARCH installer. Exiting...\n"
-        exit 1
-    fi
     printf "\n### $GOOS/$GOARCH Installer Generated...\n"
 done
 
 printf "\n\n### Generating MSI...\n"
 cp /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe
-if ! docker run \
+docker run \
 --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ -w /output \
-{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs; then
+{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs
-    printf "\n### ERROR: Failed to generate MSI. Exiting...\n"
-    exit 1
-fi
 printf "\n### MSI Generated...\n"
 
 # Verify installers were created

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -202,9 +202,26 @@ fi
 ### Finalization ###
 
 # Query for Enrollment Tokens for default policies
-ENDPOINTSENROLLMENTOKEN=$(elastic_fleet_active_enrollment_token "endpoints-initial") || exit 1
+if ENDPOINTSENROLLMENTOKEN_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
-GRIDNODESENROLLMENTOKENGENERAL=$(elastic_fleet_active_enrollment_token "so-grid-nodes_general") || exit 1
+    ENDPOINTSENROLLMENTOKEN=$(echo "$ENDPOINTSENROLLMENTOKEN_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
-GRIDNODESENROLLMENTOKENHEAVY=$(elastic_fleet_active_enrollment_token "so-grid-nodes_heavy") || exit 1
+else
+    echo -e "\nFailed to query for Endpoints enrollment token"
+    exit 1
+fi
+
+if GRIDNODESENROLLMENTOKENGENERAL_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
+    GRIDNODESENROLLMENTOKENGENERAL=$(echo "$GRIDNODESENROLLMENTOKENGENERAL_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_general")) | .api_key')
+else
+    echo -e "\nFailed to query for Grid nodes - General enrollment token"
+    exit 1
+fi
+
+if GRIDNODESENROLLMENTOKENHEAVY_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
+    GRIDNODESENROLLMENTOKENHEAVY=$(echo "$GRIDNODESENROLLMENTOKENHEAVY_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_heavy")) | .api_key')
+else
+    echo -e "\nFailed to query for Grid nodes - Heavy enrollment token"
+    exit 1
+fi
 
 # Store needed data in minion pillar
 pillar_file=/opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls

salt/elasticsearch/config.sls

@@ -32,14 +32,13 @@ elasticsearch:
     - gid: 930
     - home: /opt/so/conf/elasticsearch
     - createhome: False
-    - shell: /sbin/nologin
 
 elasticsearch_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://elasticsearch/tools/sbin
-    - user: root
+    - user: 930
-    - group: root
+    - group: 939
     - file_mode: 755
     - exclude_pat:
       - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state
@@ -50,8 +49,8 @@ so-elasticsearch-system-indices-patch-script:
   file.managed:
     - name: /usr/sbin/so-elasticsearch-system-indices-patch
     - source: salt://elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch
-    - user: root
+    - user: 930
-    - group: root
+    - group: 939
     - mode: 755
     - show_changes: False
 
@@ -59,8 +58,8 @@ elasticsearch_sbin_jinja:
   file.recurse:
     - name: /usr/sbin
     - source: salt://elasticsearch/tools/sbin_jinja
-    - user: root
+    - user: 939
-    - group: root
+    - group: 939 
     - file_mode: 755
     - template: jinja
     - exclude_pat:
@@ -73,8 +72,8 @@ so-elasticsearch-ilm-policy-load-script:
   file.managed:
     - name: /usr/sbin/so-elasticsearch-ilm-policy-load
     - source: salt://elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load
-    - user: root
+    - user: 930
-    - group: root
+    - group: 939
     - mode: 754
     - template: jinja
     - defaults:
@@ -85,8 +84,8 @@ so-elasticsearch-pipelines-script:
   file.managed:
     - name: /usr/sbin/so-elasticsearch-pipelines
     - source: salt://elasticsearch/tools/sbin/so-elasticsearch-pipelines
-    - user: root
+    - user: 930
-    - group: root
+    - group: 939
     - mode: 754
     - show_changes: False
 

salt/elasticsearch/defaults.yaml

@@ -3309,7 +3309,6 @@ elasticsearch:
         composed_of:
         - event-mappings
         - logs-system.security@package
-        - so-fleet_system.security_caseless-1
         - logs-system.security@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -4176,7 +4175,6 @@ elasticsearch:
       index_template:
         composed_of:
         - logs-windows.forwarded@package
-        - so-fleet_process_caseless-1
         - logs-windows.forwarded@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -4226,7 +4224,6 @@ elasticsearch:
       index_template:
         composed_of:
         - logs-windows.powershell@package
-        - so-fleet_process_caseless-1
         - logs-windows.powershell@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -4276,7 +4273,6 @@ elasticsearch:
       index_template:
         composed_of:
         - logs-windows.powershell_operational@package
-        - so-fleet_process_caseless-1
         - logs-windows.powershell_operational@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -4326,7 +4322,6 @@ elasticsearch:
       index_template:
         composed_of:
         - logs-windows.sysmon_operational@package
-        - so-fleet_process_caseless-1
         - logs-windows.sysmon_operational@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1

salt/elasticsearch/files/ingest/global@custom

@@ -99,7 +99,7 @@
         },
         {
             "set": {
-                "if": "ctx.tags != null && ctx.tags.contains('import') && ctx._index != null && ctx._index.startsWith('logs-import-')",
+                "if": "ctx.tags != null && ctx.tags.contains('import')",
                 "override": true,
                 "field": "data_stream.dataset",
                 "value": "import"
@@ -107,7 +107,7 @@
         },
         {
             "set": {
-                "if": "ctx.tags != null && ctx.tags.contains('import') && ctx._index != null && ctx._index.startsWith('logs-import-')",
+                "if": "ctx.tags != null && ctx.tags.contains('import')",
                 "override": true,
                 "field": "data_stream.namespace",
                 "value": "so"

salt/elasticsearch/files/ingest/import.evtx

@@ -1,31 +0,0 @@
-{
-  "description" : "import.evtx: normalize imported EVTX and reroute to logs-<dataset>-import",
-  "processors" : [
-    { "script":  {
-        "description": "Host from the event, not the importing node",
-        "lang": "painless",
-        "source": "Map host = ['os': ['type': 'windows', 'family': 'windows', 'platform': 'windows']]; def cn = ctx.winlog?.computer_name; if (cn != null && cn.toString().length() > 0) { String name = cn.toString(); int dot = name.indexOf('.'); if (dot > 0) { name = name.substring(0, dot); } host.put('hostname', name); host.put('name', name.toLowerCase()); } ctx.host = host;"
-    } },
-    { "script":  {
-        "description": "String event IDs, as Winlogbeat sends",
-        "lang": "painless",
-        "source": "if (ctx.winlog?.event_id != null) { ctx.winlog.event_id = ctx.winlog.event_id.toString(); } if (ctx.event?.code != null) { ctx.event.code = ctx.event.code.toString(); }"
-    } },
-    { "script":  {
-        "description": "Unnamed <Data> to param1..N, as Winlogbeat",
-        "lang": "painless",
-        "if": "ctx.winlog?.event_data?.Data instanceof Map && ctx.winlog.event_data.Data['#text'] != null",
-        "source": "def t = ctx.winlog.event_data.Data['#text']; List vals = t instanceof List ? t : [t]; for (int i = 0; i < vals.size(); i++) { ctx.winlog.event_data['param' + (i + 1)] = vals.get(i); } ctx.winlog.event_data.remove('Data');"
-    } },
-    { "script":  {
-        "description": "String values and LF line endings, as Winlogbeat",
-        "lang": "painless",
-        "if": "ctx.winlog?.event_data instanceof Map || ctx.winlog?.user_data instanceof Map",
-        "source": "String lf = String.valueOf((char) 10); String crlf = String.valueOf((char) 13) + lf; for (def key : ['event_data', 'user_data']) { def m = ctx.winlog[key]; if (!(m instanceof Map)) { continue; } for (def e : m.entrySet()) { def v = e.getValue(); if (v instanceof String) { e.setValue(v.replace(crlf, lf)); } else if (v instanceof Number || v instanceof Boolean) { e.setValue(v.toString()); } } }"
-    } },
-    { "set":     { "description": "event.kind, as Winlogbeat", "field": "event.kind", "value": "event", "override": false } },
-    { "set":     { "field": "data_stream.dataset", "copy_from": "event.dataset", "override": true, "ignore_empty_value": true } },
-    { "set":     { "field": "data_stream.namespace", "value": "import", "override": true } },
-    { "reroute": { "dataset": "{{data_stream.dataset}}", "namespace": "{{data_stream.namespace}}" } }
-  ]
-}

salt/elasticsearch/files/ingest/suricata.fileinfo

@@ -5,8 +5,7 @@
     { "rename": 	{ "field": "message2.proto", 			"target_field": "network.transport",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.fileinfo.filename", 	"target_field": "file.name",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.fileinfo.gaps", 		"target_field": "suricata.fileinfo.gaps",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fileinfo.gaps", 		"target_field": "file.bytes.missing",	"ignore_missing": true 	} },
-    { "set":		{ "if": "ctx.suricata?.fileinfo?.gaps == false", 	"field": "file.bytes.missing",	"value": 0 		} },
     { "rename": 	{ "field": "message2.fileinfo.magic", 		"target_field": "file.mime_type",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.fileinfo.md5", 		"target_field": "hash.md5",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.fileinfo.sha1", 		"target_field": "hash.sha1",		"ignore_missing": true 	} },

salt/elasticsearch/templates/component/elastic-agent/so-fleet_process_caseless-1.json

@@ -1,123 +0,0 @@
-{
-  "_meta": {
-    "managed_by": "security_onion",
-    "managed": true,
-    "description": "Adds .caseless for Lucene queries. Restates each field's package type and .text."
-  },
-  "template": {
-    "mappings": {
-      "properties": {
-        "process": {
-          "properties": {
-            "executable": {
-              "type": "keyword",
-              "ignore_above": 1024,
-              "fields": {
-                "caseless": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "normalizer": "lowercase"
-                },
-                "text": {
-                  "type": "match_only_text"
-                }
-              }
-            },
-            "name": {
-              "type": "keyword",
-              "ignore_above": 1024,
-              "fields": {
-                "caseless": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "normalizer": "lowercase"
-                },
-                "text": {
-                  "type": "match_only_text"
-                }
-              }
-            },
-            "command_line": {
-              "type": "wildcard",
-              "ignore_above": 1024,
-              "fields": {
-                "caseless": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "normalizer": "lowercase"
-                },
-                "text": {
-                  "type": "match_only_text"
-                }
-              }
-            },
-            "parent": {
-              "properties": {
-                "executable": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "fields": {
-                    "caseless": {
-                      "type": "keyword",
-                      "ignore_above": 1024,
-                      "normalizer": "lowercase"
-                    },
-                    "text": {
-                      "type": "match_only_text"
-                    }
-                  }
-                },
-                "name": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "fields": {
-                    "caseless": {
-                      "type": "keyword",
-                      "ignore_above": 1024,
-                      "normalizer": "lowercase"
-                    },
-                    "text": {
-                      "type": "match_only_text"
-                    }
-                  }
-                },
-                "command_line": {
-                  "type": "wildcard",
-                  "ignore_above": 1024,
-                  "fields": {
-                    "caseless": {
-                      "type": "keyword",
-                      "ignore_above": 1024,
-                      "normalizer": "lowercase"
-                    },
-                    "text": {
-                      "type": "match_only_text"
-                    }
-                  }
-                }
-              }
-            }
-          }
-        },
-        "file": {
-          "properties": {
-            "path": {
-              "type": "keyword",
-              "ignore_above": 1024,
-              "fields": {
-                "caseless": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "normalizer": "lowercase"
-                },
-                "text": {
-                  "type": "match_only_text"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-}

salt/elasticsearch/templates/component/elastic-agent/so-fleet_system.security_caseless-1.json

@@ -1,80 +0,0 @@
-{
-  "_meta": {
-    "managed_by": "security_onion",
-    "managed": true,
-    "description": "Adds .caseless for Lucene queries. Keeps each field's existing keyword type."
-  },
-  "template": {
-    "mappings": {
-      "properties": {
-        "process": {
-          "properties": {
-            "command_line": {
-              "type": "keyword",
-              "ignore_above": 1024,
-              "fields": {
-                "caseless": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "normalizer": "lowercase"
-                }
-              }
-            },
-            "parent": {
-              "properties": {
-                "executable": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "fields": {
-                    "caseless": {
-                      "type": "keyword",
-                      "ignore_above": 1024,
-                      "normalizer": "lowercase"
-                    }
-                  }
-                },
-                "name": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "fields": {
-                    "caseless": {
-                      "type": "keyword",
-                      "ignore_above": 1024,
-                      "normalizer": "lowercase"
-                    }
-                  }
-                },
-                "command_line": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "fields": {
-                    "caseless": {
-                      "type": "keyword",
-                      "ignore_above": 1024,
-                      "normalizer": "lowercase"
-                    }
-                  }
-                }
-              }
-            }
-          }
-        },
-        "file": {
-          "properties": {
-            "path": {
-              "type": "keyword",
-              "ignore_above": 1024,
-              "fields": {
-                "caseless": {
-                  "type": "keyword",
-                  "ignore_above": 1024,
-                  "normalizer": "lowercase"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-}

salt/elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch

@@ -19,8 +19,6 @@ INDEX_PATTERNS=(
   '.entities.v2.latest.security_*'
   '.entities.v2.history.security_*'
   'risk-score.risk-score-latest-*'
-  '.metrics-endpoint.metadata_united_*'
-  '.metrics-endpoint.metadata_current_*'
 )
 DATA_STREAM_PATTERNS=(
   '.entities.v2.updates.security_*'
@@ -35,8 +33,6 @@ TEMPLATE_PATTERNS=(
   '.risk-score.risk-score-default-index-template'
   '.rule-events'
   '.alert-actions'
-  '.metrics-endpoint.metadata_united_default-template'
-  '.metrics-endpoint.metadata_current_default-template'
 )
 
 query_es() {

salt/firewall/iptables.jinja

@@ -4,19 +4,11 @@
 {%- set role = GLOBALS.role.split('-')[1] %}
 {%- from 'firewall/containers.map.jinja' import NODE_CONTAINERS %}
 
-{%- set NODE_NETWORKS = [] %}
-{%- for NETNAME, NETWORK in DOCKERMERGED.networks.items() %}
-{%-   if not NETWORK.get('manager_only') or GLOBALS.get('is_manager', False) %}
-{%-     do NODE_NETWORKS.append(NETNAME) %}
-{%-   endif %}
-{%- endfor %}
-
 {%- set PR = [] %}
 {%- set D1 = [] %}
 {%- set D2 = [] %}
 {%- for container in NODE_CONTAINERS %}
 {%-   set IP = DOCKERMERGED.containers[container].ip %}
-{%-   set BRIDGE = DOCKERMERGED.containers[container].network %}
 {%-   if DOCKERMERGED.containers[container].port_bindings is defined %}
 {%-     for binding in DOCKERMERGED.containers[container].port_bindings %}
 {#-       cant split int so we convert to string #}
@@ -43,11 +35,11 @@
 {%-         endif %}
 {%-         do PR.append("-A POSTROUTING -s " ~ DOCKERMERGED.containers[container].ip ~ "/32 -d " ~ DOCKERMERGED.containers[container].ip ~ "/32 -p " ~  proto ~ " -m " ~ proto  ~ " --dport " ~ containerPort ~ " -j MASQUERADE") %}
 {%-         if bindip | length and bindip != '0.0.0.0' %}
-{%-           do D1.append("-A DOCKER -d " ~ bindip ~ "/32 ! -i " ~ BRIDGE ~ " -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
+{%-           do D1.append("-A DOCKER -d " ~ bindip ~ "/32 ! -i sobridge -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
 {%-         else %}
-{%-           do D1.append("-A DOCKER ! -i " ~ BRIDGE ~ " -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
+{%-           do D1.append("-A DOCKER ! -i sobridge -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
 {%-         endif %}
-{%-         do D2.append("-A DOCKER -d " ~ DOCKERMERGED.containers[container].ip ~ "/32 ! -i " ~ BRIDGE ~ " -o " ~ BRIDGE ~ " -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ containerPort ~ " -j ACCEPT") %}
+{%-         do D2.append("-A DOCKER -d " ~ DOCKERMERGED.containers[container].ip ~ "/32 ! -i sobridge -o sobridge -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ containerPort ~ " -j ACCEPT") %}
 {%-     endfor %}
 {%-   endif %}
 {%- endfor %}
@@ -60,15 +52,11 @@
 :DOCKER - [0:0]
 -A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER
 -A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER
-{%- for NETNAME in NODE_NETWORKS %}
+-A POSTROUTING -s {{DOCKERMERGED.range}} ! -o sobridge -j MASQUERADE
--A POSTROUTING -s {{ DOCKERMERGED.networks[NETNAME].range }} ! -o {{ NETNAME }} -j MASQUERADE
-{%- endfor %}
 {%- for rule in PR %}
 {{ rule }}
 {%- endfor %}
-{%- for NETNAME in NODE_NETWORKS %}
+-A DOCKER -i sobridge -j RETURN
--A DOCKER -i {{ NETNAME }} -j RETURN
-{%- endfor %}
 {%- for rule in D1 %}
 {{ rule }}
 {%- endfor %}
@@ -109,12 +97,10 @@ COMMIT
 {%- endif %}
 -A FORWARD -j DOCKER-USER
 -A FORWARD -j DOCKER-ISOLATION-STAGE-1
-{%- for NETNAME in NODE_NETWORKS %}
+-A FORWARD -o sobridge -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
--A FORWARD -o {{ NETNAME }} -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A FORWARD -o sobridge -j DOCKER
--A FORWARD -o {{ NETNAME }} -j DOCKER
+-A FORWARD -i sobridge ! -o sobridge -j ACCEPT
--A FORWARD -i {{ NETNAME }} ! -o {{ NETNAME }} -j ACCEPT
+-A FORWARD -i sobridge -o sobridge -j ACCEPT
--A FORWARD -i {{ NETNAME }} -o {{ NETNAME }} -j ACCEPT
-{%- endfor %}
 -A FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
 -A FORWARD -i lo -j ACCEPT
 -A FORWARD -m conntrack --ctstate INVALID -j DROP
@@ -126,18 +112,13 @@ COMMIT
 {%- for rule in D2 %}
 {{ rule }}
 {%- endfor %}
-{% for NETNAME in NODE_NETWORKS %}
+
--A DOCKER-ISOLATION-STAGE-1 -i {{ NETNAME }} ! -o {{ NETNAME }} -j DOCKER-ISOLATION-STAGE-2
+-A DOCKER-ISOLATION-STAGE-1 -i sobridge ! -o sobridge -j DOCKER-ISOLATION-STAGE-2
-{%- endfor %}
 -A DOCKER-ISOLATION-STAGE-1 -j RETURN
-{%- for NETNAME in NODE_NETWORKS %}
+-A DOCKER-ISOLATION-STAGE-2 -o sobridge -j DROP
--A DOCKER-ISOLATION-STAGE-2 -o {{ NETNAME }} -j DROP
-{%- endfor %}
 -A DOCKER-ISOLATION-STAGE-2 -j RETURN
-{%- for NETNAME in NODE_NETWORKS %}
+-A DOCKER-USER ! -i sobridge -o sobridge -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
--A DOCKER-USER ! -i {{ NETNAME }} -o {{ NETNAME }} -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-USER ! -i sobridge -o sobridge -j LOGGING
--A DOCKER-USER ! -i {{ NETNAME }} -o {{ NETNAME }} -j LOGGING
-{%- endfor %}
 -A DOCKER-USER -j RETURN
 -A LOGGING -m limit --limit 2/min -j LOG --log-prefix "IPTables-dropped: "
 -A LOGGING -j DROP

salt/firewall/map.jinja

@@ -4,12 +4,8 @@
 
 {# add our ip to self #}
 {% do FIREWALL_DEFAULT.firewall.hostgroups.self.append(GLOBALS.node_ip) %}
-{# add dockernet ranges #}
+{# add dockernet range #}
-{% for NETNAME, NETWORK in DOCKERMERGED.networks.items() %}
+{% do FIREWALL_DEFAULT.firewall.hostgroups.dockernet.append(DOCKERMERGED.range) %}
-{%   if not NETWORK.get('manager_only') or GLOBALS.get('is_manager', False) %}
-{%     do FIREWALL_DEFAULT.firewall.hostgroups.dockernet.append(NETWORK.range) %}
-{%   endif %}
-{% endfor %}
 
 {% if GLOBALS.role == 'so-idh' %}
 {%   from 'idh/opencanary_config.map.jinja' import IDH_PORTGROUPS %}

salt/hydra/enabled.sls

@@ -26,8 +26,8 @@ so-hydra:
     - hostname: hydra
     - name: so-hydra
     - networks:
-      - soauth:
+      - sobridge:
-        - ipv4_address: {{ DOCKERMERGED.containers['so-hydra'].ips['soauth'] }}
+        - ipv4_address: {{ DOCKERMERGED.containers['so-hydra'].ip }}
     - binds:
       - /opt/so/conf/hydra/:/hydra-conf:ro
       - /opt/so/log/hydra/:/hydra-log:rw
@@ -73,7 +73,7 @@ delete_so-hydra_so-status.disabled:
 
 wait_for_hydra:
   http.wait_for_successful_query:
-    - name: 'http://{{ DOCKERMERGED.containers['so-hydra'].ips['soauth'] }}:4444/health/alive'
+    - name: 'http://{{ GLOBALS.manager }}:4444/health/alive'
     - ssl: True
     - verify_ssl: False
     - status:

salt/hypervisor/init.sls

@@ -21,16 +21,12 @@ hypervisor_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://hypervisor/tools/sbin
-    - user: root
-    - group: root
     - file_mode: 744
 
 hypervisor_sbin_jinja:
   file.recurse:
     - name: /usr/sbin
     - source: salt://hypervisor/tools/sbin_jinja
-    - user: root
-    - group: root
     - template: jinja
     - file_mode: 744
 

salt/idh/config.sls

@@ -86,8 +86,8 @@ idh_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://idh/tools/sbin
-    - user: root
+    - user: 939
-    - group: root
+    - group: 939
     - file_mode: 755
 
 #idh_sbin_jinja:

salt/influxdb/config.sls

@@ -41,8 +41,8 @@ influxdb_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://influxdb/tools/sbin
-    - user: root
+    - user: 939
-    - group: root
+    - group: 939
     - file_mode: 755
 
 #influxdb_sbin_jinja:

salt/influxdb/enabled.sls

@@ -94,11 +94,9 @@ metrics_link_file:
       - docker_container: so-influxdb
 
 # Install cron job to determine size of influxdb for telegraf
-# telegraf reads this while the cron rewrites it, so write aside and rename rather than
-# truncating in place. tgraflogdir recurses ownership, so the temp file is chowned to match
 get_influxdb_size:
   cron.present:
-    - name: 'du -s -k /nsm/influxdb | cut -f1 > /opt/so/log/telegraf/influxdb_size.log.tmp 2>&1; chown 939:939 /opt/so/log/telegraf/influxdb_size.log.tmp; mv -f /opt/so/log/telegraf/influxdb_size.log.tmp /opt/so/log/telegraf/influxdb_size.log'
+    - name: 'du -s -k /nsm/influxdb | cut -f1 > /opt/so/log/telegraf/influxdb_size.log 2>&1'
     - identifier: get_influxdb_size
     - user: root
     - minute: '*/1'

salt/kafka/config.sls

@@ -21,7 +21,6 @@ kafka_user:
     - gid: 960
     - home: /opt/so/conf/kafka
     - createhome: False
-    - shell: /sbin/nologin
 
 kafka_home_dir:
   file.absent:
@@ -31,16 +30,16 @@ kafka_sbin_tools:
   file.recurse:
     - name: /usr/sbin
     - source: salt://kafka/tools/sbin
-    - user: root
+    - user: 960
-    - group: root
+    - group: 960
     - file_mode: 755
 
 kafka_sbin_jinja_tools:
   file.recurse:
     - name: /usr/sbin
     - source: salt://kafka/tools/sbin_jinja
-    - user: root
+    - user: 960
-    - group: root
+    - group: 960
     - file_mode: 755
     - template: jinja
     - defaults:

salt/kibana/config.sls

@@ -22,7 +22,6 @@ kibana:
     - gid: 932
     - home: /opt/so/conf/kibana
     - createhome: False
-    - shell: /sbin/nologin
 
 # Drop the correct nginx config based on role
 
@@ -37,16 +36,16 @@ kibana_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://kibana/tools/sbin
-    - user: root
+    - user: 932
-    - group: root
+    - group: 939
     - file_mode: 755
 
 kibana_sbin_jinja:
   file.recurse:
     - name: /usr/sbin
     - source: salt://kibana/tools/sbin_jinja
-    - user: root
+    - user: 932
-    - group: root
+    - group: 939 
     - file_mode: 755
     - template: jinja
     - defaults:

salt/kratos/config.sls

@@ -27,7 +27,6 @@ kratos:
     - uid: 928
     - gid: 928
     - home: /opt/so/conf/kratos
-    - shell: /sbin/nologin
 
 kratosdir:
   file.directory:

salt/kratos/enabled.sls

@@ -19,8 +19,8 @@ so-kratos:
     - hostname: kratos
     - name: so-kratos
     - networks:
-      - soauth:
+      - sobridge:
-        - ipv4_address: {{ DOCKERMERGED.containers['so-kratos'].ips['soauth'] }}
+        - ipv4_address: {{ DOCKERMERGED.containers['so-kratos'].ip }}
     - binds:
       - /opt/so/conf/kratos/:/kratos-conf:ro
       - /opt/so/log/kratos/:/kratos-log:rw
@@ -71,7 +71,7 @@ delete_so-kratos_so-status.disabled:
 
 wait_for_kratos:
   http.wait_for_successful_query:
-    - name: 'http://{{ DOCKERMERGED.containers['so-kratos'].ips['soauth'] }}:4434/'
+    - name: 'http://{{ GLOBALS.manager }}:4434/'
     - ssl: True
     - verify_ssl: False
     - status:

salt/libvirt/64962/init.sls

@@ -6,8 +6,6 @@ so-fix-salt-ldap_script:
   file.managed:
     - name: /usr/sbin/so-fix-salt-ldap.py
     - source: salt://libvirt/64962/scripts/so-fix-salt-ldap.py
-    - user: root
-    - group: root
     - mode: 744
 
 fix-salt-ldap:

salt/logrotate/defaults.yaml

@@ -220,26 +220,6 @@ logrotate:
       - extension .log
       - dateext
       - dateyesterday
-    /opt/so/log/salt/virtual_node_manager:
-      - daily
-      - rotate 14
-      - missingok
-      - copytruncate
-      - compress
-      - create
-      - extension .log
-      - dateext
-      - dateyesterday
-    /opt/so/log/salt/so-salt-cloud:
-      - daily
-      - rotate 14
-      - missingok
-      - copytruncate
-      - compress
-      - create
-      - extension .log
-      - dateext
-      - dateyesterday
     /opt/so/log/salt/so-soup-grid-highstate:
       - daily
       - rotate 14

salt/logrotate/soc_logrotate.yaml

@@ -140,20 +140,6 @@ logrotate:
       multiline: True
       global: True
       forcedType: "[]string"
-    "/opt/so/log/salt/virtual_node_manager":
-      description: List of logrotate options for this file.
-      title: /opt/so/log/salt/virtual_node_manager
-      advanced: True
-      multiline: True
-      global: True
-      forcedType: "[]string"
-    "/opt/so/log/salt/so-salt-cloud":
-      description: List of logrotate options for this file.
-      title: /opt/so/log/salt/so-salt-cloud
-      advanced: True
-      multiline: True
-      global: True
-      forcedType: "[]string"
     "/opt/so/log/salt/so-soup-grid-highstate":
       description: List of logrotate options for this file.
       title: /opt/so/log/salt/so-soup-grid-highstate

salt/logstash/config.sls

@@ -35,14 +35,13 @@ logstash:
     - uid: 931
     - gid: 931
     - home: /opt/so/conf/logstash
-    - shell: /sbin/nologin
 
 logstash_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://logstash/tools/sbin
-    - user: root
+    - user: 931
-    - group: root
+    - group: 939
     - file_mode: 755
 
 #logstash_sbin_jinja:
@@ -82,14 +81,6 @@ ls_custom_pipeline_conf_{{assigned_pipeline}}_{{pipeline}}:
 
 
 {% for assigned_pipeline in ASSIGNED_PIPELINES %}
-{# a blank per-pipeline setting falls back to the global logstash.yml value #}
-{% set PARSED_OVERRIDES = LOGSTASH_MERGED.get('pipeline_settings', {}).get(assigned_pipeline, {}) %}
-{% if PARSED_OVERRIDES is not mapping %}
-{%   do salt.log.warning('logstash: ignoring malformed pipeline_settings for pipeline ' ~ assigned_pipeline ~ '; expected a set of settings') %}
-{% endif %}
-{% set PIPELINE_OVERRIDES = PARSED_OVERRIDES if PARSED_OVERRIDES is mapping else {} %}
-{% set THREADS = PIPELINE_OVERRIDES.get('pipeline_x_workers') or LOGSTASH_MERGED.config.pipeline_x_workers %}
-{% set BATCH = PIPELINE_OVERRIDES.get('pipeline_x_batch_x_size') or LOGSTASH_MERGED.config.pipeline_x_batch_x_size %}
     {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %}
 ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}:
   file.managed:
@@ -101,8 +92,8 @@ ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_")
         GLOBALS: {{ GLOBALS }}
         ES_USER: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') }}"
         ES_PASS: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') }}"
-        THREADS: {{ THREADS }}
+        THREADS: {{ LOGSTASH_MERGED.config.pipeline_x_workers }}
-        BATCH: {{ BATCH }}
+        BATCH: {{ LOGSTASH_MERGED.config.pipeline_x_batch_x_size }}
       {% else %}
     - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1]}}
       {% endif %}
@@ -134,14 +125,6 @@ lspipelinesyml:
     - defaults:
         ASSIGNED_PIPELINES: {{ ASSIGNED_PIPELINES }}
 
-lslog4j2:
-  file.managed:
-    - name: /opt/so/conf/logstash/etc/log4j2.properties
-    - source: salt://logstash/etc/log4j2.properties.jinja
-    - template: jinja
-    - user: 931
-    - group: 939
-
 lsetcsync:
   file.recurse:
     - name: /opt/so/conf/logstash/etc
@@ -150,11 +133,7 @@ lsetcsync:
     - group: 939
     - template: jinja
     - clean: True
-{#- both names are matched: the .jinja source so the recurse does not copy it verbatim,
+    - exclude_pat: pipelines*
-    and the rendered file so clean: True does not delete what lslog4j2 wrote #}
-    - exclude_pat:
-      - pipelines*
-      - log4j2.properties*
     - defaults:
         LOGSTASH_MERGED: {{ LOGSTASH_MERGED }}
 

salt/logstash/defaults.yaml

@@ -42,11 +42,6 @@ logstash:
     custom2: []
     custom3: []
     custom4: []
-    custom5: []
-    custom6: []
-    custom7: []
-    custom8: []
-    custom9: []
   pipeline_config:
     custom001: |-
       filter {
@@ -65,405 +60,10 @@ logstash:
     custom008: PLACEHOLDER
     custom009: PLACEHOLDER
     custom010: PLACEHOLDER
-  pipeline_settings:
-    fleet:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    manager:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    receiver:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    search:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom0:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom1:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom2:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom3:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom4:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom5:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom6:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom7:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom8:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom9:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
   settings:
     lsheap: 500m
   config:
     api_x_http_x_host: 0.0.0.0
-    log_x_level: info
-    log_x_format: plain
     path_x_logs: /var/log/logstash
     pipeline_x_workers: 1
     pipeline_x_batch_x_size: 125

salt/logstash/enabled.sls

@@ -105,8 +105,6 @@ so-logstash:
     {% endif %}
     - watch:
       - file: lsetcsync
-      - file: lslog4j2
-      - file: lspipelinesyml
       - file: trusttheca
       {% if GLOBALS.is_manager %}
       - file: elasticsearch_cacerts

salt/logstash/etc/log4j2.properties

@@ -1,4 +1,3 @@
-{%- from 'logstash/map.jinja' import LOGSTASH_MERGED -%}
 status = error
 name = LogstashPropertiesConfig
 
@@ -17,14 +16,8 @@ name = LogstashPropertiesConfig
 appender.rolling.type = RollingFile
 appender.rolling.name = rolling
 appender.rolling.fileName = /var/log/logstash/logstash.log
-{%- if LOGSTASH_MERGED.config.get('log_x_format', 'plain') == 'json' %}
-appender.rolling.layout.type = JSONLayout
-appender.rolling.layout.compact = true
-appender.rolling.layout.eventEol = true
-{%- else %}
 appender.rolling.layout.type = PatternLayout
 appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %.10000m%n
-{%- endif %}
 appender.rolling.filePattern = /var/log/logstash/logstash-%d{yyyy-MM-dd}.log.gz
 appender.rolling.policies.type = Policies
 appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
@@ -39,5 +32,7 @@ appender.rolling.strategy.action.condition.type = IfFileName
 appender.rolling.strategy.action.condition.glob = *.gz
 appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified
 appender.rolling.strategy.action.condition.nested_condition.age = 7D
-rootLogger.level = ${sys:ls.log.level}
+rootLogger.level = info
 rootLogger.appenderRef.rolling.ref = rolling
+#rootLogger.level = ${sys:ls.log.level}
+#rootLogger.appenderRef.console.ref = ${sys:ls.log.format}_console

salt/logstash/etc/pipelines.yml.jinja

@@ -1,17 +1,4 @@
-{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
-{%- set PIPELINE_SETTINGS = LOGSTASH_MERGED.get('pipeline_settings', {}) %}
 {%- for assigned_pipeline in ASSIGNED_PIPELINES %}
 - pipeline.id: {{ assigned_pipeline }}
   path.config: "/usr/share/logstash/pipelines/{{ assigned_pipeline }}/"
-{%-   set extra = PIPELINE_SETTINGS.get(assigned_pipeline, {}) %}
-{%-   if extra is mapping %}
-{#-     values are emitted unquoted so yaml re-infers the type logstash expects:
-        4 as an integer, false as a boolean, 1024mb and auto as strings #}
-{%-     for key, value in extra | dictsort %}
-{%-       set rendered = key | replace('_x_', '.') %}
-{%-       if value not in ['', None] and rendered not in ['pipeline.id', 'path.config'] %}
-  {{ rendered }}: {{ value }}
-{%-       endif %}
-{%-     endfor %}
-{%-   endif %}
 {% endfor -%}

salt/logstash/soc_logstash.yaml

@@ -16,7 +16,6 @@ logstash:
       heavynode: *assigned_pipelines
       searchnode: *assigned_pipelines
       manager: *assigned_pipelines
-      managerhype: *assigned_pipelines
       managersearch: *assigned_pipelines
       fleet: *assigned_pipelines
   defined_pipelines:
@@ -35,11 +34,6 @@ logstash:
     custom2: *defined_pipelines
     custom3: *defined_pipelines
     custom4: *defined_pipelines
-    custom5: *defined_pipelines
-    custom6: *defined_pipelines
-    custom7: *defined_pipelines
-    custom8: *defined_pipelines
-    custom9: *defined_pipelines
   pipeline_config:
     custom001: &pipeline_config
       description: Pipeline configuration for Logstash
@@ -57,351 +51,6 @@ logstash:
     custom008: *pipeline_config
     custom009: *pipeline_config
     custom010: *pipeline_config
-  pipeline_settings:
-    manager: &pipeline_settings
-      pipeline_x_workers:
-        description: >-
-          Number of worker threads that run filters and outputs for this pipeline. May be set higher
-          than the CPU core count when outputs spend time waiting on I/O. Leave blank to use the value
-          from logstash.yml.
-        title: pipeline.workers
-        regex: '^$|^[1-9][0-9]*$'
-        regexFailureMessage: Must be blank, or a positive whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_batch_x_size:
-        description: >-
-          Maximum number of events an individual worker thread collects before running filters and
-          outputs. Larger batches are more efficient but increase heap use; total in-flight events is
-          workers multiplied by batch size. Leave blank to use the value from logstash.yml.
-        title: pipeline.batch.size
-        regex: '^$|^[1-9][0-9]*$'
-        regexFailureMessage: Must be blank, or a positive whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_batch_x_delay:
-        description: >-
-          Milliseconds a worker waits for the next event before running a batch that is not yet full.
-          Leave blank to use the value from logstash.yml.
-        title: pipeline.batch.delay
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_batch_x_metrics_x_sampling_mode:
-        description: >-
-          Controls how often batch size metrics are collected for this pipeline, which helps tune
-          pipeline.batch.size to the batch sizes actually being processed. Fuller sampling consumes
-          additional heap. Elastic marks this setting as a technical preview that may change in a
-          future release. Leave blank to use the value from logstash.yml.
-        title: pipeline.batch.metrics.sampling_mode
-        options:
-        - ''
-        - 'disabled'
-        - 'minimal'
-        - 'full'
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_ordered:
-        description: >-
-          Whether event order is preserved through this pipeline. auto enables ordering only when
-          pipeline.workers is explicitly set to 1, and does nothing otherwise. Setting this to true
-          requires pipeline.workers to be 1 as well; with more workers this pipeline fails to start.
-          Leave blank to use the value from logstash.yml.
-        title: pipeline.ordered
-        options:
-        - ''
-        - 'auto'
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_ecs_compatibility:
-        description: >-
-          Elastic Common Schema compatibility mode for plugins in this pipeline. Security Onion sets
-          this globally and it should rarely be changed per pipeline. Elastic considers values other
-          than disabled to be BETA, and they may produce unintended consequences when upgrading
-          Logstash. Leave blank to use the value from logstash.yml.
-        title: pipeline.ecs_compatibility
-        options:
-        - ''
-        - 'disabled'
-        - 'v1'
-        - 'v8'
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_reloadable:
-        description: >-
-          Whether this pipeline may be reloaded when its configuration changes. Leave blank to use the
-          value from logstash.yml.
-        title: pipeline.reloadable
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_type:
-        description: >-
-          Queue backing this pipeline. persisted buffers events to disk under /nsm/logstash so they
-          survive a restart, at some throughput cost; memory does not. Leave blank to use the value
-          from logstash.yml.
-        title: queue.type
-        options:
-        - ''
-        - 'memory'
-        - 'persisted'
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_max_bytes:
-        description: >-
-          Total capacity of the persistent queue for this pipeline, in bytes. Only applies when
-          queue.type is persisted. The disk backing /nsm/logstash must have room for this much data or
-          the pipeline fails to start, reporting that it was unable to allocate the space. If both
-          queue.max_events and queue.max_bytes are set, whichever is reached first applies. Leave
-          blank to use the value from logstash.yml.
-        title: queue.max_bytes
-        regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$'
-        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb, or 64k. Units are lowercase.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_page_capacity:
-        description: >-
-          Size of the individual append-only page data files that make up the persistent queue for
-          this pipeline. Only applies when queue.type is persisted. Leave blank to use the value from
-          logstash.yml.
-        title: queue.page_capacity
-        regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$'
-        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb, or 64k. Units are lowercase.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_max_events:
-        description: >-
-          Maximum number of unread events in the persistent queue for this pipeline. 0 means
-          unlimited. Only applies when queue.type is persisted. Leave blank to use the value from
-          logstash.yml.
-        title: queue.max_events
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_checkpoint_x_acks:
-        description: >-
-          Maximum number of acknowledged events before a checkpoint is forced. 0 means unlimited. Only
-          applies when queue.type is persisted. Leave blank to use the value from logstash.yml.
-        title: queue.checkpoint.acks
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_checkpoint_x_writes:
-        description: >-
-          Maximum number of written events before a checkpoint is forced. Setting this to 1 gives
-          maximum durability at a severe performance cost. 0 means unlimited. Only applies when
-          queue.type is persisted. Leave blank to use the value from logstash.yml.
-        title: queue.checkpoint.writes
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_checkpoint_x_interval:
-        description: >-
-          Milliseconds between forced checkpoints on the persistent queue head page. 0 eliminates
-          periodic checkpoints. Deprecated by Elastic as of Logstash 9.1. Only applies when queue.type
-          is persisted. Leave blank to use the value from logstash.yml.
-        title: queue.checkpoint.interval
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_checkpoint_x_retry:
-        description: >-
-          When enabled, Logstash retries four times per attempted checkpoint write that fails; later
-          errors are not retried. Elastic describes this as a workaround for failed checkpoint writes
-          seen only on Windows and on filesystems with non-standard behaviour such as SANs, and does
-          not recommend enabling it otherwise. Only applies when queue.type is persisted. Leave blank
-          to use the value from logstash.yml.
-        title: queue.checkpoint.retry
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_compression:
-        description: >-
-          Compression applied to persistent queue pages for this pipeline, trading CPU for disk: speed
-          favours the fastest operation, size the smallest files, and balanced sits between them. Once
-          compressed events have been written, that queue cannot be read by Logstash releases earlier
-          than 9.2. Only applies when queue.type is persisted. Leave blank to use the value from
-          logstash.yml.
-        title: queue.compression
-        options:
-        - ''
-        - 'none'
-        - 'speed'
-        - 'balanced'
-        - 'size'
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_drain:
-        description: >-
-          When enabled, Logstash waits for the persistent queue to drain before shutting down this
-          pipeline. Draining a large queue makes shutdown take considerably longer. Only applies when
-          queue.type is persisted. Leave blank to use the value from logstash.yml.
-        title: queue.drain
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_enable:
-        description: >-
-          Whether events this pipeline cannot process are written to a dead letter queue instead of
-          being dropped. Leave blank to use the value from logstash.yml.
-        title: dead_letter_queue.enable
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_max_bytes:
-        description: >-
-          Total capacity of the dead letter queue for this pipeline, in bytes. Only applies when
-          dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
-        title: dead_letter_queue.max_bytes
-        regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$'
-        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb, or 64k. Units are lowercase.
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_flush_interval:
-        description: >-
-          Milliseconds before an incomplete dead letter queue segment is flushed and made available to
-          the dead_letter_queue input. Lower values write more, smaller segment files; higher values
-          add latency before events can be read. Only applies when dead_letter_queue.enable is true.
-          Leave blank to use the value from logstash.yml.
-        title: dead_letter_queue.flush_interval
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_flush_check_interval:
-        description: >-
-          Milliseconds between checks for a stale dead letter queue segment needing a flush. Cannot be
-          set lower than 1000. Smaller values rotate segments sooner at the cost of CPU. Only applies
-          when dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
-        title: dead_letter_queue.flush_check_interval
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_storage_policy:
-        description: >-
-          Action taken when dead_letter_queue.max_bytes is reached: drop_newer stops accepting new
-          events, drop_older removes the oldest events to make room. Only applies when
-          dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
-        title: dead_letter_queue.storage_policy
-        options:
-        - ''
-        - 'drop_newer'
-        - 'drop_older'
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_retain_x_age:
-        description: >-
-          How long an event is kept in the dead letter queue before Logstash removes it, such as 5d.
-          Units are d, h, m and s; there is no default unit, so one must be given. Only applies when
-          dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
-        title: dead_letter_queue.retain.age
-        regex: '^$|^[0-9]+\s*[dhms]$'
-        regexFailureMessage: Must be blank, or a number followed by d, h, m, or s, such as 5d.
-        advanced: True
-        global: False
-        helpLink: logstash
-      path_x_queue:
-        description: >-
-          Directory inside the Logstash container holding the persistent queue for this pipeline. The
-          default lives under the /nsm/logstash bind mount; a path outside it will not survive a
-          container restart. Logstash creates the directory if it is missing, requires it to be
-          writable, and refuses to start if the path is a symlink. Only applies when queue.type is
-          persisted. Leave blank to use the value from logstash.yml.
-        title: path.queue
-        advanced: True
-        global: False
-        helpLink: logstash
-      path_x_dead_letter_queue:
-        description: >-
-          Directory inside the Logstash container holding the dead letter queue for this pipeline. The
-          default lives under the /nsm/logstash bind mount; a path outside it will not survive a
-          container restart. Logstash creates the directory if it is missing, requires it to be
-          writable, and refuses to start if the path is a symlink. Only applies when
-          dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
-        title: path.dead_letter_queue
-        advanced: True
-        global: False
-        helpLink: logstash
-      config_x_debug:
-        description: >-
-          Whether the fully compiled configuration for this pipeline is written to the log. The output
-          may contain sensitive values from the pipeline configuration. Leave blank to use the value
-          from logstash.yml.
-        title: config.debug
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      config_x_support_escapes:
-        description: >-
-          Whether escape sequences such as \n and \t in this pipeline's quoted strings are
-          interpreted. Leave blank to use the value from logstash.yml.
-        title: config.support_escapes
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-    fleet: *pipeline_settings
-    receiver: *pipeline_settings
-    search: *pipeline_settings
-    custom0: *pipeline_settings
-    custom1: *pipeline_settings
-    custom2: *pipeline_settings
-    custom3: *pipeline_settings
-    custom4: *pipeline_settings
-    custom5: *pipeline_settings
-    custom6: *pipeline_settings
-    custom7: *pipeline_settings
-    custom8: *pipeline_settings
-    custom9: *pipeline_settings
   settings:
     lsheap: 
       description: Heap size to use for logstash
@@ -413,35 +62,6 @@ logstash:
       helpLink: logstash
       readonly: True
       advanced: True
-    log_x_level:
-      description: >-
-        Verbosity of the Logstash log at /opt/so/log/logstash/logstash.log. debug and trace produce
-        a very large volume of log data on a busy node and should be used only while troubleshooting;
-        the log rotates at 1GB and rotated files are deleted after 7 days. Setting this to debug is
-        also what makes the per-pipeline config.debug setting emit anything.
-      title: log.level
-      options:
-      - 'fatal'
-      - 'error'
-      - 'warn'
-      - 'info'
-      - 'debug'
-      - 'trace'
-      advanced: True
-      global: False
-      helpLink: logstash
-    log_x_format:
-      description: >-
-        Layout of the Logstash log. plain writes human readable lines; json writes one JSON object
-        per line, which is easier to parse but harder to read directly. The file name and location
-        do not change.
-      title: log.format
-      options:
-      - 'plain'
-      - 'json'
-      advanced: True
-      global: False
-      helpLink: logstash
     path_x_logs: 
       description: Path inside the container to wrote logs.
       helpLink: logstash

salt/manager/files/beacons_pushstate.conf.jinja

@@ -3,16 +3,9 @@ beacons:
   postgres_pillar_beacon:
     - interval: {{ AUTOAPPLY.drain_interval }}
     - disable_during_state_run: False
-  local_files_beacon:
+  rules_beacon:
     - interval: {{ AUTOAPPLY.drain_interval }}
     - disable_during_state_run: False
-    # Tags are app names in salt/reactor/pillar_push_map.yaml.
-    # Allowlist on purpose: salt writes elsewhere under local/salt/ and would self-retrigger.
     - paths:
         /opt/so/saltstack/local/salt/suricata/rules: suricata
         /opt/so/saltstack/local/salt/strelka/rules/compiled: strelka
-        /opt/so/saltstack/local/salt/zeek/policy: zeek
-        /opt/so/saltstack/local/salt/zeek/zkg: zeek
-        /opt/so/saltstack/local/salt/elasticsearch/files/ingest: elasticsearch
-        /opt/so/saltstack/local/salt/elasticsearch/roles: elasticsearch
-        /opt/so/saltstack/local/salt/logstash/pipelines/config/custom: logstash

salt/manager/init.sls

@@ -113,8 +113,8 @@ manager_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://manager/tools/sbin
-    - user: root
+    - user: 939
-    - group: root
+    - group: 939
     - file_mode: 755
     - exclude_pat:
       - "*_test.py"
@@ -124,8 +124,8 @@ manager_sbin_jinja:
   file.recurse:
     - name: /usr/sbin/
     - source: salt://manager/tools/sbin_jinja/
-    - user: root
+    - user: socore
-    - group: root
+    - group: socore
     - file_mode: 755
     - template: jinja
     - show_changes: False
@@ -166,7 +166,7 @@ so-repo-sync:
 
 so_fleetagent_status:
   cron.present:
-    - name: '/usr/sbin/so-elasticagent-status > /opt/so/log/agents/agentstatus.log.tmp 2>&1; mv -f /opt/so/log/agents/agentstatus.log.tmp /opt/so/log/agents/agentstatus.log'
+    - name: /usr/sbin/so-elasticagent-status > /opt/so/log/agents/agentstatus.log 2>&1
     - identifier: so_fleetagent_status
     - user: root
     - minute: '*/5'
@@ -190,15 +190,11 @@ so_fleetagent_monitor:
   - month: '*'
   - dayweek: '*'
 
-# This tree is the source of every root-executed script (/usr/sbin, reactors, _runners,
+socore_own_saltstack_default:
-# engines, salt-relay.sh). SOC mounts /opt/so/saltstack rw as uid 939 but only writes
-# under local/. Do not add dir_mode/file_mode here -- SOC reads default/ and 750/640
-# would break its config load.
-root_own_saltstack_default:
   file.directory:
     - name: /opt/so/saltstack/default
-    - user: root
+    - user: socore
-    - group: root
+    - group: socore
     - recurse:
       - user
       - group

salt/manager/tools/sbin/so-client

@@ -106,8 +106,7 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
-hydraContainer=${HYDRA_CONTAINER:-so-hydra}
+hydraUrl=${HYDRA_URL:-http://127.0.0.1:4445}
-hydraUrl=${HYDRA_URL:-http://localhost:4445}
 socRolesFile=${SOC_ROLES_FILE:-/opt/so/conf/soc/soc_clients_roles}
 soUID=${SOCORE_UID:-939}
 soGID=${SOCORE_GID:-939}
@@ -125,10 +124,6 @@ function fail() {
   exit 1
 }
 
-function hydraCurl() {
-  docker exec "$hydraContainer" curl "$@"
-}
-
 function require() {
   cmd=$1
   which "$1" 2>&1 > /dev/null
@@ -138,8 +133,8 @@ function require() {
 # Verify this environment is capable of running this script
 function verifyEnvironment() {
   require "jq"
-  require "docker"
+  require "curl"
-  response=$(hydraCurl -Ss -L ${hydraUrl}/health/alive)
+  response=$(curl -Ss -L ${hydraUrl}/health/alive)
   [[ "$response" != '{"status":"ok"}' ]] && fail "Unable to communicate with Hydra; specify URL via HYDRA_URL environment variable"
 }
 
@@ -169,7 +164,7 @@ function ensureRoleFileExists() {
 }
 
 function listClients() {
-  response=$(hydraCurl -Ss -L -f ${hydraUrl}/admin/clients)
+  response=$(curl -Ss -L -f ${hydraUrl}/admin/clients)
   [[ $? != 0 ]] && fail "Unable to communicate with Hydra"
 
   clientIds=$(echo "${response}" | jq -r ".[] | .client_id" | sort)
@@ -256,7 +251,7 @@ function createClient() {
 EOF
   )
   
-  response=$(hydraCurl -Ss -L --fail-with-body -X POST ${hydraUrl}/admin/clients -d "$body")
+  response=$(curl -Ss -L --fail-with-body -X POST ${hydraUrl}/admin/clients -d "$body")
   if [[ $? != 0 ]]; then
     error=$(echo $response | jq .error)
     fail "Failed to submit request to Hydra: $error"
@@ -288,7 +283,7 @@ function update() {
 EOF
   )
   
-  response=$(hydraCurl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body")
+  response=$(curl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body")
   if [[ $? != 0 ]]; then
     error=$(echo $response | jq .error)
     fail "Failed to submit request to Hydra: $error"
@@ -310,7 +305,7 @@ function generateSecret() {
 EOF
   )
   
-  response=$(hydraCurl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body")
+  response=$(curl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body")
   if [[ $? != 0 ]]; then
     error=$(echo $response | jq .error)
     fail "Failed to submit request to Hydra: $error"
@@ -322,7 +317,7 @@ function deleteClient() {
 
   [[ ${identityId} == "" ]] && fail "Client not found"
 
-  response=$(hydraCurl -Ss -XDELETE -L --fail-with-body "${hydraUrl}/admin/clients/$identityId")
+  response=$(curl -Ss -XDELETE -L --fail-with-body "${hydraUrl}/admin/clients/$identityId")
   if [[ $? != 0 ]]; then
     error=$(echo $response | jq .error)
     fail "Failed to submit request to Hydra: $error"

salt/manager/tools/sbin/so-minion

@@ -121,14 +121,8 @@ for i in "$@"; do
 	esac
 done
 
-if [[ -n "$MINION_ID" && ! "$MINION_ID" =~ ^[A-Za-z0-9._-]{1,253}$ ]]; then
+PILLARFILE=/opt/so/saltstack/local/pillar/minions/$MINION_ID.sls
-	echo "Invalid minion id: $MINION_ID"
+ADVPILLARFILE=/opt/so/saltstack/local/pillar/minions/adv_$MINION_ID.sls
-	log "ERROR" "Invalid minion id: $MINION_ID"
-	exit 1
-fi
-
-readonly PILLARFILE=/opt/so/saltstack/local/pillar/minions/$MINION_ID.sls
-readonly ADVPILLARFILE=/opt/so/saltstack/local/pillar/minions/adv_$MINION_ID.sls
 
 function getinstallinfo() {
 	log "INFO" "Getting install info for minion $MINION_ID"
@@ -139,23 +133,10 @@ function getinstallinfo() {
 		return 1
 	fi
 
-	# install.txt is controlled by the minion; only accept known keys and never eval or export them
+	while read -r var; do export "$var"; done <<< "$INSTALLVARS"
-	local line key
+	if [ $? -ne 0 ]; then
-	while IFS= read -r line; do
+		log "ERROR" "Failed to source install variables"
-		[[ "$line" == *=* ]] || continue
+		return 1
-		key=${line%%=*}
-		case "$key" in
-			MAINIP|MNIC|NODE_DESCRIPTION|ES_HEAP_SIZE|PATCHSCHEDULENAME|INTERFACE|NODETYPE|CORECOUNT|LSHOSTNAME|LSHEAP|CPUCORES|IDH_MGTRESTRICT|IDH_SERVICES)
-				printf -v "$key" '%s' "${line#*=}"
-				;;
-			*)
-				log "WARN" "Ignoring unexpected install var from $MINION_ID: ${key:0:64}"
-				;;
-		esac
-	done <<< "$INSTALLVARS"
-
-	if [[ "$NODE_DESCRIPTION" == \'*\' ]]; then
-		NODE_DESCRIPTION=${NODE_DESCRIPTION:1:-1}
 	fi
 
 	log "INFO" "Fetched install info for $MINION_ID (node type: ${NODETYPE:-unset})"
@@ -195,12 +176,6 @@ function pcapspace() {
 		fi
 	fi
 
-	# Must be checked before arithmetic expansion, which evaluates array subscripts
-	if [[ ! "$SPACESIZE" =~ ^[0-9]+$ ]]; then
-		log "ERROR" "Invalid disk size for $MINION_ID: ${SPACESIZE:0:64}"
-		return 1
-	fi
-
 	local s=$(( $SPACESIZE / 1000000 ))
 	local s1=$(( $s / 4 * $PCAP_PERCENTAGE ))
 
@@ -1075,57 +1050,6 @@ function updateMineAndApplyStates() {
 	fi
 }
 
-# Values end up in a Jinja-rendered pillar and in bash, and may come from the minion
-function validate_minion_vars() {
-	local error_msg=""
-	# Inline rather than valid_ip4: so-common is not installed yet when setup runs -o=setup
-	local octet='(25[0-5]|2[0-4][0-9]|1?[0-9]?[0-9])'
-	local ip4_re="^($octet\.){3}$octet$"
-
-	case "$NODETYPE" in
-		EVAL|STANDALONE|MANAGER|MANAGERSEARCH|MANAGERHYPE|IMPORT)
-			# Manager pillars also rewrite the CA pillar, so never accept them from a remote node
-			[[ "$OPERATION" == "setup" ]] || error_msg="Node type $NODETYPE can only be configured during setup"
-			;;
-		FLEET|IDH|HEAVYNODE|SENSOR|SEARCHNODE|RECEIVER|HYPERVISOR|DESKTOP)
-			;;
-		*)
-			error_msg="Invalid node type: ${NODETYPE:0:64}"
-			;;
-	esac
-
-	if [[ -z "$error_msg" ]]; then
-		if [[ ! "$MAINIP" =~ $ip4_re ]]; then
-			error_msg="Invalid MAINIP: ${MAINIP:0:64}"
-		elif [[ ! "$MNIC" =~ ^[A-Za-z0-9._-]*$ ]]; then
-			error_msg="Invalid MNIC: ${MNIC:0:64}"
-		elif [[ ! "$INTERFACE" =~ ^[A-Za-z0-9._-]*$ ]]; then
-			error_msg="Invalid INTERFACE: ${INTERFACE:0:64}"
-		elif [[ ! "$LSHOSTNAME" =~ ^[A-Za-z0-9._-]*$ ]]; then
-			error_msg="Invalid LSHOSTNAME: ${LSHOSTNAME:0:64}"
-		elif [[ ! "$ES_HEAP_SIZE" =~ ^([0-9]+[kKmMgG]?)?$ ]]; then
-			error_msg="Invalid ES_HEAP_SIZE: ${ES_HEAP_SIZE:0:64}"
-		elif [[ ! "$LSHEAP" =~ ^([0-9]+[kKmMgG]?)?$ ]]; then
-			error_msg="Invalid LSHEAP: ${LSHEAP:0:64}"
-		elif [[ ! "$CORECOUNT" =~ ^[0-9]*$ ]]; then
-			error_msg="Invalid CORECOUNT: ${CORECOUNT:0:64}"
-		elif [[ ! "$CPUCORES" =~ ^[0-9]*$ ]]; then
-			error_msg="Invalid CPUCORES: ${CPUCORES:0:64}"
-		elif [[ ! "$IDH_MGTRESTRICT" =~ ^(True|False)?$ ]]; then
-			error_msg="Invalid IDH_MGTRESTRICT: ${IDH_MGTRESTRICT:0:64}"
-		fi
-	fi
-
-	if [[ -n "$error_msg" ]]; then
-		log "ERROR" "$error_msg"
-		echo "$error_msg"
-		return 1
-	fi
-
-	# Free text; removing braces is enough to prevent any Jinja delimiter
-	NODE_DESCRIPTION=${NODE_DESCRIPTION//[\{\}[:cntrl:]]/}
-}
-
 function setupMinionFiles() {
 	log "INFO" "Setting up minion files for $MINION_ID (pillar: $PILLARFILE)"
 	
@@ -1137,8 +1061,6 @@ function setupMinionFiles() {
 		return 1
 	fi
 
-	validate_minion_vars || return 1
-
 	# Create the base minion files
 	create_minion_files || return 1
 

salt/manager/tools/sbin/so-push-drainer

@@ -20,7 +20,7 @@ is older than debounce_seconds, this script:
     with the deduped actions list passed as pillar kwargs
   * deletes the contributed intent files on successful dispatch
 
-Reactor sls files (push_files, push_pillar) write intents
+Reactor sls files (push_suricata, push_strelka, push_pillar) write intents
 but never dispatch directly
 """
 

salt/manager/tools/sbin/so-saltstack-update

@@ -124,8 +124,8 @@ copy_new_files() {
   
   rsync -a salt $default_salt_dir/
   rsync -a pillar $default_salt_dir/
-  chown -R root:root $default_salt_dir/salt
+  chown -R socore:socore $default_salt_dir/salt
-  chown -R root:root $default_salt_dir/pillar
+  chown -R socore:socore $default_salt_dir/pillar
   chmod 755 $default_salt_dir/pillar/firewall/addfirewall.sh
 
   rm -rf /tmp/sogh

salt/manager/tools/sbin/so-user

@@ -129,8 +129,7 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
-kratosContainer=${KRATOS_CONTAINER:-so-kratos}
+kratosUrl=${KRATOS_URL:-http://127.0.0.1:4434/admin}
-kratosUrl=${KRATOS_URL:-http://localhost:4434/admin}
 databasePath=${KRATOS_DB_PATH:-/nsm/kratos/db/db.sqlite}
 databaseTimeout=${KRATOS_DB_TIMEOUT:-5000}
 bcryptRounds=${BCRYPT_ROUNDS:-12}
@@ -155,10 +154,6 @@ function fail() {
   exit 1
 }
 
-function kratosCurl() {
-  docker exec "$kratosContainer" curl "$@"
-}
-
 function require() {
   cmd=$1
   which "$1" 2>&1 > /dev/null
@@ -169,18 +164,18 @@ function require() {
 function verifyEnvironment() {
   require "htpasswd"
   require "jq"
-  require "docker"
+  require "curl"
   require "openssl"
   require "sqlite3"
   [[ ! -f $databasePath ]] && fail "Unable to find database file; specify path via KRATOS_DB_PATH environment variable"
-  response=$(kratosCurl -Ss -L ${kratosUrl}/)
+  response=$(curl -Ss -L ${kratosUrl}/)
   [[ "$response" != "404 page not found" ]] && fail "Unable to communicate with Kratos; specify URL via KRATOS_URL environment variable"
 }
 
 function findIdByEmail() {
   email=${1,,}
 
-  response=$(kratosCurl -Ss -L ${kratosUrl}/identities)
+  response=$(curl -Ss -L ${kratosUrl}/identities)
   identityId=$(echo "${response}" | jq -r ".[] | select(.verifiable_addresses[0].value == \"$email\") | .id")
   echo $identityId
 }
@@ -421,7 +416,7 @@ function syncAll() {
 }
 
 function listUsers() {
-  response=$(kratosCurl -Ss -L ${kratosUrl}/identities)
+  response=$(curl -Ss -L ${kratosUrl}/identities)
   [[ $? != 0 ]] && fail "Unable to communicate with Kratos"
 
   users=$(echo "${response}" | jq -r ".[] | .verifiable_addresses[0].value" | sort)
@@ -500,7 +495,7 @@ function createUser() {
 EOF
   )
   
-  response=$(kratosCurl -Ss -L ${kratosUrl}/identities -d "$addUserJson")
+  response=$(curl -Ss -L ${kratosUrl}/identities -d "$addUserJson")
   [[ $? != 0 ]] && fail "Unable to communicate with Kratos"
 
   identityId=$(echo "${response}" | jq -r ".id")
@@ -523,7 +518,7 @@ function updateStatus() {
   identityId=$(findIdByEmail "$email")
   [[ ${identityId} == "" ]] && fail "User not found"
 
-  response=$(kratosCurl -Ss -L "${kratosUrl}/identities/$identityId")
+  response=$(curl -Ss -L "${kratosUrl}/identities/$identityId")
   [[ $? != 0 ]] && fail "Unable to communicate with Kratos"
 
   schemaId=$(echo "$response" | jq -r .schema_id)
@@ -536,7 +531,7 @@ function updateStatus() {
     state="inactive"
   fi
   body="{ \"schema_id\": \"$schemaId\", \"state\": \"$state\", \"traits\": $traitBlock }"
-  response=$(kratosCurl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body")
+  response=$(curl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body")
   [[ $? != 0 ]] && fail "Unable to update user"
 }
 
@@ -555,7 +550,7 @@ function updateUserProfile() {
   identityId=$(findIdByEmail "$email")
   [[ ${identityId} == "" ]] && fail "User not found"
 
-  response=$(kratosCurl -Ss -L "${kratosUrl}/identities/$identityId")
+  response=$(curl -Ss -L "${kratosUrl}/identities/$identityId")
   [[ $? != 0 ]] && fail "Unable to communicate with Kratos"
 
   schemaId=$(echo "$response" | jq -r .schema_id)
@@ -564,7 +559,7 @@ function updateUserProfile() {
   traitBlock="{\"email\":\"$email\",\"firstName\":\"$firstName\",\"lastName\":\"$lastName\",\"note\":\"$note\"}"
 
   body="{ \"schema_id\": \"$schemaId\", \"state\": \"$state\", \"traits\": $traitBlock }"
-  response=$(kratosCurl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body")
+  response=$(curl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body")
   [[ $? != 0 ]] && fail "Unable to update user"
 }
 
@@ -574,7 +569,7 @@ function deleteUser() {
   identityId=$(findIdByEmail "$email")
   [[ ${identityId} == "" ]] && fail "User not found"
 
-  response=$(kratosCurl -Ss -XDELETE -L "${kratosUrl}/identities/$identityId")
+  response=$(curl -Ss -XDELETE -L "${kratosUrl}/identities/$identityId")
   [[ $? != 0 ]] && fail "Unable to communicate with Kratos"
 
   rolesTmpFile="${socRolesFile}.tmp"

salt/manager/tools/sbin/so-yaml.py

@@ -42,8 +42,7 @@ def loadYaml(filename):
     try:
         with open(filename, "r") as file:
             content = file.read()
-            loaded = yaml.safe_load(content)
+            return yaml.safe_load(content)
-            return loaded if loaded is not None else {}
     except FileNotFoundError:
         print(f"File not found: {filename}", file=sys.stderr)
         sys.exit(1)

salt/manager/tools/sbin/so-yaml_test.py

@@ -95,20 +95,6 @@ class TestRemove(unittest.TestCase):
         expected = "key1:\n  child1: 123\n  child2:\n    deep2: ab\nkey2: false\n"
         self.assertEqual(actual, expected)
 
-    def test_remove_empty_file(self):
-        filename = "/tmp/so-yaml_test-remove-empty.yaml"
-        file = open(filename, "w")
-        file.close()
-
-        code = soyaml.remove([filename, "key1"])
-        self.assertEqual(code, 0)
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        self.assertEqual(actual, "{}\n")
-
     def test_remove_missing_args(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
             with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -308,36 +294,6 @@ class TestRemove(unittest.TestCase):
         expected = "key1:\n  child1: 123\n  child2:\n    deep1: 45\n    deep2: d\nkey2: false\nkey3:\n- e\n- f\n- g\n"
         self.assertEqual(actual, expected)
 
-    def test_add_empty_file(self):
-        filename = "/tmp/so-yaml_test-add-empty.yaml"
-        file = open(filename, "w")
-        file.close()
-
-        code = soyaml.add([filename, "telegraf.output", "BOTH"])
-        self.assertEqual(code, 0)
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "telegraf:\n  output: BOTH\n"
-        self.assertEqual(actual, expected)
-
-    def test_add_empty_file_simple(self):
-        filename = "/tmp/so-yaml_test-add-empty-simple.yaml"
-        file = open(filename, "w")
-        file.close()
-
-        code = soyaml.add([filename, "telegraf", "BOTH"])
-        self.assertEqual(code, 0)
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "telegraf: BOTH\n"
-        self.assertEqual(actual, expected)
-
     def test_replace_missing_arg(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
             with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -390,21 +346,6 @@ class TestRemove(unittest.TestCase):
         expected = "key1:\n  child1: 123\n  child2:\n    deep1: 46\nkey2: false\nkey3:\n- e\n- f\n- g\n"
         self.assertEqual(actual, expected)
 
-    def test_replace_empty_file(self):
-        filename = "/tmp/so-yaml_test-replace-empty.yaml"
-        file = open(filename, "w")
-        file.close()
-
-        code = soyaml.replace([filename, "telegraf.output", "BOTH"])
-        self.assertEqual(code, 0)
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "telegraf:\n  output: BOTH\n"
-        self.assertEqual(actual, expected)
-
     def test_convert(self):
         self.assertEqual(soyaml.convertType("foo"), "foo")
         self.assertEqual(soyaml.convertType("foo.bar"), "foo.bar")
@@ -565,18 +506,6 @@ class TestRemove(unittest.TestCase):
             self.assertEqual(result, 2)
             self.assertEqual("", mock_stdout.getvalue())
 
-    def test_get_empty_file(self):
-        with patch('sys.stdout', new=StringIO()) as mock_stdout:
-            with patch('sys.stderr', new=StringIO()) as mock_stderr:
-                filename = "/tmp/so-yaml_test-get-empty.yaml"
-                file = open(filename, "w")
-                file.close()
-
-                result = soyaml.get([filename, "telegraf.output"])
-                self.assertEqual(result, 2)
-                self.assertEqual("", mock_stdout.getvalue())
-                self.assertIn("Key 'telegraf.output' not found by so-yaml.py", mock_stderr.getvalue())
-
     def test_get_usage(self):
         with patch('sys.exit', new=MagicMock()) as sysmock:
             with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -1062,29 +991,3 @@ class TestLoadYaml(unittest.TestCase):
                     soyaml.loadYaml("/tmp/so-yaml_test-unreadable.yaml")
                     sysmock.assert_called_with(1)
                     self.assertIn("Error reading file", mock_stderr.getvalue())
-
-    def test_load_yaml_empty_file(self):
-        filename = "/tmp/so-yaml_test-load-empty.yaml"
-        file = open(filename, "w")
-        file.close()
-
-        result = soyaml.loadYaml(filename)
-        self.assertEqual(result, {})
-
-    def test_load_yaml_whitespace_only(self):
-        filename = "/tmp/so-yaml_test-load-whitespace.yaml"
-        file = open(filename, "w")
-        file.write("   \n\n  \n")
-        file.close()
-
-        result = soyaml.loadYaml(filename)
-        self.assertEqual(result, {})
-
-    def test_load_yaml_comments_only(self):
-        filename = "/tmp/so-yaml_test-load-comments.yaml"
-        file = open(filename, "w")
-        file.write("# Just a comment\n# Another comment\n")
-        file.close()
-
-        result = soyaml.loadYaml(filename)
-        self.assertEqual(result, {})

salt/manager/tools/sbin/soup

@@ -28,7 +28,6 @@ INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
 # percentage like "25%"). Empty means so-soup-grid-highstate uses the salt:auto_apply:batch
 # pillar default.
 BATCHSIZE=
-DEFAULT_DOCKER_RANGE='172.17.1.0/24'
 SOUP_LOG=/root/soup.log
 SOUP_DEBUG_LOG=/root/soup-debug.log
 WHATWOULDYOUSAYYAHDOHERE=soup
@@ -121,9 +120,6 @@ check_err() {
       161)
         echo 'Required intermediate Elasticsearch upgrade not complete'
       ;;
-      162)
-        echo 'One or more Elastic Agent nodes do not support the x86-64-v3 CPU instruction set'
-      ;;
       170)
         echo "Intermediate upgrade completed successfully to $next_step_so_version, but next soup to Security Onion $originally_requested_so_version could not be started automatically."
         echo "Start soup again manually to continue the upgrade to Security Onion $originally_requested_so_version."
@@ -335,96 +331,14 @@ check_pillar_items() {
 }
 
 check_cluster_health() {
-  # Require a 'green' cluster before upgrading
   echo "Checking Elasticsearch cluster health."
+  # Require a 'green' cluster before upgrading; anything less (yellow, red, or
+  # unreachable) blocks. Modeled on the wait used in so-elasticsearch-roles-load.
   if so-elasticsearch-query "_cluster/health?wait_for_status=green&timeout=120s" --fail > /dev/null 2>&1; then
     printf "\nThe Elasticsearch cluster is healthy (green). We can proceed with SOUP.\n\n"
-    return
-  fi
-
-  if command -v so-elasticsearch-troubleshoot > /dev/null 2>&1; then
-    printf "\nRunning so-elasticsearch-troubleshoot for additional detail.\n"
-    so-elasticsearch-troubleshoot || true
-  fi
-
-  printf "\nThe Elasticsearch cluster is not green. Resolve the cluster health issue so the cluster is green before running SOUP again.\n\n"
-  exit 0
-}
-
-no_soup_for_you() {
-  echo ""
-  echo "No soup for you!"
-  exit 162
-}
-
-check_cpu_compatibility() {
-  # Roles running a container built from the so-elastic-agent image; mirrors the
-  # elasticagent and elasticfleet entries in salt/reactor/pillar_push_map.yaml.
-  local cpu_target='G@role:so-heavynode or G@role:so-eval or G@role:so-fleet or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone'
-  local expected_nodes cpu_results node result confirm
-  local -a unsupported=() offline=()
-
-  echo "Checking that Elastic Agent nodes support the x86-64-v3 CPU instruction set now required by Elastic."
-
-  if [[ "$SKIP_CPU_CHECK" == "true" ]]; then
-    printf "\nSkipping the x86-64-v3 CPU check because --skip-cpu-check was specified.\n\n"
-    return
-  fi
-
-  # Nodes that never answer are absent from the results, so diff against who should have.
-  expected_nodes=$(salt -C "$cpu_target" --preview-target --out=json 2>/dev/null | jq -r '.[]?') || true
-  if [[ -z "$expected_nodes" ]]; then
-    printf "\nCould not determine which nodes run the Elastic Agent, so the x86-64-v3 CPU check cannot run.\n"
-    no_soup_for_you
-  fi
-
-  cpu_results=$(salt -t 30 -C "$cpu_target" cmd.run "/lib64/ld-linux-x86-64.so.2 --help | grep x86-64-v3" --out=json 2>/dev/null) || true
-
-  while IFS= read -r node; do
-    [[ -z "$node" ]] && continue
-    result=$(jq -r --arg node "$node" '.[$node] // empty' <<< "$cpu_results" 2>/dev/null)
-    if [[ -z "$result" || "$result" == *"did not return"* ]]; then
-      offline+=("$node")
-    elif [[ "$result" != *"x86-64-v3 (supported"* ]]; then
-      # glibc appends "(supported, searched)" only when supported; the open paren keeps
-      # this from matching a future "(unsupported".
-      unsupported+=("$node")
-    fi
-  done <<< "$expected_nodes"
-
-  if [[ ${#unsupported[@]} -eq 0 && ${#offline[@]} -eq 0 ]]; then
-    printf "\nAll Elastic Agent nodes support x86-64-v3. We can proceed with SOUP.\n\n"
-    return
-  fi
-
-  echo ""
-  if [[ ${#unsupported[@]} -gt 0 ]]; then
-    echo "The following node(s) do NOT support the x86-64-v3 CPU instruction set:"
-    printf '  %s\n' "${unsupported[@]}"
-    echo ""
-    echo "Upstream Elastic now builds its binaries for x86-64-v3, so these nodes can no"
-    echo "longer run Elastic. Upgrading them WILL BREAK them."
-    echo ""
-  fi
-  if [[ ${#offline[@]} -gt 0 ]]; then
-    echo "The following node(s) did not respond and could not be checked:"
-    printf '  %s\n' "${offline[@]}"
-    echo ""
-    echo "These nodes are offline, so we cannot confirm they support x86-64-v3, which"
-    echo "upstream Elastic now requires."
-    echo ""
-  fi
-
-  if [[ -n $UNATTENDED ]]; then
-    echo "Unattended mode cannot prompt for an override. Re-run soup interactively, or pass --skip-cpu-check to bypass this check."
-    no_soup_for_you
-  fi
-
-  read -rp "Type 'override' to continue anyway, or press Enter to exit: " confirm
-  if [[ "${confirm,,}" == "override" ]]; then
-    printf "\nOverride accepted. Continuing at your own risk.\n\n"
   else
-    no_soup_for_you
+    printf "\nThe Elasticsearch cluster is not green. Please resolve the cluster health issue so the cluster is green before running SOUP again.\n\n"
+    exit 0
   fi
 }
 
@@ -545,13 +459,13 @@ upgrade_searchnode_elasticsearch() {
   # Run the elasticsearch state across the true elastic cluster (non-heavy) with a retry attempt
   #   Excludes the manager, so that kibana & elasticfleet are not upgraded until searchnodes are upgraded.
   echo "Getting ready to upgrade Elasticsearch across the grid. This may take a while..."
-  if salt -b 10% -C "I@elasticsearch:enabled and not G@role:so-heavynode and not G@id:${MINIONID}" state.apply elasticsearch queue=True; then
+  if salt -C "I@elasticsearch:enabled and not G@role:so-heavynode and not G@id:${MINIONID}" state.apply elasticsearch queue=True batch=10%; then
     return 0
   fi
   
   echo "Initial elasticsearch state attempt had a problem; retrying in 30 seconds."
   sleep 30
-  salt -b 10% -C "I@elasticsearch:enabled and not G@role:so-heavynode and not G@id:${MINIONID}" state.apply elasticsearch queue=True
+  salt -C "I@elasticsearch:enabled and not G@role:so-heavynode and not G@id:${MINIONID}" state.apply elasticsearch queue=True batch=10%
 }
 
 push_grid_highstate() {
@@ -606,7 +520,6 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == "3.0.0" ]] && up_to_3.1.0
     [[ "$INSTALLEDVERSION" == "3.1.0" ]] && up_to_3.2.0
     [[ "$INSTALLEDVERSION" == "3.2.0" ]] && up_to_3.3.0
-    [[ "$INSTALLEDVERSION" == "3.3.0" ]] && up_to_3.4.0
     true
 }
 
@@ -625,7 +538,6 @@ postupgrade_changes() {
     [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0
     [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0
     [[ "$POSTVERSION" == "3.2.0" ]] && post_to_3.3.0
-    [[ "$POSTVERSION" == "3.3.0" ]] && post_to_3.4.0
     # All applicable post-upgrade steps completed; clear the resume marker.
     rm -f "$POSTVERSION_FILE"
     true
@@ -1103,6 +1015,9 @@ update_kafka_metadata() {
 up_to_3.2.0() {
   ensure_salt_local_pillar
 
+  # download 9.3.7 elastic agent packages
+  determine_elastic_agent_upgrade
+
   fix_logstash_0013_lumberjack_pipeline_name
 
   pin_elasticsearch_data_retention_method
@@ -1121,6 +1036,10 @@ post_to_3.2.0() {
   # SOC database bootstrap is handled by the postgres.enabled highstate.
   scrub_postgres_log_passwords
 
+  # Generate 9.3.7 elastic agent installers
+  echo "Regenerating Elastic Agent Installers"
+  /sbin/so-elastic-agent-gen-installers
+
   kibana_backport_streams_index_template
 
   update_kafka_metadata "4.3"
@@ -1129,17 +1048,8 @@ post_to_3.2.0() {
 }
 ### 3.2.0 End ###
 
-### 3.3.0 Scripts ###
+### 3.2.0 Scripts ###
 up_to_3.3.0() {
-  # download 9.4.5 elastic agent packages
-  determine_elastic_agent_upgrade
-  
-  # remove existing (patched) elasticsearch index template to match integration naming change
-  if ! remove_elasticsearch_index_template "so-logs-sentinel_one_cloud_funnel.login" "sentinel_one_cloud_funnel.login changed to sentinel_one_cloud_funnel.logins"; then
-    FINAL_MESSAGE_QUEUE+=("WARNING: Unable to automatically remove the so-logs-sentinel_one_cloud_funnel.login index template. This step can be performed manually using the following command:")
-    FINAL_MESSAGE_QUEUE+=("    - sudo so-elasticsearch-query _index_template/so-logs-sentinel_one_cloud_funnel.login -XDELETE && so-checkin")
-  fi
-
   INSTALLEDVERSION=3.3.0
 }
 
@@ -1166,103 +1076,10 @@ post_to_3.3.0() {
   # Recollate again since some internal DBs were excluded during 3.2.0 soup
   recollate_postgres
 
-  # Generate 9.4.5 elastic agent installers
-  echo "Regenerating Elastic Agent Installers"
-  /sbin/so-elastic-agent-gen-installers
-
   telegraf_repair
-
-  set_postversion 3.3.0
 }
 ### 3.3.0 End ###
 
-### 3.4.0 Scripts ###
-up_to_3.4.0() {
-  set_soauth_range
-
-  echo "Removing so-kratos, so-hydra and so-soc so they are recreated on the soauth network."
-  docker rm -f so-kratos so-hydra so-soc >> $SOUP_LOG 2>&1
-
-  for template in so-metrics-logstash.node so-metrics-logstash.stack_monitoring.node; do
-    if ! remove_elasticsearch_index_template "$template" "logstash node and node_cel index patterns reversed"; then
-      FINAL_MESSAGE_QUEUE+=("WARNING: Unable to automatically remove the $template index template. Addon integration templates may fail to load until it is removed:")
-      FINAL_MESSAGE_QUEUE+=("    - sudo so-elasticsearch-query _index_template/$template -XDELETE && so-checkin")
-    fi
-  done
-
-  INSTALLEDVERSION=3.4.0
-}
-
-set_soauth_range() {
-  local pillar_file=/opt/so/saltstack/local/pillar/docker/soc_docker.sls
-  local current_range suggested authnet authgw input
-
-  [[ -f "$pillar_file" ]] || return 0
-
-  current_range=$(so-yaml.py get -r "$pillar_file" docker.range 2>/dev/null) || return 0
-
-  # A default range gets the 172.17.2.0/24 from docker/defaults.yaml, same as a fresh
-  # install, so there is nothing to ask about.
-  [[ -n "$current_range" && "$current_range" != "$DEFAULT_DOCKER_RANGE" ]] || return 0
-
-  if so-yaml.py get -r "$pillar_file" docker.networks.soauth.range >/dev/null 2>&1; then
-    return 0
-  fi
-
-  suggested=$(echo "${current_range%%/*}" | awk -F'.' '{ printf "%s.%s.%s.%s", $1, $2, ($3 + 1) % 256, $4 }')
-
-  if [[ -z $UNATTENDED ]]; then
-    echo ""
-    echo "This grid uses a custom Docker range ($current_range). The authentication"
-    echo "services are moving to their own isolated network, which needs a second /24"
-    echo "that does not overlap it."
-    echo ""
-    while :; do
-      read -rp "Enter the network without the /24 suffix, or press Enter for ${suggested}: " input
-      [[ -z "$input" ]] && input="$suggested"
-      if valid_soauth_range "$input" "$current_range"; then
-        authnet="$input"
-        break
-      fi
-      echo "That range must be a valid IPv4 network, must not be within 172.17.0.0/24, and must not overlap ${current_range}."
-    done
-  else
-    if ! valid_soauth_range "$suggested" "$current_range"; then
-      FINAL_MESSAGE_QUEUE+=("WARNING: Unable to pick a range for the authentication network alongside $current_range. Set it manually before the next highstate:")
-      FINAL_MESSAGE_QUEUE+=("    - so-yaml.py add $pillar_file docker.networks.soauth.range <network>/24")
-      FINAL_MESSAGE_QUEUE+=("    - so-yaml.py add $pillar_file docker.networks.soauth.gateway <gateway>")
-      return 0
-    fi
-    authnet="$suggested"
-    FINAL_MESSAGE_QUEUE+=("NOTE: The authentication services moved to an isolated Docker network and were assigned ${authnet}/24.")
-    FINAL_MESSAGE_QUEUE+=("    - If that conflicts with your environment, update docker.networks.soauth in $pillar_file and run so-checkin.")
-  fi
-
-  authgw=$(echo "$authnet" | awk -F'.' '{print $1,$2,$3,1}' OFS='.')
-
-  echo "Assigning the authentication network the range ${authnet}/24."
-  so-yaml.py add "$pillar_file" docker.networks.soauth.range "${authnet}/24" >> $SOUP_LOG 2>&1
-  so-yaml.py add "$pillar_file" docker.networks.soauth.gateway "$authgw" >> $SOUP_LOG 2>&1
-}
-
-valid_soauth_range() {
-  local candidate=$1 docker_range=$2
-
-  valid_ip4 "$candidate" || return 1
-  [[ $candidate =~ ^172\.17\.0\. ]] && return 1
-  [[ "${candidate}/24" == "$docker_range" ]] && return 1
-  return 0
-}
-
-post_to_3.4.0() {
-  for idx in "metrics-logstash.node-default" "metrics-logstash.stack_monitoring.node-default"; do
-    rollover_index "$idx"
-  done
-
-  set_postversion 3.4.0
-}
-### 3.4.0 End ###
-
 
 repo_sync() {
   echo "Sync the local repo."
@@ -2147,9 +1964,6 @@ main() {
 
   echo "Let's see if we need to update Security Onion."
   upgrade_check
-
-  check_cpu_compatibility
-
   upgrade_space
 
   echo "Verifying Elasticsearch version compatibility across the grid before upgrading."
@@ -2428,17 +2242,6 @@ fi
   echo "### soup has been served at $(date) ###"
 }
 
-SKIP_CPU_CHECK=false
-declare -a SOUP_ARGS=()
-for arg in "$@"; do
-  if [[ "$arg" == "--skip-cpu-check" ]]; then
-    SKIP_CPU_CHECK=true
-  else
-    SOUP_ARGS+=("$arg")
-  fi
-done
-set -- "${SOUP_ARGS[@]}"
-
 while getopts ":b:f:y" opt; do
   case ${opt} in
     b )
@@ -2462,7 +2265,7 @@ while getopts ":b:f:y" opt; do
       ISOLOC="$OPTARG"
     ;;
     \? )
-      echo "Usage: soup [-b] [-y] [-f <iso location>] [--skip-cpu-check]"
+      echo "Usage: soup [-b] [-y] [-f <iso location>]"
       exit 1
     ;;
     : )

salt/nginx/config.sls

@@ -57,8 +57,8 @@ nginx_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://nginx/tools/sbin
-    - user: root
+    - user: 939
-    - group: root
+    - group: 939
     - file_mode: 755
 
 #nginx_sbin_jinja:

salt/nginx/etc/nginx.conf

@@ -183,7 +183,7 @@ http {
 		ssl_prefer_server_ciphers on;
 		ssl_protocols TLSv1.2 TLSv1.3;
 
-		location ~* (^/login|^/login/.*|^/js/.*|^/css/.*|^/images/.*|^/pages/.*|^/docs/.*) {
+		location ~* (^/login/.*|^/js/.*|^/css/.*|^/images/.*|^/pages/.*|^/docs/.*) {
 			proxy_pass            http://{{ GLOBALS.manager }}:9822;
 			proxy_read_timeout    90;
 			proxy_connect_timeout 90;
@@ -260,7 +260,7 @@ http {
 		}
 
 {%  if 'api' in salt['pillar.get']('features', []) %}
-		location ~* (^/oauth2/token.*|^/\.well-known/jwks.json|^/\.well-known/openid-configuration) {
+		location ~* (^/oauth2/token.*|^.well-known/jwks.json|^.well-known/openid-configuration) {
 			limit_req             zone=auth_throttle burst={{ NGINXMERGED.config.throttle_login_burst }} nodelay;
 			limit_req_status      429;
 			proxy_pass            http://{{ GLOBALS.manager }}:4444;

salt/postgres/files/init-db.sh

@@ -29,8 +29,6 @@ psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-E
     -- revoking CONNECT closes the soft edge entirely.
     REVOKE CONNECT ON DATABASE "$POSTGRES_DB" FROM PUBLIC;
     GRANT CONNECT ON DATABASE "$POSTGRES_DB" TO "$SO_POSTGRES_USER";
-
-    CREATE EXTENSION IF NOT EXISTS vector;
 EOSQL
 
 # Bootstrap the Telegraf metrics database. Per-minion roles + schemas are

salt/reactor/pillar_push_map.yaml

@@ -1,6 +1,3 @@
-# Read by push_pillar.sls (SOC config saves) and push_files.sls (local/salt file edits);
-# both key on the app name. An app missing here waits for the next scheduled highstate.
-#
 # One pillar directory can map to multiple (state, tgt) actions.
 # tgt is a raw salt compound expression. tgt_type is always "compound".
 # Per-action `batch` / `batch_wait` override the orch defaults (25% / 15s).

salt/reactor/push_files.sls

@@ -1,131 +0,0 @@
-#!py
-
-# Reactor invoked by local_files_beacon when a watched directory under
-# /opt/so/saltstack/local/salt/ changes. The beacon tag is an app name in
-# pillar_push_map.yaml, so file changes and pillar changes route through the same
-# table -- see salt/reactor/push_pillar.sls.
-#
-# The app comes from the event tag, not the payload: salt's beacon loop pops the
-# beacon's 'tag' key off the data and appends it to the event tag instead (see
-# salt/beacons/__init__.py). The reactor renderer sets both `tag` and `data` as
-# module globals.
-#
-# Reactors never dispatch directly. The so-push-drainer schedule picks up ready
-# intents, dedupes across pending files, and dispatches orch.push_batch.
-
-import fcntl
-import json
-import logging
-import os
-import time
-
-from salt.client import Caller
-import yaml
-
-LOG = logging.getLogger(__name__)
-
-PENDING_DIR = '/opt/so/state/push_pending'
-LOCK_FILE = os.path.join(PENDING_DIR, '.lock')
-MAX_PATHS = 20
-
-# The pillar_push_map.yaml is shipped via salt:// but the reactor runs on the
-# master, which mounts the default saltstack tree at this path.
-PUSH_MAP_PATH = '/opt/so/saltstack/default/salt/reactor/pillar_push_map.yaml'
-
-_PUSH_MAP_CACHE = {'mtime': 0, 'data': None}
-
-
-def _load_push_map():
-    try:
-        st = os.stat(PUSH_MAP_PATH)
-    except OSError:
-        LOG.warning('push_files: %s not found', PUSH_MAP_PATH)
-        return {}
-    if _PUSH_MAP_CACHE['mtime'] != st.st_mtime:
-        try:
-            with open(PUSH_MAP_PATH, 'r') as f:
-                _PUSH_MAP_CACHE['data'] = yaml.safe_load(f) or {}
-        except Exception:
-            LOG.exception('push_files: failed to load %s', PUSH_MAP_PATH)
-            _PUSH_MAP_CACHE['data'] = {}
-        _PUSH_MAP_CACHE['mtime'] = st.st_mtime
-    return _PUSH_MAP_CACHE['data'] or {}
-
-
-def _push_enabled():
-    try:
-        caller = Caller()
-        return bool(caller.cmd('pillar.get', 'salt:auto_apply:enabled', True))
-    except Exception:
-        LOG.exception('push_files: pillar.get salt:auto_apply:enabled failed, assuming enabled')
-        return True
-
-
-def _write_intent(key, actions, path):
-    now = time.time()
-    try:
-        os.makedirs(PENDING_DIR, exist_ok=True)
-    except OSError:
-        LOG.exception('push_files: cannot create %s', PENDING_DIR)
-        return
-
-    intent_path = os.path.join(PENDING_DIR, '{}.json'.format(key))
-    lock_fd = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR, 0o644)
-    try:
-        fcntl.flock(lock_fd, fcntl.LOCK_EX)
-
-        intent = {}
-        if os.path.exists(intent_path):
-            try:
-                with open(intent_path, 'r') as f:
-                    intent = json.load(f)
-            except (IOError, ValueError):
-                intent = {}
-
-        intent.setdefault('first_touch', now)
-        intent['last_touch'] = now
-        intent['actions'] = actions
-        paths = intent.get('paths', [])
-        if path and path not in paths:
-            paths.append(path)
-            paths = paths[-MAX_PATHS:]
-        intent['paths'] = paths
-
-        tmp_path = intent_path + '.tmp'
-        with open(tmp_path, 'w') as f:
-            json.dump(intent, f)
-        os.rename(tmp_path, intent_path)
-    except Exception:
-        LOG.exception('push_files: failed to write intent %s', intent_path)
-    finally:
-        try:
-            fcntl.flock(lock_fd, fcntl.LOCK_UN)
-        finally:
-            os.close(lock_fd)
-
-
-def run():
-    if not _push_enabled():
-        LOG.info('push_files: push disabled, skipping')
-        return {}
-
-    event = data.get('data', data)  # noqa: F821 -- data provided by reactor
-    path = event.get('path', '')
-    app = tag.rsplit('/', 1)[-1].strip()  # noqa: F821 -- tag provided by reactor
-
-    if not app:
-        LOG.debug('push_files: ignoring event with no app segment: tag=%s', tag)  # noqa: F821
-        return {}
-
-    entry = _load_push_map().get(app)
-    if not entry:
-        LOG.warning(
-            'push_files: app "%s" is not in pillar_push_map.yaml; change will be '
-            'picked up at the next scheduled highstate (path=%s)',
-            app, path,
-        )
-        return {}
-
-    _write_intent('files_{}'.format(app), list(entry), path)
-    LOG.info('push_files: intent updated for %s (path=%s)', app, path)
-    return {}

salt/reactor/push_strelka.sls

@@ -0,0 +1,96 @@
+#!py
+
+# Reactor invoked by the rules_beacon poll beacon (salt/_beacons/rules_beacon.py) on rule
+# file changes under /opt/so/saltstack/local/salt/strelka/rules/compiled/.
+#
+# Writes (or updates) a push intent at /opt/so/state/push_pending/rules_strelka.json
+# and returns {}. The so-push-drainer schedule picks up ready intents, dedupes
+# across pending files, and dispatches orch.push_batch. Reactors never dispatch
+# directly
+
+import fcntl
+import json
+import logging
+import os
+import time
+
+from salt.client import Caller
+
+LOG = logging.getLogger(__name__)
+
+PENDING_DIR = '/opt/so/state/push_pending'
+LOCK_FILE = os.path.join(PENDING_DIR, '.lock')
+MAX_PATHS = 20
+
+# Mirrors GLOBALS.sensor_roles in salt/vars/globals.map.jinja. Sensor-side
+# strelka runs on exactly these four roles; so-import gets strelka.manager
+# instead, which is not fired on pillar changes.
+SENSOR_ROLES = ['so-eval', 'so-heavynode', 'so-sensor', 'so-standalone']
+
+
+def _sensor_compound():
+    return ' or '.join('G@role:{}'.format(r) for r in SENSOR_ROLES)
+
+
+def _push_enabled():
+    try:
+        caller = Caller()
+        return bool(caller.cmd('pillar.get', 'salt:auto_apply:enabled', True))
+    except Exception:
+        LOG.exception('push_strelka: pillar.get salt:auto_apply:enabled failed, assuming enabled')
+        return True
+
+
+def _write_intent(key, actions, path):
+    now = time.time()
+    try:
+        os.makedirs(PENDING_DIR, exist_ok=True)
+    except OSError:
+        LOG.exception('push_strelka: cannot create %s', PENDING_DIR)
+        return
+
+    intent_path = os.path.join(PENDING_DIR, '{}.json'.format(key))
+    lock_fd = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR, 0o644)
+    try:
+        fcntl.flock(lock_fd, fcntl.LOCK_EX)
+
+        intent = {}
+        if os.path.exists(intent_path):
+            try:
+                with open(intent_path, 'r') as f:
+                    intent = json.load(f)
+            except (IOError, ValueError):
+                intent = {}
+
+        intent.setdefault('first_touch', now)
+        intent['last_touch'] = now
+        intent['actions'] = actions
+        paths = intent.get('paths', [])
+        if path and path not in paths:
+            paths.append(path)
+            paths = paths[-MAX_PATHS:]
+        intent['paths'] = paths
+
+        tmp_path = intent_path + '.tmp'
+        with open(tmp_path, 'w') as f:
+            json.dump(intent, f)
+        os.rename(tmp_path, intent_path)
+    except Exception:
+        LOG.exception('push_strelka: failed to write intent %s', intent_path)
+    finally:
+        try:
+            fcntl.flock(lock_fd, fcntl.LOCK_UN)
+        finally:
+            os.close(lock_fd)
+
+
+def run():
+    if not _push_enabled():
+        LOG.info('push_strelka: push disabled, skipping')
+        return {}
+
+    path = data.get('path', '')  # noqa: F821 -- data provided by reactor
+    actions = [{'state': 'strelka', 'tgt': _sensor_compound()}]
+    _write_intent('rules_strelka', actions, path)
+    LOG.info('push_strelka: intent updated for path=%s', path)
+    return {}

salt/reactor/push_suricata.sls

@@ -0,0 +1,95 @@
+#!py
+
+# Reactor invoked by the rules_beacon poll beacon (salt/_beacons/rules_beacon.py) on rule
+# file changes under /opt/so/saltstack/local/salt/suricata/rules/.
+#
+# Writes (or updates) a push intent at /opt/so/state/push_pending/rules_suricata.json
+# and returns {}. The so-push-drainer schedule picks up ready intents, dedupes
+# across pending files, and dispatches orch.push_batch. Reactors never dispatch
+# directly
+
+import fcntl
+import json
+import logging
+import os
+import time
+
+from salt.client import Caller
+
+LOG = logging.getLogger(__name__)
+
+PENDING_DIR = '/opt/so/state/push_pending'
+LOCK_FILE = os.path.join(PENDING_DIR, '.lock')
+MAX_PATHS = 20
+
+# Mirrors GLOBALS.sensor_roles in salt/vars/globals.map.jinja. Suricata also
+# runs on so-import per salt/top.sls, so that role is appended below.
+SENSOR_ROLES = ['so-eval', 'so-heavynode', 'so-sensor', 'so-standalone']
+
+
+def _sensor_compound_plus_import():
+    return ' or '.join('G@role:{}'.format(r) for r in SENSOR_ROLES) + ' or G@role:so-import'
+
+
+def _push_enabled():
+    try:
+        caller = Caller()
+        return bool(caller.cmd('pillar.get', 'salt:auto_apply:enabled', True))
+    except Exception:
+        LOG.exception('push_suricata: pillar.get salt:auto_apply:enabled failed, assuming enabled')
+        return True
+
+
+def _write_intent(key, actions, path):
+    now = time.time()
+    try:
+        os.makedirs(PENDING_DIR, exist_ok=True)
+    except OSError:
+        LOG.exception('push_suricata: cannot create %s', PENDING_DIR)
+        return
+
+    intent_path = os.path.join(PENDING_DIR, '{}.json'.format(key))
+    lock_fd = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR, 0o644)
+    try:
+        fcntl.flock(lock_fd, fcntl.LOCK_EX)
+
+        intent = {}
+        if os.path.exists(intent_path):
+            try:
+                with open(intent_path, 'r') as f:
+                    intent = json.load(f)
+            except (IOError, ValueError):
+                intent = {}
+
+        intent.setdefault('first_touch', now)
+        intent['last_touch'] = now
+        intent['actions'] = actions
+        paths = intent.get('paths', [])
+        if path and path not in paths:
+            paths.append(path)
+            paths = paths[-MAX_PATHS:]
+        intent['paths'] = paths
+
+        tmp_path = intent_path + '.tmp'
+        with open(tmp_path, 'w') as f:
+            json.dump(intent, f)
+        os.rename(tmp_path, intent_path)
+    except Exception:
+        LOG.exception('push_suricata: failed to write intent %s', intent_path)
+    finally:
+        try:
+            fcntl.flock(lock_fd, fcntl.LOCK_UN)
+        finally:
+            os.close(lock_fd)
+
+
+def run():
+    if not _push_enabled():
+        LOG.info('push_suricata: push disabled, skipping')
+        return {}
+
+    path = data.get('path', '')  # noqa: F821 -- data provided by reactor
+    actions = [{'state': 'suricata', 'tgt': _sensor_compound_plus_import()}]
+    _write_intent('rules_suricata', actions, path)
+    LOG.info('push_suricata: intent updated for path=%s', path)
+    return {}

salt/redis/config.sls

@@ -50,16 +50,16 @@ redis_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://redis/tools/sbin
-    - user: root
+    - user: 939
-    - group: root
+    - group: 939
     - file_mode: 755
 
 redis_sbin_jinja:
   file.recurse:
     - name: /usr/sbin
     - source: salt://redis/tools/sbin_jinja
-    - user: root
+    - user: 939
-    - group: root
+    - group: 939 
     - file_mode: 755
     - template: jinja
 

salt/repo/client/map.jinja

@@ -9,7 +9,7 @@
        'epel-testing.repo',
        'saltstack.repo',
        'salt-latest.repo',
-       'wazuh.repo',
+       'wazuh.repo'
        'Rocky-Base.repo',
        'Rocky-CR.repo',
        'Rocky-Debuginfo.repo',

salt/salt/files/reactor_pushstate.conf

@@ -1,5 +1,7 @@
 reactor:
-  - 'salt/beacon/*/local_files_beacon/*':
+  - 'salt/beacon/*/rules_beacon/suricata':
-    - salt://reactor/push_files.sls
+    - salt://reactor/push_suricata.sls
+  - 'salt/beacon/*/rules_beacon/strelka':
+    - salt://reactor/push_strelka.sls
   - 'salt/beacon/*/postgres_pillar_beacon/audit_settings':
     - salt://reactor/push_pillar.sls

salt/salt/init.sls

@@ -3,8 +3,6 @@ salt_bootstrap:
   file.managed:
     - name: /usr/sbin/bootstrap-salt.sh
     - source: salt://salt/scripts/bootstrap-salt.sh
-    - user: root
-    - group: root
     - mode: 755
     - show_changes: False
 
@@ -12,6 +10,6 @@ salt_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://salt/tools/sbin
-    - user: root
+    - user: 939
-    - group: root
+    - group: 939
     - file_mode: 755

salt/salt/minion/boot_highstate.sls

@@ -3,8 +3,6 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-
 # Manages /etc/systemd/system/so-boot-highstate.service, a Type=oneshot
 # RemainAfterExit=yes unit that runs `salt-call state.highstate` exactly once
 # per system boot. Replaces the legacy `startup_states: highstate` minion
@@ -21,25 +19,9 @@ so_boot_highstate_unit_file:
     - onchanges_in:
       - module: systemd_reload
 
-# Non-managers never apply salt.minion during setup, so reaching this state means
-# setup is finished and the marker is safe to write unconditionally. This also
-# heals nodes installed before this fix, which have no marker and no legacy
-# startup_states line to grep for. Managers do highstate mid-setup, so they only
-# get the marker from the legacy upgrade signal; fresh installs get it from
-# mark_setup_complete in setup/so-functions.
-mark_setup_complete:
-  file.managed:
-    - name: /opt/so/state/setup-complete
-    - replace: false
-    - makedirs: True
-{% if GLOBALS.is_manager %}
-    - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion"
-{% endif %}
-    - require_in:
-      - service: so_boot_highstate_service
-
 # Only enable once setup is complete. Until then the gate file is missing and
-# the unit's own ConditionPathExists would no-op it anyway.
+# the unit's own ConditionPathExists would no-op it anyway -- this just keeps
+# `systemctl is-enabled` honest for the sync_es_users gate.
 so_boot_highstate_service:
   service.enabled:
     - name: so-boot-highstate.service

salt/salt/minion/init.sls

@@ -87,15 +87,27 @@ set_log_levels:
 # so-boot-highstate.service (managed in salt.minion.boot_highstate), which
 # runs once per system boot only. Strip the line from /etc/salt/minion on
 # upgrade; both the commented and uncommented forms historically existed.
-# Ordered after mark_setup_complete (salt.minion.boot_highstate); the manager
-# gate there greps for this line, so it must run before we delete it.
 remove_startup_states:
   file.line:
     - name: /etc/salt/minion
     - match: 'startup_states: highstate'
     - mode: delete
-    - require:
+
-      - file: mark_setup_complete
+# Upgrade-path bridge: systems that already passed setup under the old gate
+# (`grep -x 'startup_states: highstate' /etc/salt/minion`) get a /opt/so/state/setup-complete
+# marker so so-boot-highstate.service can be enabled and the so-user_sync cron
+# in sync_es_users.sls keeps installing. Setup-in-progress systems instead get
+# the marker from `mark_setup_complete` in setup/so-functions at the right
+# moment. `replace: false` means we never overwrite a marker once written.
+mark_setup_complete_for_upgrades:
+  file.managed:
+    - name: /opt/so/state/setup-complete
+    - replace: false
+    - makedirs: True
+    - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion"
+    - require_in:
+      - file: remove_startup_states
+      - service: so_boot_highstate_service
 
 {% endif %}
 

salt/sensor/init.sls

@@ -35,8 +35,6 @@ combine_bond_script:
   file.managed:
     - name: /usr/sbin/so-combine-bond
     - source: salt://sensor/tools/sbin_jinja/so-combine-bond
-    - user: root
-    - group: root
     - mode: 755
     - template: jinja
     - defaults:

salt/sensoroni/config.sls

@@ -64,8 +64,8 @@ sensoroni_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://sensoroni/tools/sbin
-    - user: root
+    - user: 939
-    - group: root
+    - group: 939
     - file_mode: 755
 
 #sensoroni_sbin_jinja:

salt/sensoroni/files/analyzers/whoislookup/requirements.txt

@@ -1,3 +1,2 @@
-requests>=2.34.0
+requests>=2.31.0
-whoisit>=4.0.5
+whoisit>=2.7.0
-anyio>=4.15.1