Update release date for ISO image in documentation

switch analyzer deps to py 314

.github/DISCUSSION_TEMPLATE/3-0.yml

@@ -12,6 +12,8 @@ body:
         - 3.0.0
         - 3.1.0
         - 3.2.0
+        - 3.3.0
+        - 3.4.0
         - Other (please provide detail below)
     validations:
       required: true

.github/workflows/pythontest.yml

@@ -6,6 +6,10 @@ on:
       - "salt/sensoroni/files/analyzers/**"
       - "salt/manager/tools/sbin/**"
       - "salt/_beacons/**"
+      - "salt/elastalert/files/modules/**"
+      - "salt/telegraf/tools/sbin_jinja/**"
+      - "salt/telegraf/defaults.yaml"
+      - "salt/telegraf/soc_telegraf.yaml"
 
 jobs:
   build:
@@ -15,7 +19,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.14"]
-        python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin", "salt/_beacons"]
+        python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin", "salt/_beacons", "salt/elastalert/files/modules/so"]
 
     steps:
     - uses: actions/checkout@v3
@@ -34,3 +38,25 @@ jobs:
     - name: Test with pytest
       run: |
         PYTHONPATH=${{ matrix.python-code-path }} pytest ${{ matrix.python-code-path }} --cov=${{ matrix.python-code-path }} --doctest-modules --cov-report=term --cov-fail-under=100 --cov-config=pytest.ini
+
+  telegraf-collector:
+    # so-container-stats is a jinja template rather than an importable module, so it gets its
+    # own job: the test renders it the way salt does, then drives it with a faked docker engine
+    # and cgroup tree. No container runtime is needed.
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.14"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest jinja2 pyyaml
+    - name: Lint with flake8
+      run: |
+        flake8 salt/telegraf/tools/sbin_jinja/so-container-stats_test.py --config=pytest.ini
+    - name: Test with pytest
+      run: |
+        pytest salt/telegraf/tools/sbin_jinja/so-container-stats_test.py -v

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 3.1.0-20260528 ISO image released on 2026/05/28
+### 3.4.0-20261101 ISO image released on 2026/11/02
 
 
 ### Download and Verify
 
-3.1.0-20260528 ISO image:  
+3.4.0-20261101 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-3.1.0-20260528.iso
+https://download.securityonion.net/file/securityonion/securityonion-3.4.0-20261101.iso
  
-MD5: 9D6FF58DEEE24089D722C73169765B3E  
+MD5: 486097F2A33F0553BBBF6B7664656E18  
-SHA1: 2B8B816B6CEC3B7F96B3C5E040EBF502DD2C412F  
+SHA1: D6E8B100FFF8BD04DC0FC95AF9D89F54166481D1  
-SHA256: 62FAB57E247C843D6A04F0796D8162C732B65D82FC3E4A59D087135B9FD32912  
+SHA256: F65F190E5BE0BD944EA859A48D5031F75A2061FAD57D11C07B190A680CC62E9C  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.1.0-20260528.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.4.0-20261101.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.1.0-20260528.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.4.0-20261101.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-3.1.0-20260528.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-3.4.0-20261101.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-3.1.0-20260528.iso.sig securityonion-3.1.0-20260528.iso
+gpg --verify securityonion-3.4.0-20261101.iso.sig securityonion-3.4.0-20261101.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Wed 27 May 2026 03:03:59 PM EDT using RSA key ID FE507013
+gpg: Signature made Thu 29 Oct 2026 06:20:38 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

VERSION

@@ -1 +1 @@
-3.2.0
+3.4.0

salt/_beacons/local_files_beacon.py

@@ -3,13 +3,14 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-# Custom salt beacon that watches the suricata/strelka rule directories for changes
+# Custom salt beacon that watches hand-edited directories under
-# and emits a beacon event per changed directory. This replaces the stock salt
+# /opt/so/saltstack/local/salt/ for changes and emits a beacon event per changed
-# `inotify` beacon, which leaks a kernel inotify instance every time the minion
+# directory. This replaces the stock salt `inotify` beacon, which leaks a kernel
-# rebuilds the beacon loader's __context__ (orphaning the old pyinotify.Notifier
+# inotify instance every time the minion rebuilds the beacon loader's __context__
-# without closing it) until fs.inotify.max_user_instances is exhausted and the
+# (orphaning the old pyinotify.Notifier without closing it) until
-# beacon dies with EMFILE. Polling holds zero inotify instances, so the leak is
+# fs.inotify.max_user_instances is exhausted and the beacon dies with EMFILE.
-# impossible, and it keeps firing during state runs (no blackout).
+# Polling holds zero inotify instances, so the leak is impossible, and it keeps
+# firing during state runs (no blackout).
 #
 # Detection is poll-based with a per-directory fingerprint persisted to
 # WATERMARK_DIR: each pass walks the directory and hashes every file's
@@ -19,9 +20,10 @@
 # up on the next one).
 #
 # Each emitted event carries the watched directory path under the configured tag
-# (e.g. salt/beacon/<minion>/rules_beacon/suricata); the push_suricata / push_strelka
+# (e.g. salt/beacon/<minion>/local_files_beacon/zeek); the push_files reactor
-# reactors write a push intent, after which the existing so-push-drainer /
+# looks the tag up in salt/reactor/pillar_push_map.yaml and writes a push intent,
-# orch.push_batch pipeline takes over unchanged.
+# after which the existing so-push-drainer / orch.push_batch pipeline takes over
+# unchanged.
 
 import hashlib
 import logging
@@ -77,7 +79,9 @@ def _fingerprint(directory):
     h = hashlib.sha1()
     if os.path.isdir(directory):
         entries = []
-        for root, _dirs, files in os.walk(directory):
+        for root, dirs, files in os.walk(directory):
+            # zkg packages are git clones; .git churn would fire a state apply on its own.
+            dirs[:] = [d for d in dirs if d != '.git']
             for name in files:
                 full = os.path.join(root, name)
                 if _excluded(full):
@@ -94,20 +98,22 @@ def _fingerprint(directory):
     return h.hexdigest()
 
 
-def _watermark_file(tag):
+def _watermark_file(tag, directory):
-    return os.path.join(WATERMARK_DIR, 'rules_beacon_%s.hash' % tag)
+    # Keyed by directory: zeek/policy and zeek/zkg share the tag `zeek`.
+    scope = hashlib.sha1(directory.encode('utf-8', 'surrogateescape')).hexdigest()[:12]
+    return os.path.join(WATERMARK_DIR, 'local_files_beacon_%s_%s.hash' % (tag, scope))
 
 
-def _read_watermark(tag):
+def _read_watermark(tag, directory):
     try:
-        with open(_watermark_file(tag), 'r') as f:
+        with open(_watermark_file(tag, directory), 'r') as f:
             return (f.read() or '').strip() or None
     except IOError:
         return None
 
 
-def _write_watermark(tag, digest):
+def _write_watermark(tag, directory, digest):
-    path = _watermark_file(tag)
+    path = _watermark_file(tag, directory)
     try:
         os.makedirs(WATERMARK_DIR, exist_ok=True)
         tmp = path + '.tmp'
@@ -115,7 +121,7 @@ def _write_watermark(tag, digest):
             f.write(digest)
         os.rename(tmp, path)
     except OSError:
-        log.exception('rules_beacon: failed to persist watermark to %s', path)
+        log.exception('local_files_beacon: failed to persist watermark to %s', path)
 
 
 def beacon(config):
@@ -123,17 +129,17 @@ def beacon(config):
 
     for directory, tag in _paths_from_config(config).items():
         digest = _fingerprint(directory)
-        previous = _read_watermark(tag)
+        previous = _read_watermark(tag, directory)
 
         # First run / missing watermark: seed the digest and emit nothing so a
         # fresh host does not fire a spurious fleetwide push.
         if previous is None:
-            _write_watermark(tag, digest)
+            _write_watermark(tag, directory, digest)
             continue
 
         if digest != previous:
-            _write_watermark(tag, digest)
+            _write_watermark(tag, directory, digest)
             retval.append({'tag': tag, 'path': directory})
-            log.info('rules_beacon: change detected in %s, emitting %s', directory, tag)
+            log.info('local_files_beacon: change detected in %s, emitting %s', directory, tag)
 
     return retval

salt/_beacons/local_files_beacon_test.py

@@ -0,0 +1,221 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+import hashlib
+import os
+import shutil
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import local_files_beacon
+
+
+class TestRulesBeacon(unittest.TestCase):
+
+    def setUp(self):
+        # Isolate all on-disk state (watermarks and the dirs we fingerprint) in a
+        # throwaway tree, and point WATERMARK_DIR at it so the real read/write
+        # helpers run against actual files.
+        self.tmpdir = tempfile.mkdtemp()
+        self.state = os.path.join(self.tmpdir, 'state')
+        patcher = patch.object(local_files_beacon, 'WATERMARK_DIR', self.state)
+        patcher.start()
+        self.addCleanup(patcher.stop)
+
+    def tearDown(self):
+        shutil.rmtree(self.tmpdir, ignore_errors=True)
+
+    def _make_dir(self, name, files=None):
+        path = os.path.join(self.tmpdir, name)
+        os.makedirs(path, exist_ok=True)
+        for fname, content in (files or {}).items():
+            with open(os.path.join(path, fname), 'w') as f:
+                f.write(content)
+        return path
+
+    # -- trivial contract -------------------------------------------------
+
+    def test_virtual_returns_true(self):
+        self.assertTrue(local_files_beacon.__virtual__())
+
+    def test_validate_returns_valid(self):
+        self.assertEqual(local_files_beacon.validate({}), (True, 'valid'))
+
+    # -- _paths_from_config -----------------------------------------------
+
+    def test_paths_from_config_list_of_dicts(self):
+        config = [{'interval': 10}, {'paths': {'/a': 'suricata', '/b': 'strelka'}}]
+        self.assertEqual(
+            local_files_beacon._paths_from_config(config),
+            {'/a': 'suricata', '/b': 'strelka'},
+        )
+
+    def test_paths_from_config_plain_dict(self):
+        self.assertEqual(
+            local_files_beacon._paths_from_config({'paths': {'/a': 'suricata'}}),
+            {'/a': 'suricata'},
+        )
+
+    def test_paths_from_config_skips_non_dict_items(self):
+        self.assertEqual(local_files_beacon._paths_from_config(['bogus', 42]), {})
+
+    def test_paths_from_config_paths_not_a_dict(self):
+        self.assertEqual(local_files_beacon._paths_from_config({'paths': 'nope'}), {})
+
+    def test_paths_from_config_unexpected_type(self):
+        self.assertEqual(local_files_beacon._paths_from_config('nonsense'), {})
+
+    # -- _excluded --------------------------------------------------------
+
+    def test_excluded_matches_temp_and_editor_files(self):
+        for pathname in ('/rules/foo.swp', '/rules/foo~', '/rules/4913', '/rules/.#foo'):
+            self.assertTrue(local_files_beacon._excluded(pathname), pathname)
+
+    def test_excluded_allows_real_rule_files(self):
+        self.assertFalse(local_files_beacon._excluded('/rules/suricata.rules'))
+
+    # -- _fingerprint -----------------------------------------------------
+
+    def test_fingerprint_missing_dir_is_empty_tree_digest(self):
+        missing = os.path.join(self.tmpdir, 'does-not-exist')
+        self.assertEqual(local_files_beacon._fingerprint(missing), hashlib.sha1().hexdigest())
+
+    def test_fingerprint_changes_when_content_changes(self):
+        d = self._make_dir('rules', {'a.rules': 'alert'})
+        before = local_files_beacon._fingerprint(d)
+        with open(os.path.join(d, 'a.rules'), 'w') as f:
+            f.write('alert tcp any any -> any any')  # different size
+        self.assertNotEqual(local_files_beacon._fingerprint(d), before)
+
+    def test_fingerprint_ignores_excluded_files(self):
+        d = self._make_dir('rules', {'a.rules': 'alert'})
+        before = local_files_beacon._fingerprint(d)
+        with open(os.path.join(d, 'a.rules.swp'), 'w') as f:
+            f.write('editor swap')
+        self.assertEqual(local_files_beacon._fingerprint(d), before)
+
+    def test_fingerprint_skips_unstatable_entries(self):
+        # A dangling symlink appears in os.walk's file list but os.stat raises
+        # OSError, exercising the except-continue path.
+        d = self._make_dir('rules', {'a.rules': 'alert'})
+        good = local_files_beacon._fingerprint(d)
+        os.symlink(os.path.join(d, 'missing-target'), os.path.join(d, 'broken.link'))
+        self.assertEqual(local_files_beacon._fingerprint(d), good)
+
+    def test_fingerprint_prunes_git_metadata(self):
+        # zkg packages are git clones, so the watched tree carries .git.
+        d = self._make_dir('zkg', {'pkg.zeek': 'print 1;'})
+        before = local_files_beacon._fingerprint(d)
+        git_dir = os.path.join(d, 'pkg', '.git', 'refs', 'heads')
+        os.makedirs(git_dir)
+        with open(os.path.join(git_dir, 'main'), 'w') as f:
+            f.write('0' * 40)
+        self.assertEqual(local_files_beacon._fingerprint(d), before)
+
+    def test_fingerprint_still_sees_worktree_next_to_git(self):
+        d = self._make_dir('zkg', {'pkg.zeek': 'print 1;'})
+        os.makedirs(os.path.join(d, 'pkg', '.git'))
+        before = local_files_beacon._fingerprint(d)
+        with open(os.path.join(d, 'pkg', 'scripts.zeek'), 'w') as f:
+            f.write('print 2;')
+        self.assertNotEqual(local_files_beacon._fingerprint(d), before)
+
+    # -- _read_watermark / _write_watermark -------------------------------
+
+    def test_watermark_round_trip(self):
+        local_files_beacon._write_watermark('suricata', '/rules/suricata', 'deadbeef')
+        self.assertEqual(
+            local_files_beacon._read_watermark('suricata', '/rules/suricata'), 'deadbeef')
+
+    def test_read_watermark_missing_returns_none(self):
+        self.assertIsNone(local_files_beacon._read_watermark('suricata', '/rules/suricata'))
+
+    def test_read_watermark_empty_file_returns_none(self):
+        os.makedirs(self.state, exist_ok=True)
+        with open(local_files_beacon._watermark_file('suricata', '/rules/suricata'), 'w') as f:
+            f.write('')
+        self.assertIsNone(local_files_beacon._read_watermark('suricata', '/rules/suricata'))
+
+    def test_write_watermark_swallows_oserror(self):
+        with patch.object(local_files_beacon.os, 'makedirs', side_effect=OSError):
+            local_files_beacon._write_watermark('suricata', '/rules/suricata', 'deadbeef')
+        self.assertIsNone(local_files_beacon._read_watermark('suricata', '/rules/suricata'))
+
+    def test_watermark_file_differs_per_directory_within_one_tag(self):
+        # zeek/policy and zeek/zkg share the tag 'zeek'.
+        self.assertNotEqual(
+            local_files_beacon._watermark_file('zeek', '/local/zeek/policy'),
+            local_files_beacon._watermark_file('zeek', '/local/zeek/zkg'),
+        )
+
+    def test_watermarks_are_independent_within_one_tag(self):
+        local_files_beacon._write_watermark('zeek', '/local/zeek/policy', 'policyhash')
+        local_files_beacon._write_watermark('zeek', '/local/zeek/zkg', 'zkghash')
+        self.assertEqual(
+            local_files_beacon._read_watermark('zeek', '/local/zeek/policy'), 'policyhash')
+        self.assertEqual(
+            local_files_beacon._read_watermark('zeek', '/local/zeek/zkg'), 'zkghash')
+
+    # -- beacon -----------------------------------------------------------
+
+    def _config(self, mapping):
+        return [{'paths': mapping}]
+
+    def test_beacon_seeds_first_run_and_emits_nothing(self):
+        with patch.object(local_files_beacon, '_fingerprint', return_value='hash1'), \
+             patch.object(local_files_beacon, '_read_watermark', return_value=None), \
+             patch.object(local_files_beacon, '_write_watermark') as mock_write:
+            result = local_files_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
+        self.assertEqual(result, [])
+        mock_write.assert_called_once_with('suricata', '/rules/suricata', 'hash1')
+
+    def test_beacon_emits_on_change(self):
+        with patch.object(local_files_beacon, '_fingerprint', return_value='newhash'), \
+             patch.object(local_files_beacon, '_read_watermark', return_value='oldhash'), \
+             patch.object(local_files_beacon, '_write_watermark') as mock_write:
+            result = local_files_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
+        self.assertEqual(result, [{'tag': 'suricata', 'path': '/rules/suricata'}])
+        mock_write.assert_called_once_with('suricata', '/rules/suricata', 'newhash')
+
+    def test_beacon_no_change_emits_nothing(self):
+        with patch.object(local_files_beacon, '_fingerprint', return_value='samehash'), \
+             patch.object(local_files_beacon, '_read_watermark', return_value='samehash'), \
+             patch.object(local_files_beacon, '_write_watermark') as mock_write:
+            result = local_files_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
+        self.assertEqual(result, [])
+        mock_write.assert_not_called()
+
+    def test_beacon_end_to_end_with_real_files(self):
+        # Exercise the full stack (real fingerprint + real watermark files) across
+        # two poll passes: first seeds silently, second fires after a write.
+        d = self._make_dir('rules', {'a.rules': 'alert'})
+        config = self._config({d: 'suricata'})
+
+        self.assertEqual(local_files_beacon.beacon(config), [])  # seed pass
+        self.assertEqual(local_files_beacon.beacon(config), [])  # unchanged pass
+
+        with open(os.path.join(d, 'b.rules'), 'w') as f:
+            f.write('alert tcp any any -> any any')
+        self.assertEqual(local_files_beacon.beacon(config), [{'tag': 'suricata', 'path': d}])
+
+    def test_beacon_two_dirs_one_tag_do_not_flap(self):
+        # Tag-keyed watermarks would clobber each other and emit on every pass.
+        policy = self._make_dir('zeek/policy', {'intel.dat': '#fields\tindicator'})
+        zkg = self._make_dir('zeek/zkg', {'README': 'place packages here'})
+        config = self._config({policy: 'zeek', zkg: 'zeek'})
+
+        self.assertEqual(local_files_beacon.beacon(config), [])  # seed pass
+        self.assertEqual(local_files_beacon.beacon(config), [])  # idle
+        self.assertEqual(local_files_beacon.beacon(config), [])  # still idle
+
+        with open(os.path.join(policy, 'intel.dat'), 'a') as f:
+            f.write('\nevil.com\tIntel::DOMAIN\tsource\n')
+        self.assertEqual(local_files_beacon.beacon(config), [{'tag': 'zeek', 'path': policy}])
+        self.assertEqual(local_files_beacon.beacon(config), [])  # quiet again
+
+
+if __name__ == '__main__':
+    unittest.main()

salt/_beacons/rules_beacon_test.py

@@ -1,172 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-import hashlib
-import os
-import shutil
-import tempfile
-import unittest
-from unittest.mock import patch
-
-import rules_beacon
-
-
-class TestRulesBeacon(unittest.TestCase):
-
-    def setUp(self):
-        # Isolate all on-disk state (watermarks and the dirs we fingerprint) in a
-        # throwaway tree, and point WATERMARK_DIR at it so the real read/write
-        # helpers run against actual files.
-        self.tmpdir = tempfile.mkdtemp()
-        self.state = os.path.join(self.tmpdir, 'state')
-        patcher = patch.object(rules_beacon, 'WATERMARK_DIR', self.state)
-        patcher.start()
-        self.addCleanup(patcher.stop)
-
-    def tearDown(self):
-        shutil.rmtree(self.tmpdir, ignore_errors=True)
-
-    def _make_dir(self, name, files=None):
-        path = os.path.join(self.tmpdir, name)
-        os.makedirs(path, exist_ok=True)
-        for fname, content in (files or {}).items():
-            with open(os.path.join(path, fname), 'w') as f:
-                f.write(content)
-        return path
-
-    # -- trivial contract -------------------------------------------------
-
-    def test_virtual_returns_true(self):
-        self.assertTrue(rules_beacon.__virtual__())
-
-    def test_validate_returns_valid(self):
-        self.assertEqual(rules_beacon.validate({}), (True, 'valid'))
-
-    # -- _paths_from_config -----------------------------------------------
-
-    def test_paths_from_config_list_of_dicts(self):
-        config = [{'interval': 10}, {'paths': {'/a': 'suricata', '/b': 'strelka'}}]
-        self.assertEqual(
-            rules_beacon._paths_from_config(config),
-            {'/a': 'suricata', '/b': 'strelka'},
-        )
-
-    def test_paths_from_config_plain_dict(self):
-        self.assertEqual(
-            rules_beacon._paths_from_config({'paths': {'/a': 'suricata'}}),
-            {'/a': 'suricata'},
-        )
-
-    def test_paths_from_config_skips_non_dict_items(self):
-        self.assertEqual(rules_beacon._paths_from_config(['bogus', 42]), {})
-
-    def test_paths_from_config_paths_not_a_dict(self):
-        self.assertEqual(rules_beacon._paths_from_config({'paths': 'nope'}), {})
-
-    def test_paths_from_config_unexpected_type(self):
-        self.assertEqual(rules_beacon._paths_from_config('nonsense'), {})
-
-    # -- _excluded --------------------------------------------------------
-
-    def test_excluded_matches_temp_and_editor_files(self):
-        for pathname in ('/rules/foo.swp', '/rules/foo~', '/rules/4913', '/rules/.#foo'):
-            self.assertTrue(rules_beacon._excluded(pathname), pathname)
-
-    def test_excluded_allows_real_rule_files(self):
-        self.assertFalse(rules_beacon._excluded('/rules/suricata.rules'))
-
-    # -- _fingerprint -----------------------------------------------------
-
-    def test_fingerprint_missing_dir_is_empty_tree_digest(self):
-        missing = os.path.join(self.tmpdir, 'does-not-exist')
-        self.assertEqual(rules_beacon._fingerprint(missing), hashlib.sha1().hexdigest())
-
-    def test_fingerprint_changes_when_content_changes(self):
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        before = rules_beacon._fingerprint(d)
-        with open(os.path.join(d, 'a.rules'), 'w') as f:
-            f.write('alert tcp any any -> any any')  # different size
-        self.assertNotEqual(rules_beacon._fingerprint(d), before)
-
-    def test_fingerprint_ignores_excluded_files(self):
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        before = rules_beacon._fingerprint(d)
-        with open(os.path.join(d, 'a.rules.swp'), 'w') as f:
-            f.write('editor swap')
-        self.assertEqual(rules_beacon._fingerprint(d), before)
-
-    def test_fingerprint_skips_unstatable_entries(self):
-        # A dangling symlink appears in os.walk's file list but os.stat raises
-        # OSError, exercising the except-continue path.
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        good = rules_beacon._fingerprint(d)
-        os.symlink(os.path.join(d, 'missing-target'), os.path.join(d, 'broken.link'))
-        self.assertEqual(rules_beacon._fingerprint(d), good)
-
-    # -- _read_watermark / _write_watermark -------------------------------
-
-    def test_watermark_round_trip(self):
-        rules_beacon._write_watermark('suricata', 'deadbeef')
-        self.assertEqual(rules_beacon._read_watermark('suricata'), 'deadbeef')
-
-    def test_read_watermark_missing_returns_none(self):
-        self.assertIsNone(rules_beacon._read_watermark('suricata'))
-
-    def test_read_watermark_empty_file_returns_none(self):
-        os.makedirs(self.state, exist_ok=True)
-        with open(rules_beacon._watermark_file('suricata'), 'w') as f:
-            f.write('')
-        self.assertIsNone(rules_beacon._read_watermark('suricata'))
-
-    def test_write_watermark_swallows_oserror(self):
-        with patch.object(rules_beacon.os, 'makedirs', side_effect=OSError):
-            rules_beacon._write_watermark('suricata', 'deadbeef')
-        self.assertIsNone(rules_beacon._read_watermark('suricata'))
-
-    # -- beacon -----------------------------------------------------------
-
-    def _config(self, mapping):
-        return [{'paths': mapping}]
-
-    def test_beacon_seeds_first_run_and_emits_nothing(self):
-        with patch.object(rules_beacon, '_fingerprint', return_value='hash1'), \
-             patch.object(rules_beacon, '_read_watermark', return_value=None), \
-             patch.object(rules_beacon, '_write_watermark') as mock_write:
-            result = rules_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
-        self.assertEqual(result, [])
-        mock_write.assert_called_once_with('suricata', 'hash1')
-
-    def test_beacon_emits_on_change(self):
-        with patch.object(rules_beacon, '_fingerprint', return_value='newhash'), \
-             patch.object(rules_beacon, '_read_watermark', return_value='oldhash'), \
-             patch.object(rules_beacon, '_write_watermark') as mock_write:
-            result = rules_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
-        self.assertEqual(result, [{'tag': 'suricata', 'path': '/rules/suricata'}])
-        mock_write.assert_called_once_with('suricata', 'newhash')
-
-    def test_beacon_no_change_emits_nothing(self):
-        with patch.object(rules_beacon, '_fingerprint', return_value='samehash'), \
-             patch.object(rules_beacon, '_read_watermark', return_value='samehash'), \
-             patch.object(rules_beacon, '_write_watermark') as mock_write:
-            result = rules_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
-        self.assertEqual(result, [])
-        mock_write.assert_not_called()
-
-    def test_beacon_end_to_end_with_real_files(self):
-        # Exercise the full stack (real fingerprint + real watermark files) across
-        # two poll passes: first seeds silently, second fires after a write.
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        config = self._config({d: 'suricata'})
-
-        self.assertEqual(rules_beacon.beacon(config), [])  # seed pass
-        self.assertEqual(rules_beacon.beacon(config), [])  # unchanged pass
-
-        with open(os.path.join(d, 'b.rules'), 'w') as f:
-            f.write('alert tcp any any -> any any')
-        self.assertEqual(rules_beacon.beacon(config), [{'tag': 'suricata', 'path': d}])
-
-
-if __name__ == '__main__':
-    unittest.main()

salt/ca/files/blocklist/emsign-ecc-root-ca-c3.pem

@@ -0,0 +1,14 @@
+-----BEGIN CERTIFICATE-----
+MIICKzCCAbGgAwIBAgIKe3G2gla4EnycqDAKBggqhkjOPQQDAzBaMQswCQYDVQQG
+EwJVUzETMBEGA1UECxMKZW1TaWduIFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJbmMx
+IDAeBgNVBAMTF2VtU2lnbiBFQ0MgUm9vdCBDQSAtIEMzMB4XDTE4MDIxODE4MzAw
+MFoXDTQzMDIxODE4MzAwMFowWjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln
+biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMSAwHgYDVQQDExdlbVNpZ24gRUND
+IFJvb3QgQ0EgLSBDMzB2MBAGByqGSM49AgEGBSuBBAAiA2IABP2lYa57JhAd6bci
+MK4G9IGzsUJxlTm801Ljr6/58pc1kjZGDoeVjbk5Wum739D+yAdBPLtVb4Ojavti
+sIGJAnB9SMVK4+kiVCJNk7tCDK93nCOmfddhEc5lx/h//vXyqaNCMEAwHQYDVR0O
+BBYEFPtaSNCAIEDyqOkAB2kZd6fmw/TPMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB
+Af8EBTADAQH/MAoGCCqGSM49BAMDA2gAMGUCMQC02C8Cif22TGK6Q04ThHK1rt0c
+3ta13FaPWEBaLd4gTCKDypOofu4SQMfWh0/434UCMBwUZOR8loMRnLDRWmFLpg9J
+0wD8ofzkpf9/rdcw0Md3f76BB1UwUCAU9Vc4CqgxUQ==
+-----END CERTIFICATE-----

salt/ca/files/blocklist/emsign-ecc-root-ca-g3.pem

@@ -0,0 +1,15 @@
+-----BEGIN CERTIFICATE-----
+MIICTjCCAdOgAwIBAgIKPPYHqWhwDtqLhDAKBggqhkjOPQQDAzBrMQswCQYDVQQG
+EwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNo
+bm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0g
+RzMwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBrMQswCQYDVQQGEwJJ
+TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s
+b2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0gRzMw
+djAQBgcqhkjOPQIBBgUrgQQAIgNiAAQjpQy4LRL1KPOxst3iAhKAnjlfSU2fySU0
+WXTsuwYc58Byr+iuL+FBVIcUqEqy6HyC5ltqtdyzdc6LBtCGI79G1Y4PPwT01xyS
+fvalY8L1X44uT6EYGQIrMgqCZH0Wk9GjQjBAMB0GA1UdDgQWBBR8XQKEE9TMipuB
+zhccLikenEhjQjAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggq
+hkjOPQQDAwNpADBmAjEAvvNhzwIQHWSVB7gYboiFBS+DCBeQyh+KTOgNG3qxrdWB
+CUfvO6wIBHxcmbHtRwfSAjEAnbpV/KlK6O3t5nYBQnvI+GDZjVGLVTv7jHvrZQnD
++JbNR6iC8hZVdyR+EhCVBCyj
+-----END CERTIFICATE-----

salt/ca/files/blocklist/emsign-root-ca-c1.pem

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDczCCAlugAwIBAgILAK7PALrEzzL4Q7IwDQYJKoZIhvcNAQELBQAwVjELMAkG
+A1UEBhMCVVMxEzARBgNVBAsTCmVtU2lnbiBQS0kxFDASBgNVBAoTC2VNdWRocmEg
+SW5jMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEMxMB4XDTE4MDIxODE4MzAw
+MFoXDTQzMDIxODE4MzAwMFowVjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln
+biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMRwwGgYDVQQDExNlbVNpZ24gUm9v
+dCBDQSAtIEMxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAz+upufGZ
+BczYKCFK83M0UYRWEPWgTywS4/oTmifQz/l5GnRfHXk5/Fv4cI7gklL35CX5VIPZ
+HdPIWoU/Xse2B+4+wM6ar6xWQio5JXDWv7V7Nq2s9nPczdcdioOl+yuQFTdrHCZH
+3DspVpNqs8FqOp099cGXOFgFixwR4+S0uF2FHYP+eF8LRWgYSKVGczQ7/g/IdrvH
+GPMF0Ybzhe3nudkyrVWIzqa2kbBPrH4VI5b2P/AgNBbeCsbEBEV5f6f9vtKppa+c
+xSMq9zwhbL2vj07FOrLzNBL834AaSaTUqZX3noleoomslMuoaJuvimUnzYnu3Yy1
+aylwQ6BpC+S5DwIDAQABo0IwQDAdBgNVHQ4EFgQU/qHgcB4qAzlSWkK+XJGFehiq
+TbUwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL
+BQADggEBAMJKVvoVIXsoounlHfv4LcQ5lkFMOycsxGwYFYDGrK9HWS8mC+M2sO87
+/kOXSTKZEhVb3xEp/6tT+LvBeA+snFOvV71ojD1pM/CjoCNjO2RnIkSt1XHLVip4
+kqNPEjE2NuLe/gDEo2APJ62gsIq1NnpSob0n9CAnYuhNlCQT5AoE6TyrLshDCUrG
+YQTlSTR+08TI9Q/Aqum6VF7zYytPT1DU/rl7mYw9wC68AivTxEDkigcxHpvOJpkT
++xHqmiIMERnHXhuBUDDIlhJu58tBf5E7oke3VIAb3ADMmpDqw8NQBmIMMMAVSKeo
+WXzhriKi4gp6D/piq1JM4fHfyr6DDUI=
+-----END CERTIFICATE-----

salt/ca/files/blocklist/emsign-root-ca-c2.pem

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFcjCCA1qgAwIBAgIKLwq3aw3LSq8nWDANBgkqhkiG9w0BAQwFADBWMQswCQYD
+VQQGEwJVUzETMBEGA1UECxMKZW1TaWduIFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJ
+bmMxHDAaBgNVBAMTE2VtU2lnbiBSb290IENBIC0gQzIwHhcNMTgwMjE4MTgzMDAw
+WhcNNDMwMjE4MTgzMDAwWjBWMQswCQYDVQQGEwJVUzETMBEGA1UECxMKZW1TaWdu
+IFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJbmMxHDAaBgNVBAMTE2VtU2lnbiBSb290
+IENBIC0gQzIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCMfX1lA+Tb
+mh9YInmRgOW97IVx4LUJf2DRZfs837Jrml+py64aVnYgWO4t6C78fgjfS7jX+c4T
+inIzEquWcI+zi0fd4Sc8NDf7JONp27VWX0qwUYqzLDRCt+s7zpLcfx1ky0zVIJj6
+L06uPyK3kIr9+YAsrVj+39utm6e2MBQsRNstSI3fCQYAGvoQTQ8fULauTqNWaYAk
+NYFe6HUHHQPp2u1Ua00odMXiD5oRFxLcDnGAcE1I/9E9mLCdkggXijYUmico7+Xw
+ZeFoPhva6eIJ5p03Lt3Du5W3EcHR0cJmmY1pyeA36JaXKWRNM9IRjYMVNCcp4jhB
+2tIYiZ+LVk8bwQ9/1c23txmv3u97taZlV22NF4ttS1qq3J+MOp0oGULBzpKfRx0q
+GVqbPukQNGAjOLIN8KDNQNzbR1iAl2d8H+MSoicBo4Aid8TjLWcNv48oCWL53ZrF
+BMTDjaIA6frG1t4IpbnHadA7qCJJe2qpJN6n2eQKAUn6UiQDHPsSqNBlcUhQ4Y/0
+Y0mU5rghm2OB9rXQS1Fb1JRCfJMNnJIm5AUB2+2RWzq5Tgz7SbSho8NsZk0UbQnF
+xciqQ9uoVTAsK14Sk9oG8Q3zfsM08cdPoRb0WlIZklR6mKD7L8nH/zfGu8PIJv94
+GGB9RZ9U4A69r3ePmy8MvrzfNxHKtH6svwIDAQABo0IwQDAdBgNVHQ4EFgQUs/eK
+pNYPiABZ6FEXT9V+7IYigZ0wDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMB
+Af8wDQYJKoZIhvcNAQEMBQADggIBADQlpiWM0cv2nZ0H5jVsBq0x2q62Q0LwqATs
+CFvyub7gxNCytRuoA8stmPOEu/lg8Igxj4FIjoyhIrWUVxyiLU7No4P+WjEUOwUT
+xIpkEOtvGUQ9fiOlcGHtIZDNBlZq7WpktXAxeV55RPPsor26p2FNAMRFfZQh0sLX
+hKgk8iulSSggqx8ezgPye63FaiYEi4c/dzRj3HOCnsZiwZZU02df5YpNFjxSwZvE
+41cjGpsrpWMfQFI2s53RbeXp47lSAxYE4NzjBFMe+EwFuEveBCJBEAH5rvYu3pi2
+orsJ424TqWEQV1tCsCkQz+Yq/Okal7yHAkKDeOXcP7oN4A+TdXc2pdqxuVCnBO0R
+mWz2JpGSSeJjiTk/OPwRsPNWtwG/KXL04o2ta3jiPpJuICVtWDAc9R3auBEgJl5r
+ShRmBdszG0LmzsHuZPCFSYC15RBDCOBsa8bDRJ8pBFU2Wi/CVXCACEuavgoveA4F
+a5bt38o0PWxsBP+MpocCdVtDMqzQhxy9IohKuXWAGresoIvKDg3xFk6rBOrjfVwJ
+elwi/xAisojHPJVQv9W1zVIoHp+EQg/4MQC21NbIX2RoioB+V3hK439b/w7deU8x
+2M8cl1OG0nPfbnARl5GPM7vJgi470jto4SeMg6HMAW3Egb56tQcNLwI9U8mZnNvR
+gUMrkAgL
+-----END CERTIFICATE-----

salt/ca/files/blocklist/emsign-root-ca-g1.pem

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDlDCCAnygAwIBAgIKMfXkYgxsWO3W2DANBgkqhkiG9w0BAQsFADBnMQswCQYD
+VQQGEwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBU
+ZWNobm9sb2dpZXMgTGltaXRlZDEcMBoGA1UEAxMTZW1TaWduIFJvb3QgQ0EgLSBH
+MTAeFw0xODAyMTgxODMwMDBaFw00MzAyMTgxODMwMDBaMGcxCzAJBgNVBAYTAklO
+MRMwEQYDVQQLEwplbVNpZ24gUEtJMSUwIwYDVQQKExxlTXVkaHJhIFRlY2hub2xv
+Z2llcyBMaW1pdGVkMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEcxMIIBIjAN
+BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAk0u76WaK7p1b1TST0Bsew+eeuGQz
+f2N4aLTNLnF115sgxk0pvLZoYIr3IZpWNVrzdr3YzZr/k1ZLpVkGoZM0Kd0WNHVO
+8oG0x5ZOrRkVUkr+PHB1cM2vK6sVmjM8qrOLqs1D/fXqcP/tzxE7lM5OMhbTI0Aq
+d7OvPAEsbO2ZLIvZTmmYsvePQbAyeGHWDV/D+qJAkh1cF+ZwPjXnorfCYuKrpDhM
+tTk1b+oDafo6VGiFbdbyL0NVHpENDtjVaqSW0RM8LHhQ6DqS0hdW5TUaQBw+jSzt
+Od9C4INBdN+jzcKGYEho42kLVACL5HZpIQ15TjQIXhTCzLG3rdd8cIrHhQIDAQAB
+o0IwQDAdBgNVHQ4EFgQU++8Nhp6w492pufEhF38+/PB3KxowDgYDVR0PAQH/BAQD
+AgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAFn/8oz1h31x
+PaOfG1vR2vjTnGs2vZupYeveFix0PZ7mddrXuqe8QhfnPZHr5X3dPpzxz5KsbEjM
+wiI/aTvFthUvozXGaCocV685743QNcMYDHsAVhzNixl03r4PEuDQqqE/AjSxcM6d
+GNYIAwlG7mDgfrbESQRRfXBgvKqy/3lyeqYdPV8q+Mri/Tm3R7nrft8EI6/6nAYH
+6ftjk4BAtcZsCjEozgyfz7MjNYBBjWzEN3uBL4ChQEKF6dk4jeihU80Bv2noWgby
+RQuQ+q7hv53yrlc8pa6yVvSLZUDp/TGBLPQ5Cdjua6e0ph0VpZj3AYHYhX3zUVxx
+iN66zB+Afko=
+-----END CERTIFICATE-----

salt/ca/files/blocklist/emsign-root-ca-g2.pem

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFlTCCA32gAwIBAgILAIZNvw/jXtd9jtgwDQYJKoZIhvcNAQEMBQAwZzELMAkG
+A1UEBhMCSU4xEzARBgNVBAsTCmVtU2lnbiBQS0kxJTAjBgNVBAoTHGVNdWRocmEg
+VGVjaG5vbG9naWVzIExpbWl0ZWQxHDAaBgNVBAMTE2VtU2lnbiBSb290IENBIC0g
+RzIwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBnMQswCQYDVQQGEwJJ
+TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s
+b2dpZXMgTGltaXRlZDEcMBoGA1UEAxMTZW1TaWduIFJvb3QgQ0EgLSBHMjCCAiIw
+DQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMNwGIWW2kHfHK+sXTNwxF07K+IV
+ySTuyFM2r1v002wUfcdT+zs5OM5QbMYFFnedXQI6gCFLsjKrcaej48Zt37OyEb3i
+aPs7CsP4kAyTwzKH9aZe6gXYHrJq40/ZVMNcQVI2PcIp40B/SAN2gUZ+ZaUtIOvV
+jEx26/ebNaXRIsthlkOG/caB+QRwDw1tl7338Zlv0M2oTBUy4B3e7dGP5pgXH71M
+jqHPCoNo+xv9f0NTBT+hUDa8h8wUtcGQq9CDeJTpjWcD2bP2AMdVG6oVpMAUeUzo
+cCyglvtFdUMjggxBbw4qhau1HXPG8Ot9hwL7ZMi8tkTzrvUIxxb8G9LF/7kKeCE7
+tGZaVzDTnXuifl3msR4ErHsQ4P7lVu2AIjIAhrAXoedDidb7pMcf7TABdrYUT1Jo
+G/AiK+J9jO6GTjeADD4LMDSBZhHMuBK/PJ/g0kGBt+/C1L+/HURzQhJkMlRnM6Rv
+XoCtfKopSlns5trZmTi971Wjbn88QXP61lGpBCUPwCjs7rpOYvSUJtI+lcbF+37q
+kIqOXYkVT3cupDSpw+H89kFtj5GKY+Xny4LxY+3IvDIRiyd6ky1DPj713DI0yqve
+EpsIr3A0PdwuyUI7CS1jg0NnGFT6Xxyr0xB+VDt83FJYW8v16k2pbaQ4kVxA3aXd
+X9dZYyVR1S59KM75AgMBAAGjQjBAMB0GA1UdDgQWBBTt7E1FYRgo57MjKBEcTaUn
+DV7s9DAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B
+AQwFAAOCAgEACFC/ilQg8KTCVBxFJW/sazomkS0kNYbEIZg4B3obqwsJ7SX98z8Z
+gfzBpz0nYClwwJjWbFN1R2zY8pCEot6/dgmA8Vbq0GxhwPM5YN/SZquNyRIxO3cU
+dlAcwf+vSezdVCf9wOzvSAF3q0a5ljvbdbNJNpfScQVp7UUd5sBsZk8jXO1KQ/go
+/Vf/GDPnrIFmxpAIGE3sgnO8lAv9FzUaAeuv7HWe47xN9J7+bQzF93yHuIXACPTL
+pQHhg2zMv5C7BAbuDHfbj1Cu294Z832yhSfBcziWGskOvl3es2EcHytbS9c9P+0z
+Mpka7zGC1FHrvLb/FoduH86TeZt0QjZ6pcplNzoaxDnDvzTJ6CC2Eny+qH/APFCu
+VUv5/wjwF+HPm8Pup2ARj9cEp92+0qcerfHacNq5hMeGZdbA/dzdUR/5z5zXdxAk
+nl8mcfGb0eMNSTXQmmB/i4AecNnr72uYjzlaXUGYN7Nrb6XouG0pnh0/BBtWWp0U
+ShIPpWEAqs7RJBj6+1ZUYXZ4ObrCw962DxhN2p19Hxw9LtuUUcLqqTPrFXYvwO4t
+ouj7KJnAkaTUfXGdEaFVtFig1EA30WzJY2X1vAQ7hVnniCjgaXAGqjsU6sklNM9n
+xDx5rFCCCEtj9Kh8UHjGK2QqgP5kwgttjOApQMaCoezMfK4KD7WpOXU=
+-----END CERTIFICATE-----

salt/ca/files/blocklist/emsign-root-cs-ca-g2.pem

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFdjCCA16gAwIBAgIQCIYhdXKnxAzDuy7l1y1ubDANBgkqhkiG9w0BAQwFADBV
+MQswCQYDVQQGEwJJTjElMCMGA1UECgwcZU11ZGhyYSBUZWNobm9sb2dpZXMgTGlt
+aXRlZDEfMB0GA1UEAwwWZW1TaWduIFJvb3QgQ1MgQ0EgLSBHMjAeFw0yNDAyMDgw
+ODE1MzlaFw00OTAyMDcwODE1MzlaMFUxCzAJBgNVBAYTAklOMSUwIwYDVQQKDBxl
+TXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVkMR8wHQYDVQQDDBZlbVNpZ24gUm9v
+dCBDUyBDQSAtIEcyMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA4CVO
+xuBjdLJXpkrjosA+7ygTGaYzufUJrhQOeIZEuCqeCPI8hDqIu+MIqEqOlgDFetVM
+3Cor3CGBlGJzfzzlI31vyElvQq88jjCfo12Ghqcf+NAZDFduJk5+NgC0z3lfdiB6
+NNzxbfVkA+X6oDNNRioVUZsVFuIzT9Qt0Sh4KwQPRFBc/wI2yGpDuKHNsLehcECx
+Uw00m33xGYh9UkeL3reOmAaLYWZiO+Pd2vxPtrkXI4wyN4BMlESJ73uP0tyX9lq4
+M4yN2yh3i+ETI8/jzPjxHD7QFaVkCoS+uj2YmBa3mytlqlBgyOoKueNeotX8hfzB
+FNm962abpV4mN0SoC6hjkYv8IayseRPFv9VI0meQXg9OLRoNvBOqG5kw937FH0Li
+q3CIzKyvdyeYRT0LJdWA2iuB9l+4zzIfCwGyT4M4VDmJuPBhrTmjXKae6AUtm1eO
+q2f8gftA6J6AlbD0gRd/t8CrFssFNm+LYAEd36kWRBuolbaOfbSbEww84xNAgj4U
+SPwYjtNDsCkQjtgSOaDtDOmolN+Mbk528Y+LHMlbd4T3Xmp+VIWp5+70GhBDOA91
+hhtims+hHkXjRI2U7WrJLcTlcF8ykiymRbFKhwOBvTmtZNbHmDI7scU4wWUAsmW6
+tWaeeoM50KBl8ai28A6yBGYMO//03Ubvqiqst80CAwEAAaNCMEAwHQYDVR0OBBYE
+FOYcxqontIvyvYVkX6/0unKm0AF1MA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8E
+BTADAQH/MA0GCSqGSIb3DQEBDAUAA4ICAQCv5Asord7fdQMT30emwb+oLuDUxdmJ
+DQHnzxIXoNvymDTbNHF9LqjeXc+X+XcjIV8PTp4o2I9wyVppHmcaVgRjN/G9FxeN
+5tCQ97v5sdx4eAdPljK1xqg7Vhfmzcp+7rSjJG7Sn9gMlZx81kL3Oa2qZj+W5Fi6
+tN4vaR9OLEyvKKHDG/eFxS/qghD1d/Clkb7UGGjBcWJlaZTWVALMKG+QCxiMoELi
+oeKaoBbZoXooxfF0mWdkIl9RWNU4r9O/yt66LBE9rj49dX19L+rGg71/4bTi5ua+
++Wz60XaOc2CdHJaBSols5hWEx/LBjS7UzfU3uJfnpQXVDi0US5sFzIHIlS6kjh7h
+YuF4+/9ZAl4Bx3w2TNN7pVm7I39pgYl7XN6KZRwfkAv4aRswPqhWbQfr3+qfcC93
+cgTyFy2RG5YE/OfmpSs6VtYKawqsdEODz/p2nhZfSDrVDnEPvERqahttY5+gaWNc
+2fbibgKMfuejTZaJ3yttPslVSko5SDs0xKU423swu34sb/fdISqtOu4NzPCZiHSG
+ladaoITmSA6UWFk4fdqaLBI+xSxY5Et3S1APp5r3j/l1KrWPNbb1/+ydHN+YLbh8
+hCBIecKdk6VbnnFRlulm0d2RSgpfCRrWLomN8JHCxCMVv/hBQXz3AXGM1ZlfYTyl
+E1BpDby0G2eEkw==
+-----END CERTIFICATE-----

salt/ca/init.sls

@@ -10,3 +10,4 @@ include:
   - ca.server
 {% endif %}
   - ca.trustca
+  - ca.trust

salt/ca/trust.map.jinja

@@ -0,0 +1,12 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{# Containers mount the extracted directory rather than single files: update-ca-trust replaces
+   files by rename, so a file bind mount would keep pointing at the old inode. #}
+{% set CATRUST = {
+    'dir': '/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro',
+    'java': '/etc/pki/ca-trust/extracted/java/cacerts',
+    'pem': '/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem'
+} %}

salt/ca/trust.sls

@@ -0,0 +1,37 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+ca_trust_anchor:
+  file.managed:
+    - name: /etc/pki/ca-trust/source/anchors/so-intca.crt
+    - source: salt://ca/files/ca.crt
+    - show_changes: False
+    - makedirs: True
+
+ca_trust_legacy_anchor:
+  file.absent:
+    - name: /etc/pki/ca-trust/source/anchors/ca.crt
+
+ca_trust_legacy_bundles:
+  file.absent:
+    - name: /opt/so/conf/ca
+
+# clean removes blocklist entries that are no longer shipped
+ca_trust_blocklist:
+  file.recurse:
+    - name: /etc/pki/ca-trust/source/blocklist
+    - source: salt://ca/files/blocklist
+    - clean: True
+    - makedirs: True
+    - file_mode: '0644'
+    - dir_mode: '0755'
+
+ca_trust_extract:
+  cmd.run:
+    - name: update-ca-trust extract
+    - onchanges:
+      - file: ca_trust_anchor
+      - file: ca_trust_legacy_anchor
+      - file: ca_trust_blocklist

salt/common/init.sls

@@ -117,14 +117,25 @@ elastic_curl_config:
 {% endif %}
 
 
+# A non-root owner here can chmod the directory and replace any script in it, including
+# the root-owned ones. 555 is the mode the filesystem RPM ships; root ignores it anyway.
+usr_sbin_perms:
+  file.directory:
+    - name: /usr/sbin
+    - user: root
+    - group: root
+    - mode: 555
+
 common_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://common/tools/sbin
-    - user: 939
+    - user: root
-    - group: 939
+    - group: root
     - file_mode: 755
     - show_changes: False
+    - require:
+      - file: usr_sbin_perms
 {% if GLOBALS.role == 'so-heavynode' %}
     - exclude_pat:
       - so-pcap-import
@@ -141,12 +152,26 @@ pin_nic_names:
       - file: common_sbin
       - file: statedir
 
+# Once a node is actually running UEK8, the stock EL9 (RHCK) kernel packages are dead weight.
+# They can't be removed any earlier -- dnf protects the running kernel -- so the cleanup waits
+# for the reboot, which makes the highstate the natural place to catch it: fresh installs
+# reboot at the end of setup, and upgraded nodes reboot whenever the admin schedules it.
+# so-kernel-upgrade --cleanup checks rpm before touching dnf, so this costs an rpm query on
+# every highstate after the first pass. The package list lives in the script only, so there
+# is nothing here to drift out of sync with it.
+remove_stock_kernel:
+  cmd.run:
+    - name: /usr/sbin/so-kernel-upgrade --cleanup
+    - onlyif: 'uname -r | grep -qE "^6\.[0-9]+.*uek"'
+    - require:
+      - file: common_sbin
+
 common_sbin_jinja:
   file.recurse:
     - name: /usr/sbin
     - source: salt://common/tools/sbin_jinja
-    - user: 939
+    - user: root
-    - group: 939 
+    - group: root
     - file_mode: 755
     - template: jinja
     - show_changes: False
@@ -159,6 +184,8 @@ so-status_script:
   file.managed:
     - name: /usr/sbin/so-status
     - source: salt://common/tools/sbin/so-status
+    - user: root
+    - group: root
     - mode: 755
 
 {% if GLOBALS.is_sensor %}
@@ -190,9 +217,11 @@ sostatus_log:
     - replace: False
 
 # Install sostatus check cron. This is used to populate Grid.
+# telegraf reads status.log on the same minute boundary this runs, so write aside and rename
+# rather than truncating the file it is reading
 so-status_check_cron:
   cron.present:
-    - name: '/usr/sbin/so-status -j > /opt/so/log/sostatus/status.log 2>&1'
+    - name: '/usr/sbin/so-status -j > /opt/so/log/sostatus/status.log.tmp 2>&1; mv -f /opt/so/log/sostatus/status.log.tmp /opt/so/log/sostatus/status.log'
     - identifier: so-status_check_cron
     - user: root
     - minute: '*/1'

salt/common/soup_scripts.sls

@@ -18,47 +18,61 @@ copy_so-common_common_tools_sbin:
     - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-common
     - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-common
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_so-image-common_common_tools_sbin:
   file.copy:
     - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-image-common
     - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-image-common
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_soup_manager_tools_sbin:
   file.copy:
     - name: /opt/so/saltstack/default/salt/manager/tools/sbin/soup
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/soup
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_so-firewall_manager_tools_sbin:
   file.copy:
     - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-firewall
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_so-yaml_manager_tools_sbin:
   file.copy:
     - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-yaml.py
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-yaml.py
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_so-repo-sync_manager_tools_sbin:
   file.copy:
     - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-repo-sync
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_bootstrap-salt_manager_tools_sbin:
   file.copy:
     - name: /opt/so/saltstack/default/salt/salt/scripts/bootstrap-salt.sh
     - source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 644
 
 # This section is used to put the new script in place so that it can be called during soup.
 # It is faster than calling the states that normally manage them to put them in place.
@@ -67,46 +81,60 @@ copy_so-common_sbin:
     - name: /usr/sbin/so-common
     - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-common
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_so-image-common_sbin:
   file.copy:
     - name: /usr/sbin/so-image-common
     - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-image-common
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_soup_sbin:
   file.copy:
     - name: /usr/sbin/soup
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/soup
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_so-firewall_sbin:
   file.copy:
     - name: /usr/sbin/so-firewall
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_so-yaml_sbin:
   file.copy:
     - name: /usr/sbin/so-yaml.py
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-yaml.py
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_so-repo-sync_sbin:
   file.copy:
     - name: /usr/sbin/so-repo-sync
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755
 
 copy_bootstrap-salt_sbin:
   file.copy:
     - name: /usr/sbin/bootstrap-salt.sh
     - source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh
     - force: True
-    - preserve: True
+    - user: root
+    - group: root
+    - mode: 755

salt/common/tools/sbin/so-common

@@ -240,7 +240,8 @@ copy_new_files() {
   cd $UPDATE_DIR
   rsync -a salt $DEFAULT_SALT_DIR/ --delete "${EXCLUDE_ARGS[@]}"
   rsync -a pillar $DEFAULT_SALT_DIR/ --delete "${EXCLUDE_ARGS[@]}"
-  chown -R socore:socore $DEFAULT_SALT_DIR/
+  # Root-executed code; SOC only needs to read it. Local dirs stay socore-owned.
+  chown -R root:root $DEFAULT_SALT_DIR/
   cd /tmp
 }
 
@@ -418,6 +419,19 @@ is_single_node_grid() {
 	grep "role: so-" /etc/salt/grains | grep -E "eval|standalone|import" &> /dev/null
 }
 
+remove_elasticsearch_index_template() {
+  local template_name=$1
+  local reason=${2:-"Removing index template for upgrade"}
+
+  # check if template exists
+  if so-elasticsearch-query "_index_template/$template_name" --fail --retry 3 --retry-delay 5 >/dev/null 2>&1; then
+    echo "Removing Elasticsearch index template: $template_name ($reason)"
+    if ! so-elasticsearch-query "_index_template/$template_name" -XDELETE --fail --retry 3 --retry-delay 5; then
+      return 1
+    fi
+  fi
+}
+
 initialize_elasticsearch_indices() {
     local index_names=$1
     local default_entry=${2:-'{"@timestamp":"0"}'}

salt/common/tools/sbin/so-common-status-check

@@ -9,6 +9,7 @@ import sys
 import subprocess
 import os
 import json
+import tempfile
 
 sys.path.append('/opt/saltstack/salt/lib/python3.10/site-packages/')
 import salt.config
@@ -17,6 +18,21 @@ import salt.loader
 __opts__ = salt.config.minion_config('/etc/salt/minion')
 __grains__ = salt.loader.grains(__opts__)
 
+def write_atomic(path, value):
+  # telegraf reads these files on its own schedule; replacing them by rename means it never
+  # reads a truncated file and reports an empty value as if it were real
+  directory = os.path.dirname(path)
+  handle, temp = tempfile.mkstemp(dir=directory)
+  try:
+    with os.fdopen(handle, 'w') as f:
+      f.write(str(value))
+    os.chmod(temp, 0o644)
+    os.replace(temp, path)
+  except Exception:
+    os.path.exists(temp) and os.unlink(temp)
+    raise
+
+
 def check_needs_restarted():
   osfam = __grains__['os_family']
   val = '0'
@@ -34,8 +50,7 @@ def check_needs_restarted():
   else:
     fail("Unsupported OS")
 
-  with open(outfile, 'w') as f:
+  write_atomic(outfile, val)
-    f.write(val)
 
 def check_for_fps():
     feat = 'fps'
@@ -56,8 +71,7 @@ def check_for_fps():
           # Unknown, so assume 0
           fps = 0
 
-    with open('/opt/so/log/sostatus/fps_enabled', 'w') as f:
+    write_atomic('/opt/so/log/sostatus/fps_enabled', fps)
-       f.write(str(fps))
 
 def check_for_lks():
     feat = 'Lks'
@@ -80,8 +94,7 @@ def check_for_lks():
                                 lks = 1
         if lks:
             break
-    with open('/opt/so/log/sostatus/lks_enabled', 'w') as f:
+    write_atomic('/opt/so/log/sostatus/lks_enabled', lks)
-       f.write(str(lks))
 
 def fail(msg):
   print(msg, file=sys.stderr)

salt/common/tools/sbin/so-kernel-upgrade

@@ -5,10 +5,11 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 #
-# so-kernel-upgrade — install the UEK8 (6.x) kernel and make it the boot default.
+# so-kernel-upgrade — install the UEK8 (6.x) kernel, make it the boot default, and once the
+# node is running it, remove the stock EL9 kernel.
 #
 # Security Onion is moving off the EL9 stock kernel (RHCK, 5.14) and UEK7 (5.15) onto UEK8
-# (6.x). Three things have to happen, and the tool has to drive each one:
+# (6.x). Four things have to happen, and the tool has to drive each one:
 #
 #   1. Populate. The manager mirrors the UEK8 packages into /nsm/kernelrepo via so-repo-sync,
 #      and serves them to the grid over https://<manager>/kernelrepo. Until that sync runs the
@@ -26,10 +27,21 @@
 #        - From the stock EL9 kernel (RHCK, 5.14, no UEK) it is a flavor CROSS that is NOT
 #          auto-promoted, so the box keeps booting RHCK until grubby is told otherwise.
 #      This tool inspects the running kernel and only runs 'grubby --set-default' for RHCK.
+#   4. Clean up. Once the node is actually RUNNING UEK8 the stock kernel packages are dead
+#      weight -- disk in /boot and a stale GRUB entry. They cannot come off any earlier:
+#      dnf's protect_running_kernel refuses to erase the booted kernel-core, so the removal
+#      has to wait for the reboot. Waiting is also the safer sequencing on its own terms --
+#      the node has proven it comes up on UEK8 before its fallback is deleted. That is why
+#      the removal does not happen in the uek7 branch either, where dnf would allow it.
 #
 # Every one of those failure modes is silent by default. This tool handles each case and fails
 # loudly when it cannot, rather than reporting success while changing nothing.
 #
+# Invocation: with no arguments it drives the whole sequence for whatever kernel the node is
+# on. With --cleanup it does the step 4 removal ONLY, and no-ops on a node that isn't running
+# UEK8 yet -- that is the form the common highstate calls (remove_stock_kernel in
+# salt/common/init.sls) so the cleanup lands grid-wide after each node reboots.
+#
 # Manager vs minion: only the manager owns /nsm/kernelrepo, so only the manager can populate
 # it. If the repo is empty here, a manager runs so-repo-sync itself; a minion has no way to
 # fix it and exits non-zero telling the admin to sync the manager first.
@@ -49,6 +61,11 @@ KERNEL_REPO_DIR="/nsm/kernelrepo"
 REPOSYNC_CONF="/opt/so/conf/reposync/repodownload.conf"
 GLOBAL_PILLAR="/opt/so/saltstack/local/pillar/global/soc_global.sls"
 
+# Stock EL9 (RHCK) kernel packages, removed only once the node is running UEK8 (see step 4
+# in the header). Left deliberately narrow: UEK7 kernel-uek builds age out on their own via
+# installonly_limit=3, and kernel-devel/kernel-headers are not touched.
+RHCK_PKGS="kernel kernel-core kernel-modules kernel-modules-core kernel-tools kernel-tools-libs"
+
 log() { echo "[so-kernel-upgrade] $*"; }
 die() { echo "[so-kernel-upgrade] ERROR: $*" >&2; exit 1; }
 
@@ -149,8 +166,13 @@ ensure_kernel_repo() {
 }
 
 reboot_notice() {
-    [ "$(uname -r)" = "$(basename "$1" | sed 's/^vmlinuz-//')" ] \
+    [ "$(uname -r)" = "$(basename "$1" | sed 's/^vmlinuz-//')" ] && return 0
-        || log "REBOOT REQUIRED to start using the UEK8 kernel (currently running $(uname -r))."
+    log "REBOOT REQUIRED to start using the UEK8 kernel (currently running $(uname -r))."
+    # The stock kernel can't be removed until it stops being the running one, so say when
+    # that will happen rather than leaving the admin to wonder if it was missed.
+    [ -n "$(rhck_installed)" ] \
+        && log "The stock EL9 kernel is left in place until then; it is removed by the next highstate after the reboot."
+    return 0
 }
 
 # Keep future kernel updates on the UEK line rather than falling back to RHCK. Oracle ships
@@ -162,6 +184,32 @@ set_default_kernel_conf() {
     fi
 }
 
+# Which of RHCK_PKGS are actually installed, one per line. rpm -qa treats each argument as a
+# name glob and prints only what it finds, so a package that was never installed (or is
+# already gone) simply doesn't appear -- no "not installed" noise and no non-zero exit.
+rhck_installed() {
+    rpm -qa $RHCK_PKGS 2>/dev/null
+}
+
+# Remove the stock EL9 kernel. Only ever called once the running kernel is UEK8. The rpm
+# check above is the idempotency guard, so this is a cheap no-op on every highstate after
+# the first one -- it costs an rpm query, not a dnf transaction.
+remove_rhck() {
+    local installed; installed="$(rhck_installed)"
+    if [ -z "$installed" ]; then
+        log "no stock EL9 (RHCK) kernel packages installed; nothing to remove."
+        return 0
+    fi
+
+    log "running UEK8; removing the stock EL9 (RHCK) kernel packages:"
+    echo "$installed" | sed 's/^/[so-kernel-upgrade]   /'
+    dnf -y remove $RHCK_PKGS || die "failed to remove the stock EL9 kernel packages"
+
+    installed="$(rhck_installed)"
+    [ -z "$installed" ] || die "dnf reported success but these remain: $(echo $installed)"
+    log "stock EL9 kernel packages removed."
+}
+
 # Make sure a UEK8 kernel is installed, leaving its boot entry in INSTALLED_UEK8. If one is
 # already present we leave the repo alone -- it may be disabled or empty and we don't need it
 # just to flip the boot default. Otherwise install the explicit NEVRA, not the bare package
@@ -184,12 +232,38 @@ ensure_uek8_installed() {
     log "installed UEK8 kernel: $INSTALLED_UEK8"
 }
 
+# --cleanup does step 4 and nothing else. It exits 0 rather than failing on a node that
+# isn't on UEK8 yet: the highstate gates on 'uname -r' before calling this, and a state that
+# fails whenever that gate races would be worse than one that says what it's waiting for.
+case "$1" in
+"")
+    ;;
+--cleanup)
+    if [ "$(running_flavor)" != uek8 ]; then
+        log "not running a UEK8 kernel yet (currently $(uname -r)); leaving the stock EL9 kernel in place."
+        log "Run so-kernel-upgrade with no arguments to install UEK8, then reboot."
+        exit 0
+    fi
+    set_default_kernel_conf
+    remove_rhck
+    exit 0
+    ;;
+*)
+    echo "Usage: so-kernel-upgrade [--cleanup]" >&2
+    echo "  (no arguments)  install UEK8, make it the boot default, clean up once it's running" >&2
+    echo "  --cleanup       remove the stock EL9 kernel; no-op unless already running UEK8" >&2
+    exit 1
+    ;;
+esac
+
 case "$(running_flavor)" in
 uek8)
     # Already on the 6.x UEK line. A plain 'dnf update' keeps this node current within the
-    # lineage and auto-promotes newer builds, so there is nothing for this tool to do.
+    # lineage and auto-promotes newer builds, so there is no install or grubby work left --
-    log "already running a UEK8 kernel ($(uname -r)); nothing to do."
+    # only the step 4 cleanup, which this is the first point in the sequence that can run it.
-    exit 0
+    log "already running a UEK8 kernel ($(uname -r)); no kernel install needed."
+    set_default_kernel_conf
+    remove_rhck
     ;;
 
 uek7)

salt/common/tools/sbin/so-log-check

@@ -134,6 +134,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Redis may have been restarted" # Redis likely restarted by salt
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|file already closed"          # Go logging race condition during container restart
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|relation \"audit_settings\" does not exist" # salt checking for changes before SOC starts
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error in plugin: elasticsearch: Unable to retrieve master node information" # expected error while ES is upgrading/electing a master 
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
@@ -154,6 +155,8 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|id.orig_h"                    # false positive (zeek test data)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|emerging-all.rules"           # false positive (error in rulename)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|invalid query input"          # false positive (Invalid user input in hunt query)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no data available for the requested dates" # false positive (pcap cypress test submits a job with an empty time frame)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no job processor"             # false positive (same empty-time-frame job on import nodes, where no pcap processor runs)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|example"                      # false positive (example test data)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200"                   # false positive (request successful, contained error string in content)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|app_layer.error"              # false positive (suricata 7) in stats.log e.g. app_layer.error.imap.parser | Total | 0
@@ -169,6 +172,13 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error while parsing document for index \[.ds-logs-kratos-so-.*object mapping for \[file\]" # false positive (mapping error occuring BEFORE kratos index has rolled over in 2.4.210)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|No such container"            # false positive (telegraf trying to run stats on an old container)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|passwords do not match"       # false positive (automated hydra test)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Request did not pass preprocessing" # expected WARN log lines indicating invalid auth header
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Missing or invalid authorization header for bearer token" # expected WARN log lines indicating invalid auth header
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unexpected authorization header" # expected WARN log lines indicating invalid auth header
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Missing ory_kratos_session cookie" # expected WARN log lines indicating invalid auth header
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Static assets preprocessor only supports GET and HEAD requests" # expected WARN log lines indicating invalid auth header
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|respondError"                 # respondError is a function name, output via http middleware as standard request logging
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|GET /kibana/"                 # Ignore Kibana queries with triggered words
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
@@ -231,8 +241,9 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|from NIC checksum offloading" # zeek reporter.log
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal"           # docker container getting recycled
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-(tychon|aws_billing|microsoft_defender_endpoint|armis|o365_metrics|microsoft_sentinel|snyk|cyera|island_browser).*user so_kibana lacks the required permissions \[(logs|metrics)-\1" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user. This error should not be seen on fresh ES 9.3.3 installs or after SO 3.1.0 with soups addition of check_transform_health_and_reauthorize()
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown"             # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown"             # so-dockerregistry logs a tag lookup miss during image copy; not tied to one docker version
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Could not index event to Elasticsearch.*\"version\" => \"9.0.8\"" # Expected during Elastic upgrade temporarily, as policies referencing older pipelines are updated
 fi
 
 RESULT=0
@@ -297,4 +308,4 @@ else
     echo -e "\nResult: One or more errors found"
 fi
 
-exit $RESULT
+exit $RESULT

salt/common/tools/sbin/so-restart

@@ -35,9 +35,6 @@ case $1 in
     "elastic-fleet"|"elasticfleet")
         docker_check_running "elastic-fleet" "--stop"
         docker rm "so-elastic-fleet" 2> /dev/null
-        # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy
-        rm -rf /opt/so/conf/elastic-fleet/state
-
         salt-call state.apply elasticfleet queue=True
         ;;
     *)

salt/common/tools/sbin/so-sensor-clean

@@ -8,21 +8,37 @@
 # Elastic License 2.0.
 
 
-SENSOR_DIR='/nsm'
+SENSOR_DIR="${SENSOR_DIR:-/nsm}"
 CRIT_DISK_USAGE=90
-CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
+LOG="${LOG:-/opt/so/log/sensor_clean.log}"
-LOG="/opt/so/log/sensor_clean.log"
+LOCK="${LOCK:-/var/tmp/so-sensor-clean.lock}"
-TODAY=$(date -u "+%Y-%m-%d")
+MAX_PASSES=100
 
+ZEEK_LOGS="$SENSOR_DIR/zeek/logs"
+STRELKA_FILES="$SENSOR_DIR/strelka/processed"
+SURICATA_LOGS="$SENSOR_DIR/suricata"
+PCAPS="$SENSOR_DIR/pcapout"
+
+log() {
+	echo "$(date) - $*" >>"$LOG"
+}
+
+disk_usage() {
+	df -P "$SENSOR_DIR" | tail -1 | awk '{print $5}' | tr -d %
+}
+
+disk_avail() {
+	df -P "$SENSOR_DIR" | tail -1 | awk '{print $4}'
+}
+
+# sets REMOVED=1 if anything was actually deleted
 clean() {
 	## find the oldest Zeek logs directory
-	OLDEST_DIR=$(ls /nsm/zeek/logs/ | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
+	OLDEST_DIR=$(ls "$ZEEK_LOGS" 2>/dev/null | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
-	if [ -z "$OLDEST_DIR" -o "$OLDEST_DIR" == ".." -o "$OLDEST_DIR" == "." ]; then
+	if [ -n "$OLDEST_DIR" ]; then
-		echo "$(date) - No old Zeek logs available to clean up in /nsm/zeek/logs/" >>$LOG
+		log "Removing directory: $ZEEK_LOGS/$OLDEST_DIR"
-		#exit 0
+		rm -rf "$ZEEK_LOGS/$OLDEST_DIR"
-	else
+		REMOVED=1
-		echo "$(date) - Removing directory: /nsm/zeek/logs/$OLDEST_DIR" >>$LOG
-		rm -rf /nsm/zeek/logs/"$OLDEST_DIR"
 	fi
 
 	## Remarking for now, as we are moving extracted files to /nsm/strelka/processed
@@ -43,58 +59,73 @@ clean() {
 	#fi
 
 	## Clean up Zeek extracted files processed by Strelka
-	STRELKA_FILES='/nsm/strelka/processed'
+	OLDEST_STRELKA=$(find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
-	OLDEST_STRELKA=$(find $STRELKA_FILES -type f -printf '%T+ %p\n' | sort -n | head -n 1)
+	if [ -n "$OLDEST_STRELKA" ]; then
-	if [ -z "$OLDEST_STRELKA" -o "$OLDEST_STRELKA" == ".." -o "$OLDEST_STRELKA" == "." ]; then
-		echo "$(date) - No old files available to clean up in $STRELKA_FILES" >>$LOG
-	else
 		OLDEST_STRELKA_DATE=$(echo $OLDEST_STRELKA | awk '{print $1}' | cut -d+ -f1)
-		OLDEST_STRELKA_FILE=$(echo $OLDEST_STRELKA | awk '{print $2}')
+		log "Removing extracted files for $OLDEST_STRELKA_DATE"
-		echo "$(date) - Removing extracted files for $OLDEST_STRELKA_DATE" >>$LOG
+		REMOVED=1
-		find $STRELKA_FILES -type f -printf '%T+ %p\n' | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
+		find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
-			echo "$(date) - Removing file: $FILE" >>$LOG
+			log "Removing file: $FILE"
 			rm -f "$FILE"
 		done
 	fi
 
 	## Clean up Suricata log files
-	SURICATA_LOGS='/nsm/suricata'
+	OLDEST_SURICATA=$(find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
-	OLDEST_SURICATA=$(find $SURICATA_LOGS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
+	if [ -n "$OLDEST_SURICATA" ]; then
-	if [[ -z "$OLDEST_SURICATA" ]] || [[ "$OLDEST_SURICATA" == ".." ]] || [[ "$OLDEST_SURICATA" == "." ]]; then
-		echo "$(date) - No old files available to clean up in $SURICATA_LOGS" >>$LOG
-	else
 		OLDEST_SURICATA_DATE=$(echo $OLDEST_SURICATA | awk '{print $1}' | cut -d+ -f1)
-		OLDEST_SURICATA_FILE=$(echo $OLDEST_SURICATA | awk '{print $2}')
+		log "Removing logs for $OLDEST_SURICATA_DATE"
-		echo "$(date) - Removing logs for $OLDEST_SURICATA_DATE" >>$LOG
+		REMOVED=1
-		find $SURICATA_LOGS -type f -printf '%T+ %p\n' | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
+		find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
-			echo "$(date) - Removing file: $FILE" >>$LOG
+			log "Removing file: $FILE"
 			rm -f "$FILE"
 		done
 	fi
 
 	## Clean up extracted pcaps
-	PCAPS='/nsm/pcapout'
+	OLDEST_PCAP=$(find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
-	OLDEST_PCAP=$(find $PCAPS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
+	if [ -n "$OLDEST_PCAP" ]; then
-	if [ -z "$OLDEST_PCAP" -o "$OLDEST_PCAP" == ".." -o "$OLDEST_PCAP" == "." ]; then
-		echo "$(date) - No old files available to clean up in $PCAPS" >>$LOG
-	else
 		OLDEST_PCAP_DATE=$(echo $OLDEST_PCAP | awk '{print $1}' | cut -d+ -f1)
-		OLDEST_PCAP_FILE=$(echo $OLDEST_PCAP | awk '{print $2}')
+		log "Removing extracted files for $OLDEST_PCAP_DATE"
-		echo "$(date) - Removing extracted files for $OLDEST_PCAP_DATE" >>$LOG
+		REMOVED=1
-		find $PCAPS -type f -printf '%T+ %p\n' | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
+		find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
-			echo "$(date) - Removing file: $FILE" >>$LOG
+			log "Removing file: $FILE"
 			rm -f "$FILE"
 		done
 	fi
 }
 
-# Check to see if we are already running
+# Only one instance at a time; the lock is the fd, so it releases on any exit
-NUM_RUNNING=$(pgrep -cf "/bin/bash /usr/sbin/so-sensor-clean")
+exec 9>"$LOCK" || exit 1
-[ "$NUM_RUNNING" -gt 1 ] && echo "$(date) - $NUM_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0
+if ! flock -n 9; then
-
+	log "another so-sensor-clean is already running (lock $LOCK held); exiting"
-if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then
+	exit 0
-	while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
-		clean
-		CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
-	done
 fi
+
+CUR_USAGE=$(disk_usage)
+[ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ] || exit 0
+
+log "$SENSOR_DIR at ${CUR_USAGE}% (threshold ${CRIT_DISK_USAGE}%); starting cleanup"
+
+PASS=0
+while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
+	PASS=$((PASS + 1))
+	if [ "$PASS" -gt "$MAX_PASSES" ]; then
+		log "stopping after $MAX_PASSES passes; $SENSOR_DIR still at ${CUR_USAGE}%"
+		break
+	fi
+
+	REMOVED=0
+	BEFORE=$(disk_avail)
+	clean
+	CUR_USAGE=$(disk_usage)
+
+	if [ "$REMOVED" -eq 0 ]; then
+		log "nothing left to remove in $ZEEK_LOGS, $STRELKA_FILES, $SURICATA_LOGS, $PCAPS; $SENSOR_DIR still at ${CUR_USAGE}% - space is consumed outside of NSM cleanup scope"
+		break
+	fi
+	if [ "$(disk_avail)" -le "$BEFORE" ]; then
+		log "pass $PASS freed no space; $SENSOR_DIR still at ${CUR_USAGE}% - stopping until next run"
+		break
+	fi
+done

salt/common/tools/sbin/so-stop

@@ -29,8 +29,6 @@ case $1 in
 	"elasticfleet"|"elastic-fleet")
         docker_check_running "elastic-fleet" "--stop"
         docker rm "so-elastic-fleet" 2> /dev/null
-        # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy
-        rm -rf /opt/so/conf/elastic-fleet/state
         ;;
     *)
         docker_check_running "$1" "--stop"

salt/common/tools/sbin_jinja/so-raid-status

@@ -125,4 +125,6 @@ else
   RAIDSTATUS=1
 fi
 
-echo "nsmraid=$RAIDSTATUS" > /opt/so/log/raid/status.log
+# telegraf reads this file; write aside and rename so it never sees a half-written file
+echo "nsmraid=$RAIDSTATUS" > /opt/so/log/raid/status.log.tmp
+mv -f /opt/so/log/raid/status.log.tmp /opt/so/log/raid/status.log

salt/common/tools/sbin_jinja/so-salt-minion-check

@@ -25,7 +25,7 @@ LAST_HIGHSTATE_END=$([ -e "/opt/so/log/salt/lasthighstate" ] && date -r /opt/so/
 LAST_HEALTHCHECK_STATE_APPLY=$([ -e "/opt/so/log/salt/state-apply-test" ] && date -r /opt/so/log/salt/state-apply-test +%s || echo 0)
 # SETTING THRESHOLD TO ANYTHING UNDER 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default
 # THRESHOLD is derived from the salt schedule highstate interval + 1 hour, so the minion-check grace period tracks the schedule automatically.
-THRESHOLD=$(( ({{ SCHEDULEMERGED.highstate_interval_hours }} + 1) * 3600 )) #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted
+THRESHOLD=$(( ({{ SCHEDULEMERGED.highstate_interval_minutes }} + 60) * 60 )) #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted
 THRESHOLD_DATE=$((LAST_HEALTHCHECK_STATE_APPLY+THRESHOLD))
 
 logCmd() {

salt/desktop/trusted-ca.sls

@@ -1,18 +1,10 @@
- {% from 'vars/globals.map.jinja' import GLOBALS %}
+{% from 'vars/globals.map.jinja' import GLOBALS %}
 
 {# we only want this state to run it is CentOS #}
 {% if GLOBALS.os == 'OEL' %}
 
-trusted_ca:
+include:
-  file.managed:
+  - ca.trust
-    - name: /etc/pki/ca-trust/source/anchors/ca.crt
-    - source: salt://ca/files/ca.crt
-
-update_ca_certs:
-  cmd.run:
-    - name: update-ca-trust
-    - onchanges:
-      - file: trusted_ca
 
 {% else %}
 

salt/docker/defaults.yaml

@@ -1,6 +1,12 @@
 docker:
   range: '172.17.1.0/24'
   gateway: '172.17.1.1'
+  networks:
+    sobridge: {}
+    soauth:
+      range: '172.17.2.0/24'
+      gateway: '172.17.2.1'
+      manager_only: True
   ulimits:
     - name: nofile
       soft: 1048576
@@ -58,18 +64,18 @@ docker:
       ulimits: []
     'so-kratos':
       final_octet: 28
+      networks: ['soauth']
       port_bindings:
         - 0.0.0.0:4433:4433
-        - 0.0.0.0:4434:4434
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
       ulimits: []
     'so-hydra':
       final_octet: 30
+      networks: ['soauth']
       port_bindings:
         - 0.0.0.0:4444:4444
-        - 0.0.0.0:4445:4445
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
@@ -128,6 +134,7 @@ docker:
       ulimits: []
     'so-soc':
       final_octet: 34
+      networks: ['sobridge', 'soauth']
       port_bindings:
         - 0.0.0.0:9822:9822
       custom_bind_mounts: []

salt/docker/docker.map.jinja

@@ -1,8 +1,26 @@
 {% import_yaml 'docker/defaults.yaml' as DOCKERDEFAULTS %}
 {% set DOCKERMERGED = salt['pillar.get']('docker', DOCKERDEFAULTS.docker, merge=True) %}
-{% set RANGESPLIT = DOCKERMERGED.range.split('.') %}
+
-{% set FIRSTTHREE = RANGESPLIT[0] ~ '.' ~ RANGESPLIT[1] ~ '.' ~ RANGESPLIT[2] ~ '.' %}
+{% if DOCKERMERGED.networks.sobridge is not mapping %}
+{%   do DOCKERMERGED.networks.update({'sobridge': {}}) %}
+{% endif %}
+{% do DOCKERMERGED.networks['sobridge'].update({'range': DOCKERMERGED.range, 'gateway': DOCKERMERGED.gateway}) %}
+
+{% for netname, net in DOCKERMERGED.networks.items() %}
+{%   set RANGESPLIT = net.range.split('.') %}
+{%   do net.update({'prefix': RANGESPLIT[0] ~ '.' ~ RANGESPLIT[1] ~ '.' ~ RANGESPLIT[2] ~ '.'}) %}
+{% endfor %}
 
 {% for container, vals in DOCKERMERGED.containers.items() %}
-{%   do DOCKERMERGED.containers[container].update({'ip': FIRSTTHREE ~ DOCKERMERGED.containers[container].final_octet}) %}
+{%   set CONTAINER_NETS = vals.get('networks', ['sobridge']) %}
+{%   set IPS = {} %}
+{%   for netname in CONTAINER_NETS %}
+{%     do IPS.update({netname: DOCKERMERGED.networks[netname].prefix ~ vals.final_octet}) %}
+{%   endfor %}
+{%   do DOCKERMERGED.containers[container].update({
+         'networks': CONTAINER_NETS,
+         'ips': IPS,
+         'network': CONTAINER_NETS[0],
+         'ip': IPS[CONTAINER_NETS[0]]
+       }) %}
 {% endfor %}

salt/docker/init.sls

@@ -18,10 +18,10 @@ dockergroup:
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 2.2.1-1.el9
+      - containerd.io: 2.3.6-1.el9
-      - docker-ce: 3:29.2.1-1.el9
+      - docker-ce: 3:29.8.1-1.el9
-      - docker-ce-cli: 1:29.2.1-1.el9
+      - docker-ce-cli: 1:29.8.1-1.el9
-      - docker-ce-rootless-extras: 29.2.1-1.el9
+      - docker-ce-rootless-extras: 29.8.1-1.el9
     - hold: True
     - update_holds: True
 
@@ -71,15 +71,19 @@ dockerreserveports:
     - source: salt://common/files/99-reserved-ports.conf
     - name: /etc/sysctl.d/99-reserved-ports.conf
 
-sos_docker_net:
+{% for NETNAME, NETWORK in DOCKERMERGED.networks.items() %}
+{% if not NETWORK.get('manager_only') or GLOBALS.get('is_manager', False) %}
+sos_docker_net_{{ NETNAME }}:
   docker_network.present:
-    - name: sobridge
+    - name: {{ NETNAME }}
-    - subnet: {{ DOCKERMERGED.range }}
+    - subnet: {{ NETWORK.range }}
-    - gateway: {{ DOCKERMERGED.gateway }}
+    - gateway: {{ NETWORK.gateway }}
     - options:
-        com.docker.network.bridge.name: 'sobridge'
+        com.docker.network.bridge.name: '{{ NETNAME }}'
         com.docker.network.driver.mtu: '1500'
         com.docker.network.bridge.enable_ip_masquerade: 'true'
         com.docker.network.bridge.enable_icc: 'true'
         com.docker.network.bridge.host_binding_ipv4: '0.0.0.0'
-    - unless: ip l | grep sobridge
+    - unless: ip l | grep {{ NETNAME }}
+{% endif %}
+{% endfor %}

salt/docker/soc_docker.yaml

@@ -7,6 +7,40 @@ docker:
     description: Default docker IP range for containers.
     helpLink: docker
     advanced: True
+  networks:
+    sobridge:
+      description: |
+        The default docker network, carrying most containers. Its range and gateway are taken
+        from the docker.range and docker.gateway settings above rather than set here.
+      helpLink: docker
+      readonly: True
+      advanced: True
+      global: True
+    soauth:
+      range:
+        description: |
+          IP range for the soauth docker network, an isolated network for the authentication
+          services, so that the Kratos and Hydra admin APIs are only reachable from the
+          containers placed on it.
+        helpLink: docker
+        readonly: True
+        advanced: True
+        global: True
+      gateway:
+        description: Gateway for the soauth docker network.
+        helpLink: docker
+        readonly: True
+        advanced: True
+        global: True
+      manager_only:
+        description: |
+          Limits the soauth network to grid members running the authentication containers,
+          instead of creating it on every node.
+        helpLink: docker
+        readonly: True
+        advanced: True
+        global: True
+        forcedType: bool
   ulimits:
     description: |
       Default ulimit settings applied to all containers via the Docker daemon. Each entry specifies a resource name (e.g. nofile, memlock, core, nproc) with soft and hard limits. Individual container ulimits override these defaults. Valid resource names include: cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime.
@@ -34,6 +68,16 @@ docker:
         readonly: True
         advanced: True
         global: True
+      networks:
+        description: |
+          Docker networks this container is attached to. The first entry is the container's
+          primary network and determines the address its published ports are forwarded to.
+          Defaults to sobridge when unset.
+        helpLink: docker
+        readonly: True
+        advanced: True
+        global: True
+        forcedType: "[]string"
       port_bindings:
         description: List of port bindings for the container.
         helpLink: docker

salt/elastalert/config.sls

@@ -21,6 +21,7 @@ elastalert:
     - gid: 933
     - home: /opt/so/conf/elastalert
     - createhome: False
+    - shell: /sbin/nologin
 
 elastalogdir:
   file.directory:
@@ -33,8 +34,8 @@ elastalert_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://elastalert/tools/sbin
-    - user: 933
+    - user: root
-    - group: 939
+    - group: root
     - file_mode: 755
 
 #elastalert_sbin_jinja:

salt/elastalert/defaults.yaml

@@ -10,7 +10,7 @@ elastalert:
     buffer_time:
       minutes: 10
     old_query_limit:
-      minutes: 5
+      minutes: 1440
     es_port: 9200
     es_conn_timeout: 55
     max_query_size: 5000

salt/elastalert/enabled.sls

@@ -6,9 +6,11 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 
 include:
+  - ca.trust
   - elastalert.config
   - elastalert.sostatus
 
@@ -28,6 +30,7 @@ so-elastalert:
         - ipv4_address: {{ DOCKERMERGED.containers['so-elastalert'].ip }}
     - detach: True
     - binds:
+      - {{ CATRUST.dir }}
       - /opt/so/rules/elastalert:/opt/elastalert/rules/:ro
       - /opt/so/log/elastalert:/var/log/elastalert:rw
       - /opt/so/conf/elastalert/modules/:/opt/elastalert/modules/:ro
@@ -46,8 +49,9 @@ so-elastalert:
       - {{ XTRAHOST }}
         {% endfor %}
       {% endif %}
-      {% if DOCKERMERGED.containers['so-elastalert'].extra_env %}
     - environment:
+      - REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt
+      {% if DOCKERMERGED.containers['so-elastalert'].extra_env %}
         {% for XTRAENV in DOCKERMERGED.containers['so-elastalert'].extra_env %}
       - {{ XTRAENV }}
         {% endfor %}
@@ -65,6 +69,7 @@ so-elastalert:
       - file: elastacustmodulesdir
       - file: elastaconf
     - watch:
+      - cmd: ca_trust_extract
       - file: elastaconf
     - onlyif:
       - "so-elasticsearch-query / | jq -r '.version.number[0:1]' | grep -q 9" {# only run this state if elasticsearch is version 9 #}

salt/elastalert/files/modules/so/conftest.py

@@ -0,0 +1,80 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+from datetime import datetime
+import json
+import logging
+import sys
+import types
+
+# stand-ins when ElastAlert isn't installed (CI)
+try:
+    import elastalert.alerts  # noqa: F401
+except ImportError:
+
+    class Alerter:
+        def __init__(self, rule):
+            self.rule = rule
+
+    class DateTimeEncoder(json.JSONEncoder):
+        def default(self, obj):
+            return obj.isoformat() if hasattr(obj, 'isoformat') else json.JSONEncoder.default(self, obj)
+
+    class EAException(Exception):
+        pass
+
+    def lookup_es_key(doc, term):
+        for part in term.split('.'):
+            if not isinstance(doc, dict) or part not in doc:
+                return None
+            doc = doc[part]
+        return doc
+
+    def ts_to_dt(value):
+        return value if isinstance(value, datetime) else datetime.fromisoformat(value)
+
+    def elasticsearch_client(conf):
+        return None  # tests set the alerter's client
+
+    alerts = types.ModuleType('elastalert.alerts')
+    alerts.Alerter = Alerter
+    alerts.DateTimeEncoder = DateTimeEncoder
+    util = types.ModuleType('elastalert.util')
+    util.EAException = EAException
+    util.elastalert_logger = logging.getLogger('elastalert')
+    util.lookup_es_key = lookup_es_key
+    util.ts_to_dt = ts_to_dt
+    util.elasticsearch_client = elasticsearch_client
+    package = types.ModuleType('elastalert')
+    package.alerts = alerts
+    package.util = util
+    sys.modules.update({'elastalert': package, 'elastalert.alerts': alerts, 'elastalert.util': util})
+
+# stand-ins when elasticsearch-py isn't installed (CI)
+try:
+    import elasticsearch.exceptions  # noqa: F401
+except ImportError:
+
+    class ElasticsearchException(Exception):
+        pass
+
+    class TransportError(ElasticsearchException):
+        pass
+
+    class ConnectionError(TransportError):
+        pass
+
+    class ConflictError(TransportError):
+        pass
+
+    class RequestError(TransportError):
+        pass
+
+    exceptions = types.ModuleType('elasticsearch.exceptions')
+    for cls in (ElasticsearchException, TransportError, ConnectionError, ConflictError, RequestError):
+        setattr(exceptions, cls.__name__, cls)
+    es_package = types.ModuleType('elasticsearch')
+    es_package.exceptions = exceptions
+    sys.modules.update({'elasticsearch': es_package, 'elasticsearch.exceptions': exceptions})

salt/elastalert/files/modules/so/securityonion-es.py

@@ -1,63 +1,269 @@
 # -*- coding: utf-8 -*-
 
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-
+from datetime import datetime, timezone
-from time import gmtime, strftime
+import hashlib
-import requests,json
+import ipaddress
-from elastalert.alerts import Alerter
+import json
+import re
+import uuid
 
 import urllib3
+from elasticsearch.exceptions import ConflictError, ElasticsearchException, RequestError
+from elastalert.alerts import Alerter, DateTimeEncoder
+from elastalert.util import EAException, elastalert_logger, elasticsearch_client, lookup_es_key, ts_to_dt
+
+# grid runs verify_certs: false; also quiets ElastAlert's own queries
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
+ALERT_INDEX = 'logs-detections.alerts-so'
+# ES error text kept in logs and alerts
+ERROR_TEXT_LIMIT = 500
+# a match missing backend columns (window_start, count, @timestamp); the alert is still written
+MATCH_ERRORS = (KeyError, TypeError, ValueError)
+
+
 class SecurityOnionESAlerter(Alerter):
     """
     Use matched data to create alerts in Elasticsearch.
     """
 
-    required_options = set(['detection_title', 'sigma_level'])
+    required_options = {'detection_title', 'sigma_level'}
-    optional_fields = ['sigma_category', 'sigma_product', 'sigma_service']
+    optional_fields = ['sigma_category', 'sigma_product', 'sigma_service', 'sigma_correlation']
+
+    # count column and default summary per type; stored alert data, so not localized
+    CORRELATION_COUNTS = {
+        'event_count': ('event_count', '%count% events'),
+        'value_count': ('value_count', '%count% distinct values'),
+        'temporal': ('event_type_count', '%count% correlated rules matched'),
+        'value_sum': ('value_sum', 'total %count%'),
+        'value_avg': ('value_avg', 'average %count%'),
+        'value_percentile': ('value_percentile', 'percentile %count%'),
+        'value_median': ('value_median', 'median %count%'),
+    }
+    PLACEHOLDER = re.compile(r'%([^%\s]+)%')
+    # group-by fields copied into ECS related.*
+    RELATED_USERS = {'user.name', 'winlog.event_data.TargetUserName', 'winlog.event_data.SubjectUserName'}
+    RELATED_HOSTS = {'host.name', 'host.hostname', 'winlog.computer_name'}
+
+    def __init__(self, rule):
+        super().__init__(rule)
+        # uses the grid's TLS, auth and timeout settings
+        self.es = elasticsearch_client(rule)
+
+    @property
+    def is_correlation(self):
+        return bool(self.rule.get('sigma_correlation'))
+
+    def query_keys(self):
+        """compound_query_key holds the list; query_key is flattened to a string."""
+        if self.rule.get('compound_query_key'):
+            return self.rule['compound_query_key']
+        if self.rule.get('query_key'):
+            return [self.rule['query_key']]
+        return []
+
+    def alert_id(self, match):
+        """Stable id: window end + group values for correlations, source _id otherwise; random without one."""
+        if self.is_correlation:
+            # ungrouped rows have a hashed _id that changes with the count
+            values = ''.join(f"|{lookup_es_key(match, k)}" for k in self.query_keys())
+            key = f"{self.rule['detection_public_id']}|{ts_to_dt(match['@timestamp']).isoformat()}{values}"
+        elif match.get('_id'):
+            key = f"{self.rule['detection_public_id']}|{match['_id']}"
+        else:
+            return uuid.uuid4().hex
+
+        return hashlib.sha256(key.encode('utf-8')).hexdigest()
+
+    def group(self, match):
+        """Group-by values, joined like ElastAlert's realert key."""
+        return ', '.join(str(lookup_es_key(match, k)) for k in self.query_keys())
+
+    def related_bucket(self, key):
+        if key == 'ip' or key.endswith('.ip'):
+            return 'ip'
+        if key in self.RELATED_USERS or key.endswith('.user.name'):
+            return 'user'
+        if key in self.RELATED_HOSTS:
+            return 'hosts'
+        return None
+
+    @staticmethod
+    def valid_ip(value):
+        try:
+            ipaddress.ip_address(value)
+            return True
+        except ValueError:
+            return False
+
+    def related(self, match):
+        """ECS related.* from group-by values; skips invalid IPs."""
+        related = {}
+        for key in self.query_keys():
+            bucket = self.related_bucket(key)
+            if not bucket:
+                continue
+            # original spellings of a lowercased group
+            value = lookup_es_key(match, f"{key}_spellings")
+            if value is None:
+                value = lookup_es_key(match, key)
+            for v in value if isinstance(value, list) else [value]:
+                if v is None or (bucket == 'ip' and not self.valid_ip(str(v))):
+                    continue
+                # dict: ordered and deduped
+                related.setdefault(bucket, {})[str(v)] = None
+        return {bucket: list(values) for bucket, values in related.items()}
+
+    def event_data(self, match):
+        """The match minus the compound query_key field, which ES would map by its last part."""
+        if not self.rule.get('compound_query_key'):
+            return match
+        return {k: v for k, v in match.items() if k != self.rule['query_key']}
+
+    @staticmethod
+    def format_value(value):
+        if isinstance(value, list):
+            shown = ', '.join(str(v) for v in value[:3])
+            return shown if len(value) <= 3 else f"{shown} and {len(value) - 3} more"
+        return str(value)
+
+    @staticmethod
+    def format_count(value):
+        if isinstance(value, float) and not value.is_integer():
+            return f"{value:,.2f}"
+        if isinstance(value, (int, float)):
+            return f"{int(value):,}"
+        return str(value)
+
+    @staticmethod
+    def format_duration(seconds):
+        for unit, size in (('hour', 3600), ('minute', 60)):
+            if seconds >= 2 * size:
+                return f"{seconds // size} {unit}s"
+        return f"{seconds} second{'' if seconds == 1 else 's'}"
+
+    def summary(self, match):
+        """One-line correlation summary."""
+        column, label = self.CORRELATION_COUNTS.get(self.rule['sigma_correlation'], (None, '%count%'))
+        start = ts_to_dt(match['window_start'])
+        end = ts_to_dt(match['@timestamp'])
+        values = {
+            'count': self.format_count(match.get(column)),
+            'start': start.strftime('%Y-%m-%d %H:%M:%S UTC'),
+            'end': end.strftime('%Y-%m-%d %H:%M:%S UTC'),
+            'duration': self.format_duration(int((end - start).total_seconds())),
+        }
+
+        template = self.rule.get('summary_template')
+        if not template:
+            groups = ', '.join(f"{k} %{k}%" for k in self.query_keys())
+            template = f"{label} for {groups} in %duration%" if groups else f"{label} in %duration%"
+
+        def fill(m):
+            if m[1] in values:
+                return values[m[1]]
+            value = lookup_es_key(match, m[1])
+            # unknown placeholders stay visible so typos show
+            return m[0] if value is None else self.format_value(value)
+
+        return self.PLACEHOLDER.sub(fill, template)
 
     def alert(self, matches):
         for match in matches:
-            timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S"'.000Z', gmtime())
+            try:
-            headers = {"Content-Type": "application/json"}
+                alert_id = self.alert_id(match)
+            except MATCH_ERRORS as e:
+                elastalert_logger.warning("Writing alert for rule %s without a stable id, so a retry may duplicate it: %r",
+                                          self.rule['detection_public_id'], e)
+                alert_id = uuid.uuid4().hex
+            try:
+                self.write(alert_id, self.payload(match))
+            except ElasticsearchException as e:
+                # EAException makes ElastAlert retry
+                raise EAException(f"Unable to write the alert to Elasticsearch: {str(e)[:ERROR_TEXT_LIMIT]}") from e
 
-            creds = None
+    def payload(self, match):
-            if 'es_username' in self.rule and 'es_password' in self.rule:
+        rule_info = {
-                creds = (self.rule['es_username'], self.rule['es_password'])
+            "name": self.rule['detection_title'],
+            "uuid": self.rule['detection_public_id']
+        }
 
-            # Start building the rule dict
+        # Add optional fields if they are present in the rule
-            rule_info = {
+        for field in self.optional_fields:
-                "name": self.rule['detection_title'],
+            rule_key = field.split('_')[-1]  # Assumes field format "sigma_<key>"
-                "uuid": self.rule['detection_public_id']
+            if field in self.rule:
-            }
+                rule_info[rule_key] = self.rule[field]
 
-            # Add optional fields if they are present in the rule
+        event_info = {
-            for field in self.optional_fields:
+            "kind": "alert",
-                rule_key = field.split('_')[-1]  # Assumes field format "sigma_<key>"
+            "severity": self.rule['event.severity'],
-                if field in self.rule:
+            "module": self.rule['event.module'],
-                    rule_info[rule_key] = self.rule[field]
+            "dataset": self.rule['event.dataset'],
+            "severity_label": self.rule['sigma_level']
+        }
 
-            # Construct the payload with the conditional rule_info
+        payload = {
-            payload = {
+            "tags": ["alert"],
-                "tags": "alert",
+            "rule": rule_info,
-                "rule": rule_info,
+            "event": event_info,
-                "event": {
+            "sigma_level": self.rule['sigma_level'],
-                    "severity": self.rule['event.severity'],
+            "event_data": self.event_data(match),
-                    "module": self.rule['event.module'],
+            "@timestamp": datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.000Z')
-                    "dataset": self.rule['event.dataset'],
+        }
-                    "severity_label": self.rule['sigma_level']
+
-                },
+        if self.is_correlation:
-                "sigma_level": self.rule['sigma_level'],
+            keys = self.query_keys()
-                "event_data": match,
+            try:
-                "@timestamp": timestamp
+                # built before any is added, so a failure adds none
-            }
+                reason = self.summary(match)
-            url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/logs-detections.alerts-so/_doc/"
+                labels = {"correlation_group_by": ', '.join(keys), "correlation_group": self.group(match)} if keys else None
-            requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds)
+                related = self.related(match)
+            except MATCH_ERRORS as e:
+                elastalert_logger.warning("Writing alert for rule %s without its correlation summary: %r",
+                                          self.rule['detection_public_id'], e)
+            else:
+                payload["event"]["reason"] = reason
+                if labels:
+                    payload["labels"] = labels
+                if related:
+                    payload["related"] = related
+
+        return payload
+
+    def write(self, alert_id, payload):
+        try:
+            self.create(alert_id, payload)
+        except RequestError as e:
+            # mapping rejections come from event_data; retry it as text
+            rejection = str(e)[:ERROR_TEXT_LIMIT]
+            try:
+                self.create(alert_id, self.without_event_data(payload, rejection))
+            except RequestError as again:
+                elastalert_logger.error("Dropping alert %s for rule %s, rejected by Elasticsearch even without its event data: %s; first rejection: %s",
+                                        alert_id, self.rule['detection_public_id'], str(again)[:ERROR_TEXT_LIMIT], rejection)
+                return
+            elastalert_logger.warning("Stored alert %s for rule %s with its event data as text, rejected by Elasticsearch: %s",
+                                      alert_id, self.rule['detection_public_id'], rejection)
+
+    def create(self, alert_id, payload):
+        try:
+            self.es.create(index=ALERT_INDEX, id=alert_id, body=payload)
+        except ConflictError:
+            pass  # a repeat id is already stored
+
+    @staticmethod
+    def without_event_data(payload, rejection):
+        """Moves event_data to event.original; the tag keeps Fleet's final pipeline from dropping it."""
+        fallback = {k: v for k, v in payload.items() if k != 'event_data'}
+        fallback['event'] = dict(payload['event'], original=json.dumps(payload['event_data'], cls=DateTimeEncoder))
+        fallback['error'] = {'message': f"event_data rejected by Elasticsearch: {rejection}"}
+        fallback['tags'] = payload['tags'] + ['preserve_original_event']
+        return fallback
 
     def get_info(self):
         return {'type': 'SecurityOnionESAlerter'}

salt/elastalert/files/modules/so/securityonion-es_test.py

@@ -0,0 +1,260 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+import copy
+from datetime import datetime, timezone
+import importlib.util
+import json
+import os
+import unittest
+from unittest.mock import MagicMock
+
+from elasticsearch.exceptions import ConflictError, ConnectionError, RequestError
+
+spec = importlib.util.spec_from_file_location('securityonion_es', os.path.join(os.path.dirname(__file__), 'securityonion-es.py'))
+es = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(es)
+
+BASE_RULE = {
+    'detection_title': 'Many Failed Network Logons To One Host From One Source',
+    'detection_public_id': '35a42db6-6629-45af-b8aa-e1fa33c28ef5',
+    'sigma_level': 'medium',
+    'sigma_correlation': 'event_count',
+    'event.severity': 3,
+    'event.module': 'sigma',
+    'event.dataset': 'sigma.alert',
+    'summary_template': '%count% failed network logons to %host.name% from %source.ip% in %duration%',
+}
+
+PLAIN_RULE = {k: v for k, v in BASE_RULE.items() if k not in ('sigma_correlation', 'summary_template')}
+
+
+def correlation_match():
+    return {
+        'event_count': 3561,
+        'window_start': '2026-09-30T18:05:10+00:00',
+        '@timestamp': '2026-09-30T18:07:53+00:00',
+        'host': {'name': 'host-01'},
+        'source': {'ip': '192.0.2.10'},
+        '_id': '6d1c',
+        'num_hits': 1,
+        'num_matches': 1,
+    }
+
+
+class TestSecurityOnionESAlerter(unittest.TestCase):
+
+    def creates(self, rule, match, effects=None):
+        """Run alert(); return (body, id) of each create."""
+        alerter = es.SecurityOnionESAlerter(rule)
+        alerter.es = MagicMock()
+        alerter.es.create.side_effect = effects
+        alerter.alert([match])
+        calls = alerter.es.create.call_args_list
+        self.assertTrue(all(c.kwargs['index'] == 'logs-detections.alerts-so' for c in calls))
+        return [(json.loads(json.dumps(c.kwargs['body'], cls=es.DateTimeEncoder)), c.kwargs['id']) for c in calls]
+
+    def send(self, rule, match):
+        """Run alert(); return the payload it wrote and its id."""
+        (payload, alert_id), = self.creates(rule, match)
+        return payload, alert_id
+
+    def test_compound_query_key_left_out_of_event_data(self):
+        rule = dict(BASE_RULE, compound_query_key=['host.name', 'source.ip'], query_key='host.name,source.ip')
+        match = correlation_match()
+        match['host.name,source.ip'] = 'host-01, 192.0.2.10'
+        original = copy.deepcopy(match)
+
+        payload, _ = self.send(rule, match)
+
+        self.assertNotIn('host.name,source.ip', payload['event_data'])
+        self.assertEqual(payload['event_data']['host'], {'name': 'host-01'})
+        self.assertEqual(payload['event_data']['source'], {'ip': '192.0.2.10'})
+        self.assertEqual(payload['rule'], {'name': BASE_RULE['detection_title'], 'uuid': BASE_RULE['detection_public_id'], 'correlation': 'event_count'})
+        self.assertEqual(payload['labels'], {'correlation_group_by': 'host.name, source.ip', 'correlation_group': 'host-01, 192.0.2.10'})
+        self.assertEqual(payload['related'], {'hosts': ['host-01'], 'ip': ['192.0.2.10']})
+        self.assertEqual(payload['event']['kind'], 'alert')
+        self.assertEqual(payload['event']['reason'], '3,561 failed network logons to host-01 from 192.0.2.10 in 2 minutes')
+        # ElastAlert reuses the match
+        self.assertEqual(match, original)
+
+    def test_single_query_key(self):
+        rule = dict(BASE_RULE, query_key='user.name', summary_template='%count% failed SOC logins for %user.name%')
+        match = {'event_count': 3, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00',
+                 'user': {'name': 'user@example.invalid'}}
+
+        payload, _ = self.send(rule, match)
+
+        self.assertEqual(payload['labels'], {'correlation_group_by': 'user.name', 'correlation_group': 'user@example.invalid'})
+        self.assertEqual(payload['related'], {'user': ['user@example.invalid']})
+        self.assertEqual(payload['event']['reason'], '3 failed SOC logins for user@example.invalid')
+        self.assertEqual(payload['event_data'], match)
+
+    def test_related_buckets(self):
+        rule = dict(BASE_RULE, compound_query_key=['winlog.event_data.TargetUserName', 'source.ip', 'dns.highest_registered_domain'],
+                    query_key='winlog.event_data.TargetUserName,source.ip,dns.highest_registered_domain')
+        match = correlation_match()
+        match.update({'winlog': {'event_data': {'TargetUserName': ['admin1', 'svc', 'admin1']}},
+                      'source': {'ip': 'not-an-ip'}, 'dns': {'highest_registered_domain': 'example.com'}})
+
+        payload, _ = self.send(rule, match)
+
+        # deduped; invalid IP skipped; domain stays in the group only
+        self.assertEqual(payload['related'], {'user': ['admin1', 'svc']})
+        self.assertEqual(payload['labels']['correlation_group'], "['admin1', 'svc', 'admin1'], not-an-ip, example.com")
+
+        payload, _ = self.send(dict(BASE_RULE, query_key='dns.highest_registered_domain'), match)
+
+        self.assertNotIn('related', payload)
+
+    def test_related_uses_original_spellings(self):
+        rule = dict(BASE_RULE, query_key='user.name', summary_template=None)
+        match = {'event_count': 3, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00',
+                 'user': {'name': 'admin', 'name_spellings': ['Admin', 'admin', 'ADMIN']}}
+
+        payload, _ = self.send(rule, match)
+
+        # the group shows the lowercased value; related.user finds every spelling
+        self.assertEqual(payload['labels']['correlation_group'], 'admin')
+        self.assertEqual(payload['related'], {'user': ['Admin', 'admin', 'ADMIN']})
+        self.assertEqual(payload['event']['reason'], '3 events for user.name admin in 8 seconds')
+
+    def test_plain_rule_has_no_correlation_fields(self):
+        # a query_key alone (e.g. from an override) isn't a correlation
+        rule = dict(PLAIN_RULE, query_key='user.name')
+        # ElastAlert parses @timestamp for EQL hits
+        match = {'@timestamp': datetime(2026, 9, 30, 16, 50, tzinfo=timezone.utc), '_id': 'abc',
+                 'process': {'name': 'whoami.exe'}, 'user': {'name': 'user'}}
+
+        payload, alert_id = self.send(rule, match)
+
+        alerter = es.SecurityOnionESAlerter(rule)
+        self.assertEqual(payload['rule'], {'name': PLAIN_RULE['detection_title'], 'uuid': PLAIN_RULE['detection_public_id']})
+        self.assertNotIn('labels', payload)
+        self.assertNotIn('related', payload)
+        self.assertNotIn('reason', payload['event'])
+        self.assertEqual(payload['event']['kind'], 'alert')
+        self.assertEqual(payload['event_data'], dict(match, **{'@timestamp': '2026-09-30T16:50:00+00:00'}))
+        self.assertEqual(alert_id, alerter.alert_id(match))
+        self.assertNotEqual(alerter.alert_id(match), alerter.alert_id(dict(match, _id='abd')))
+        self.assertNotEqual(alerter.alert_id(match), es.SecurityOnionESAlerter(dict(rule, detection_public_id='other')).alert_id(match))
+        # without an _id, never deduplicated
+        self.assertNotEqual(alerter.alert_id({'_id': None}), alerter.alert_id({'_id': None}))
+
+    def test_ungrouped_correlation_id_ignores_row_hash(self):
+        rule = dict(BASE_RULE, summary_template=None)
+        first = {k: v for k, v in correlation_match().items() if k not in ('host', 'source')}
+        # ES|QL hashes the row into _id, so a later count changes it
+        later = dict(first, event_count=3600, _id='9f2a')
+
+        payload, alert_id = self.send(rule, first)
+
+        alerter = es.SecurityOnionESAlerter(rule)
+        self.assertEqual(alerter.alert_id(first), alerter.alert_id(later))
+        self.assertNotEqual(alerter.alert_id(first), alerter.alert_id(dict(first, **{'@timestamp': '2026-09-30T18:09:00+00:00'})))
+        self.assertEqual(alert_id, alerter.alert_id(first))
+        self.assertEqual(payload['event']['reason'], '3,561 events in 2 minutes')
+        self.assertNotIn('labels', payload)
+
+    def test_temporal_count_column(self):
+        rule = dict(BASE_RULE, sigma_correlation='temporal', summary_template=None)
+        match = {'event_type_count': 2, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00'}
+
+        payload, _ = self.send(rule, match)
+
+        self.assertEqual(payload['event']['reason'], '2 correlated rules matched in 8 seconds')
+
+    def test_grouped_correlation_id_unchanged(self):
+        """Ids of alerts already written must not change."""
+        rule = dict(BASE_RULE, compound_query_key=['host.name', 'source.ip'], query_key='host.name,source.ip')
+        key = f"{BASE_RULE['detection_public_id']}|2026-09-30T18:07:53+00:00|host-01|192.0.2.10"
+
+        self.assertEqual(es.SecurityOnionESAlerter(rule).alert_id(correlation_match()), es.hashlib.sha256(key.encode()).hexdigest())
+
+    def test_rejected_event_data_is_kept_as_text(self):
+        rule = dict(BASE_RULE, query_key='user.name', summary_template='%count% failed SOC logins for %user.name%')
+        match = {'event_count': 3, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00',
+                 'user': {'name': 'user@example.invalid'}}
+        rejected = RequestError(400, 'document_parsing_exception', {})
+
+        (first, _), (second, _) = self.creates(rule, match, [rejected, None])
+
+        self.assertIn('event_data', first)
+        self.assertNotIn('event_data', second)
+        self.assertEqual(json.loads(second['event']['original']), match)
+        self.assertEqual(second['tags'], ['alert', 'preserve_original_event'])
+        self.assertEqual(first['tags'], ['alert'])
+        self.assertTrue(second['error']['message'].startswith('event_data rejected by Elasticsearch: '))
+        self.assertIn('document_parsing_exception', second['error']['message'])
+        # everything else carries over
+        self.assertEqual({k: v for k, v in second['event'].items() if k != 'original'}, first['event'])
+        changed = ('event_data', 'event', 'error', 'tags')
+        self.assertEqual({k: v for k, v in second.items() if k not in changed}, {k: v for k, v in first.items() if k not in changed})
+
+    def test_rejected_dates_are_kept_as_text(self):
+        match = {'@timestamp': datetime(2026, 9, 30, 16, 50, tzinfo=timezone.utc), '_id': 'abc'}
+
+        _, (second, _) = self.creates(PLAIN_RULE, match, [RequestError(400, 'document_parsing_exception', {}), None])
+
+        self.assertEqual(json.loads(second['event']['original']), {'@timestamp': '2026-09-30T16:50:00+00:00', '_id': 'abc'})
+
+    def test_rejected_twice_is_dropped_without_retry(self):
+        rejected = RequestError(400, 'document_parsing_exception', {})
+        match = {'@timestamp': '2026-09-30T16:50:00+00:00', '_id': 'abc'}
+
+        with self.assertLogs('elastalert', 'ERROR') as logs:
+            self.assertEqual(len(self.creates(PLAIN_RULE, match, [rejected, rejected])), 2)
+
+        self.assertIn('Dropping alert', logs.output[0])
+
+    def test_write_failure_is_retried(self):
+        match = {'@timestamp': '2026-09-30T16:50:00+00:00', '_id': 'abc'}
+
+        with self.assertRaisesRegex(es.EAException, 'Unable to write the alert to Elasticsearch'):
+            self.creates(PLAIN_RULE, match, [ConnectionError('N/A', 'refused', None)])
+
+    def test_repeat_id_is_ignored(self):
+        match = {'@timestamp': '2026-09-30T16:50:00+00:00', '_id': 'abc'}
+
+        self.assertEqual(len(self.creates(PLAIN_RULE, match, [ConflictError(409, 'version_conflict_engine_exception', {})])), 1)
+
+    def test_correlation_fields_are_optional(self):
+        rule = dict(BASE_RULE, query_key='source.ip')
+        # no window_start: the summary cannot be built
+        match = {k: v for k, v in correlation_match().items() if k != 'window_start'}
+
+        with self.assertLogs('elastalert', 'WARNING') as logs:
+            payload, _ = self.send(rule, match)
+
+        self.assertIn('without its correlation summary', logs.output[0])
+        self.assertEqual(payload['event']['kind'], 'alert')
+        self.assertNotIn('reason', payload['event'])
+        self.assertNotIn('labels', payload)
+        self.assertNotIn('related', payload)
+
+    def test_unstable_id_still_writes(self):
+        # no @timestamp: the window end is unknown
+        match = {k: v for k, v in correlation_match().items() if k not in ('@timestamp', 'window_start')}
+
+        with self.assertLogs('elastalert', 'WARNING') as logs:
+            payload, alert_id = self.send(BASE_RULE, match)
+
+        self.assertIn('without a stable id', logs.output[0])
+        self.assertEqual(len(alert_id), 32)
+        self.assertEqual(payload['event_data'], match)
+
+    def test_summary_formats_values(self):
+        rule = dict(BASE_RULE, sigma_correlation='value_avg', query_key='source.ip',
+                    summary_template='%count% for %source.ip% to %destination.port% %no.such.field%')
+        match = {'value_avg': 2.5, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00',
+                 'source': {'ip': '192.0.2.10'}, 'destination': {'port': [22, 80, 443, 8080, 8443]}}
+
+        payload, _ = self.send(rule, match)
+
+        self.assertEqual(payload['event']['reason'], '2.50 for 192.0.2.10 to 22, 80, 443 and 2 more %no.such.field%')
+        self.assertEqual(es.SecurityOnionESAlerter.format_count('n/a'), 'n/a')
+
+    def test_get_info(self):
+        self.assertEqual(es.SecurityOnionESAlerter(PLAIN_RULE).get_info(), {'type': 'SecurityOnionESAlerter'})

salt/elastalert/soc_elastalert.yaml

@@ -120,7 +120,7 @@ elastalert:
         helpLink: elastalert
     old_query_limit:
       minutes:
-        description: Amount of time in minutes between queries to start at the most recently run query.
+        description: How long ElastAlert can be down, in minutes, and still resume each rule where it stopped. After a longer outage, rules restart from now and skip the gap.
         global: True
         helpLink: elastalert
     es_conn_timeout:

salt/elastic-fleet-package-registry/config.sls

@@ -19,6 +19,7 @@ elastic-agent-pr:
     - gid: 948
     - home: /opt/so/conf/elastic-fleet-pr
     - createhome: False
+    - shell: /sbin/nologin
 
 {% else %}
 

salt/elastic-fleet-package-registry/enabled.sls

@@ -6,9 +6,11 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 
 include:
+  - ca.trust
   - elastic-fleet-package-registry.config
   - elastic-fleet-package-registry.sostatus
 
@@ -34,8 +36,9 @@ so-elastic-fleet-package-registry:
       {% for BINDING in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].port_bindings %}
       - {{ BINDING }}
       {% endfor %}
-    {% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
     - binds:
+      - {{ CATRUST.pem }}:/etc/ssl/certs/ca-certificates.crt:ro
+    {% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
         {% for BIND in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
       - {{ BIND }}
         {% endfor %}
@@ -52,6 +55,8 @@ so-elastic-fleet-package-registry:
       - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
     {%   endfor %}
     {% endif %}
+    - watch:
+      - cmd: ca_trust_extract
 
 wait_for_so-elastic-fleet-package-registry:
   http.wait_for_successful_query:

salt/elasticagent/config.sls

@@ -20,6 +20,7 @@ elastic-agent:
     - gid: 949
     - home: /opt/so/conf/elastic-agent
     - createhome: False
+    - shell: /sbin/nologin
 
 elasticagentconfdir:
   file.directory:
@@ -39,8 +40,8 @@ elasticagent_sbin_jinja:
   file.recurse:
     - name: /usr/sbin
     - source: salt://elasticagent/tools/sbin_jinja
-    - user: 949
+    - user: root
-    - group: 939
+    - group: root
     - file_mode: 755
     - template: jinja
 

salt/elasticagent/enabled.sls

@@ -6,6 +6,7 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 
 include:
@@ -37,6 +38,7 @@ so-elastic-agent:
       - {{ BINDING }}
       {% endfor %}
     - binds:
+      - {{ CATRUST.dir }}
       - /opt/so/conf/elastic-agent/elastic-agent.yml:/usr/share/elastic-agent/elastic-agent.yml:ro
       - /opt/so/log/elasticagent:/usr/share/elastic-agent/logs
       - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro 
@@ -65,6 +67,7 @@ so-elastic-agent:
       - file: create-elastic-agent-config
       - file: trusttheca
     - watch:
+      - cmd: ca_trust_extract
       - file: create-elastic-agent-config
       - file: trusttheca
 

salt/elasticfleet/config.sls

@@ -26,13 +26,14 @@ elastic-fleet:
     - gid: 947
     - home: /opt/so/conf/elastic-fleet
     - createhome: False
+    - shell: /sbin/nologin
 
 elasticfleet_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://elasticfleet/tools/sbin
-    - user: 947
+    - user: root
-    - group: 939
+    - group: root
     - file_mode: 755
     - show_changes: False
 
@@ -40,8 +41,8 @@ elasticfleet_sbin_jinja:
   file.recurse:
     - name: /usr/sbin
     - source: salt://elasticfleet/tools/sbin_jinja
-    - user: 947
+    - user: root
-    - group: 939 
+    - group: root
     - file_mode: 755
     - template: jinja
     - exclude_pat:
@@ -81,8 +82,8 @@ eapackageupgrade:
   file.managed:
     - name: /usr/sbin/so-elastic-fleet-package-upgrade
     - source: salt://elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
-    - user: 947
+    - user: root
-    - group: 939
+    - group: root
     - mode: 755
     - template: jinja
 

salt/elasticfleet/enabled.sls

@@ -6,15 +6,12 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
 
 {#   This value is generated during node install and stored in minion pillar #}
 {%   set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %}
-{#   Prevent Elastic Agent from re-enrolling with a new agent.id everytime the container starts up.
-       - if a fresh enrollment is needed use 'so-stop elasticfleet'
-       #}
-{%   set ENROLLED = salt['file.file_exists']('/opt/so/conf/elastic-fleet/state/fleet.enc') %}
 
 include:
   - ca
@@ -67,10 +64,10 @@ so-elastic-fleet:
       - {{ BINDING }}
       {% endfor %}
     - binds:
+      - {{ CATRUST.dir }}
       - /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro
       - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro
       - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro
-      - /opt/so/conf/elastic-fleet/state:/usr/share/elastic-agent/state
       - /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs 
      {% if DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
         {% for BIND in DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
@@ -78,7 +75,6 @@ so-elastic-fleet:
         {% endfor %}
       {% endif %}      
     - environment:
-      {% if not ENROLLED %}
       - FLEET_SERVER_ENABLE=true
       - FLEET_URL=https://{{ GLOBALS.hostname }}:8220
       - FLEET_SERVER_ELASTICSEARCH_HOST=https://{{ GLOBALS.manager }}:9200
@@ -88,9 +84,6 @@ so-elastic-fleet:
       - FLEET_SERVER_CERT_KEY=/etc/pki/elasticfleet-server.key
       - FLEET_CA=/etc/pki/tls/certs/intca.crt     
       - FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt
-      {% endif %}
-      - STATE_PATH=/usr/share/elastic-agent/state
-      - CONFIG_PATH=/usr/share/elastic-agent/state
       - LOGS_PATH=logs
       {% if DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
         {% for XTRAENV in DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
@@ -104,12 +97,12 @@ so-elastic-fleet:
     {%   endfor %}
     {% endif %}
     - watch:
+      - cmd: ca_trust_extract
       - file: trusttheca
       - x509: etc_elasticfleet_key
       - x509: etc_elasticfleet_crt
     - require:
       - file: trusttheca
-      - file: eastatedir
       - x509: etc_elasticfleet_key
       - x509: etc_elasticfleet_crt
 

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -5,7 +5,7 @@
   "package": {
     "name": "endpoint",
     "title": "Elastic Defend",
-    "version": "9.3.1",
+    "version": "9.4.1",
     "requires_root": true
   },
   "enabled": true,

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -29,7 +29,7 @@
               "\\.gz$"
             ],
             "include_files": [],
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.20.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.3\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.20.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.20.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.3\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- add_fields:\n    target: event\n    fields:\n      dataset: windows.forwarded\n      module: windows\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: import.evtx\n- if:\n    equals:\n      winlog.channel: 'Security'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: system.security\n          module: system\n- if:\n    equals:\n      winlog.channel: 'Windows PowerShell'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell\n          module: windows\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n          module: system\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n          module: system\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n- add_fields:\n    target: data_stream\n    fields:\n        type: logs\n        dataset: import",
             "tags": [
               "import"
             ],

salt/elasticfleet/install_agent_grid.sls

@@ -14,8 +14,8 @@ so-elastic-agent-install:
   file.managed:
     - name: /usr/sbin/so-elastic-agent-install
     - source: salt://elasticfleet/tools/sbin/so-elastic-agent-install
-    - user: 947
+    - user: root
-    - group: 939
+    - group: root
     - mode: 755
     - show_changes: False
 

salt/elasticfleet/integration-defaults.map.jinja

@@ -16,7 +16,6 @@
     'awsfirehose.metrics': 'aws.cloudwatch',
     'cribl.logs': 'cribl',
     'cribl.metrics': 'cribl',
-    'sentinel_one_cloud_funnel.logins': 'sentinel_one_cloud_funnel.login',
     'azure_application_insights.app_insights': 'azure.app_insights',
     'azure_application_insights.app_state': 'azure.app_state',
     'azure_billing.billing': 'azure.billing',
@@ -31,8 +30,6 @@
     'azure_metrics.monitor': 'azure.monitor',
     'azure_metrics.storage_account': 'azure.storage_account',
     'azure_openai.metrics': 'azure.open_ai',
-    'beat.state': 'beats.stack_monitoring.state',
-    'beat.stats': 'beats.stack_monitoring.stats',
     'enterprisesearch.health': 'enterprisesearch.stack_monitoring.health',
     'enterprisesearch.stats': 'enterprisesearch.stack_monitoring.stats',
     'kibana.cluster_actions': 'kibana.stack_monitoring.cluster_actions',
@@ -41,7 +38,8 @@
     'kibana.node_rules': 'kibana.stack_monitoring.node_rules',
     'kibana.stats': 'kibana.stack_monitoring.stats',
     'kibana.status': 'kibana.stack_monitoring.status',
-    'logstash.node_cel': 'logstash.stack_monitoring.node',
+    'logstash.node': 'logstash.stack_monitoring.node',
+    'logstash.node_cel': 'logstash.node',
     'logstash.node_stats': 'logstash.stack_monitoring.node_stats',
     'synthetics.browser': 'synthetics-browser',
     'synthetics.browser_network': 'synthetics-browser.network',

salt/elasticfleet/manager.sls

@@ -68,6 +68,24 @@ so-elastic-fleet-package-upgrade:
     - require:
       - http: wait_for_so-kibana
 
+# initial so-elasticsearch-templates run is earlier, but it can skip over templates that have component templates not yet installed to avoid elasticsearch rejecting the template.
+so-elasticsearch-templates-after-fleet-packages:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-templates-load
+    - cwd: /opt/so
+    - unless: test -f /opt/so/state/estemplates.txt
+    - require:
+      - cmd: so-elastic-fleet-package-upgrade
+
+so-elastic-fleet-integration-upgrade:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-integration-upgrade
+    - retry:
+        attempts: 3
+        interval: 10
+    - require:
+      - cmd: so-elastic-fleet-package-upgrade
+
 so-elastic-fleet-integrations:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-integration-policy-load
@@ -86,21 +104,13 @@ so-elastic-agent-grid-upgrade:
     - require:
       - http: wait_for_so-kibana
 
-so-elastic-fleet-integration-upgrade:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-integration-upgrade
-    - retry:
-        attempts: 3
-        interval: 10
-    - require:
-      - http: wait_for_so-kibana
-
 {# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
 so-elastic-fleet-addon-integrations:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
     - require:
       - http: wait_for_so-kibana
+      - cmd: so-elasticsearch-templates-after-fleet-packages
 
 {% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
 so-elastic-defend-manage-filters-file-watch:

salt/elasticfleet/tools/sbin/so-elastic-fleet-common

@@ -30,6 +30,56 @@ fleet_api() {
     curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/${QUERYPATH}" "$@" --retry 3 --retry-delay 10 --fail 2>/dev/null
 }
 
+elastic_fleet_require_agent_policy() {
+    local AGENT_POLICY=$1
+    local POLICY_JSON
+
+    if ! POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY") || [ -z "$POLICY_JSON" ]; then
+        echo "Error: Agent policy '$AGENT_POLICY' was not found or is not visible in the current Kibana space." >&2
+        return 1
+    fi
+
+    if ! jq -e '.item.package_policies | type == "array"' <<<"$POLICY_JSON" >/dev/null 2>&1; then
+        echo "Error: Agent policy '$AGENT_POLICY' was not found or is not visible in the current Kibana space." >&2
+        return 1
+    fi
+
+    echo "$POLICY_JSON"
+}
+
+# Print the single active enrollment token for POLICY_ID.
+# Exit 1: retryable (API failure, invalid response, no active token)
+# Exit 2: multiple active tokens - Shouldn't get into this state without manual intervention
+elastic_fleet_active_enrollment_token() {
+    local POLICY_ID=$1
+    local RESP TOKEN_COUNT API_KEY
+
+    if ! RESP=$(fleet_api "enrollment_api_keys?perPage=100" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
+        echo "Error: Failed to retrieve enrollment tokens for agent policy '$POLICY_ID'." >&2
+        return 1
+    fi
+
+    if ! jq -e '.list' <<<"$RESP" >/dev/null 2>&1; then
+        echo "Error: Invalid enrollment token response for agent policy '$POLICY_ID'." >&2
+        return 1
+    fi
+
+    TOKEN_COUNT=$(jq --arg pid "$POLICY_ID" '[.list[] | select(.policy_id == $pid and .active == true)] | length' <<<"$RESP")
+
+    if [ "${TOKEN_COUNT:-0}" -eq 0 ]; then
+        echo "Error: No active enrollment token found for agent policy '$POLICY_ID'." >&2
+        return 1
+    fi
+
+    if [ "$TOKEN_COUNT" -gt 1 ]; then
+        echo "Error: Found $TOKEN_COUNT active enrollment tokens for agent policy '$POLICY_ID'; expected exactly one." >&2
+        return 2
+    fi
+
+    API_KEY=$(jq -r --arg pid "$POLICY_ID" '.list[] | select(.policy_id == $pid and .active == true) | .api_key' <<<"$RESP")
+    echo "$API_KEY"
+}
+
 # Max number of concurrent Fleet write jobs (create/update). Override via env if needed.
 MAX_FLEET_JOBS=${MAX_FLEET_JOBS:-10}
 
@@ -62,15 +112,7 @@ elastic_fleet_load_integrations_dir() {
     i=0
 
     # Fetch the agent policy a single time; we look up integration ids locally below.
-    if ! POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY"); then
+    if ! POLICY_JSON=$(elastic_fleet_require_agent_policy "$AGENT_POLICY"); then
-        echo "Error: Failed to retrieve agent policy '$AGENT_POLICY'."
-        rm -f "$FAIL_FILE"
-        rm -rf "$OUT_DIR"
-        return 1
-    fi
-
-    if ! jq -e '.item.package_policies' <<<"$POLICY_JSON" >/dev/null 2>&1; then
-        echo "Error: Invalid agent policy response for '$AGENT_POLICY'."
         rm -f "$FAIL_FILE"
         rm -rf "$OUT_DIR"
         return 1
@@ -124,9 +166,15 @@ elastic_fleet_integration_check() {
 
     JSON_STRING=$2
 
-    NAME=$(jq -r .name $JSON_STRING)
+    NAME=$(jq -r .name "$JSON_STRING")
+    INTEGRATION_ID=""
 
-    INTEGRATION_ID=$(/usr/sbin/so-elastic-fleet-agent-policy-view "$AGENT_POLICY" | jq -r '.item.package_policies[] | select(.name=="'"$NAME"'") | .id')
+    local POLICY_JSON
+    if ! POLICY_JSON=$(elastic_fleet_require_agent_policy "$AGENT_POLICY"); then
+        return 1
+    fi
+
+    INTEGRATION_ID=$(jq -r --arg n "$NAME" '.item.package_policies[]? | select(.name==$n) | .id' <<<"$POLICY_JSON")
 
 }
 
@@ -148,7 +196,16 @@ elastic_fleet_integration_remove() {
 
     NAME=$2
 
-    INTEGRATION_ID=$(/usr/sbin/so-elastic-fleet-agent-policy-view "$AGENT_POLICY" | jq -r '.item.package_policies[] | select(.name=="'"$NAME"'") | .id')
+    local POLICY_JSON
+    if ! POLICY_JSON=$(elastic_fleet_require_agent_policy "$AGENT_POLICY"); then
+        return 1
+    fi
+
+    INTEGRATION_ID=$(jq -r --arg n "$NAME" '.item.package_policies[]? | select(.name==$n) | .id' <<<"$POLICY_JSON")
+    if [ -z "$INTEGRATION_ID" ]; then
+        echo "Error: Integration '$NAME' was not found in agent policy '$AGENT_POLICY'." >&2
+        return 1
+    fi
 
     JSON_STRING=$( jq -n \
                     --arg INTEGRATIONID "$INTEGRATION_ID" \

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-defend

@@ -13,7 +13,10 @@ ERROR=false
 for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/elastic-defend/*.json
 do
   printf "\n\nInitial Endpoints Policy - Loading $INTEGRATION\n"
-  elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
+  if ! elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"; then
+    ERROR=true
+    continue
+  fi
   if [ -n "$INTEGRATION_ID" ]; then
       printf "\n\nIntegration $NAME exists - Upgrading integration policy\n"
       if ! elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"; then

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server

@@ -7,20 +7,35 @@
 . /usr/sbin/so-elastic-fleet-common
 
 # Get all the fleet policies
-json_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -L -X GET "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true')
+if ! json_output=$(fleet_api "agent_policies" -H 'kbn-xsrf: true'); then
+    echo "Error: Failed to retrieve Fleet agent policies." >&2
+    exit 1
+fi
+
+if ! jq -e '.items' <<<"$json_output" >/dev/null 2>&1; then
+    echo "Error: Invalid Fleet agent policies response." >&2
+    exit 1
+fi
 
 # Extract the IDs that start with "FleetServer_"
-POLICY=$(echo "$json_output" | jq -r '.items[] | select(.id | startswith("FleetServer_")) | .id')
+POLICY=$(jq -r '.items[] | select(.id | startswith("FleetServer_")) | .id' <<<"$json_output")
 
 # Iterate over each ID in the POLICY variable
 for POLICYNAME in $POLICY; do
     printf "\nUpdating Policy: $POLICYNAME\n"
 
-    # First get the Integration ID
+    if ! POLICY_JSON=$(elastic_fleet_require_agent_policy "$POLICYNAME"); then
-    INTEGRATION_ID=$(/usr/sbin/so-elastic-fleet-agent-policy-view "$POLICYNAME" | jq -r '.item.package_policies[] | select(.package.name == "fleet_server") | .id')  
+        exit 1
+    fi
+
+    INTEGRATION_ID=$(jq -r '.item.package_policies[]? | select(.package.name == "fleet_server") | .id' <<<"$POLICY_JSON")
+    if [ -z "$INTEGRATION_ID" ]; then
+        echo "Error: fleet_server integration was not found in agent policy '$POLICYNAME'." >&2
+        exit 1
+    fi
 
     # Modify the default integration policy to update the policy_id and an with the correct naming
-	UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "$POLICYNAME" --arg name "fleet_server-$POLICYNAME" '
+    UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "$POLICYNAME" --arg name "fleet_server-$POLICYNAME" '
     .policy_id = $policy_id |
     .name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json)
 

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers

@@ -22,12 +22,19 @@ NUM_RUNNING=$(pgrep -cf "/bin/bash /sbin/so-elastic-agent-gen-installers")
 
 for i in {1..30}
 do
-   ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys?perPage=100" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
+   ENROLLMENTOKEN=$(elastic_fleet_active_enrollment_token "endpoints-initial")
+   TOKEN_RC=$?
+   if [ "$TOKEN_RC" -eq 2 ]; then
+     exit 1
+   fi
    FLEETHOST=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts/grid-default' | jq -r  '.item.host_urls[]' | paste -sd ',')
-if [[ $FLEETHOST ]] && [[  $ENROLLMENTOKEN ]]; then break; else sleep 10; fi
+   if [[ -n "$FLEETHOST" ]] && [[ -n "$ENROLLMENTOKEN" ]]; then
+     break
+   fi
+   sleep 10
 done
 
-if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then 
+if [[ -z "$FLEETHOST" ]] || [[ -z "$ENROLLMENTOKEN" ]]; then
   printf "\nFleet Host URL, Enrollment Token or Elastic Version empty - exiting..."
   printf "\nFleet Host: $FLEETHOST, Enrollment Token: $ENROLLMENTOKEN\n"
   exit 1
@@ -67,19 +74,25 @@ for GOOS in "${GOTARGETOS[@]}"; do
     GOARCH="amd64"
     if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi
     printf "\n\n### Generating $GOOS/$GOARCH Installer...\n"
-    docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \
+    if ! docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \
     --mount type=bind,source=/etc/pki/tls/certs/,target=/workspace/files/cert/ \
     --mount type=bind,source=/nsm/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \
     --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ \
-    {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN"  -o /output/so-elastic-agent_${GOOS}_${GOARCH}
+    {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN"  -o /output/so-elastic-agent_${GOOS}_${GOARCH}; then
+        printf "\n### ERROR: Failed to generate $GOOS/$GOARCH installer. Exiting...\n"
+        exit 1
+    fi
     printf "\n### $GOOS/$GOARCH Installer Generated...\n"
 done
 
 printf "\n\n### Generating MSI...\n"
 cp /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe
-docker run \
+if ! docker run \
 --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ -w /output \
-{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs
+{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs; then
+    printf "\n### ERROR: Failed to generate MSI. Exiting...\n"
+    exit 1
+fi
 printf "\n### MSI Generated...\n"
 
 # Verify installers were created

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade

@@ -10,6 +10,25 @@
 
 PKG_LOAD_FAILURES=0
 PKG_LOAD_FAILURES_NAMES=()
+PKG_UPGRADED=0
+
+cleanup_elasticsearch_fleet_transforms() {
+    local transforms transform_id attempt
+    
+    if ! transforms=$(so-elasticsearch-query "_transform/logs-elasticsearch.index_pivot-default-*" --retry 1 --retry-delay 5); then
+        return 0
+    fi
+
+    while IFS= read -r transform_id; do
+        [ -n "$transform_id" ] || continue
+        for attempt in {1..3}; do
+            if so-elasticsearch-query "_transform/$transform_id?force=true" -XDELETE --fail --retry 1 --retry-delay 5; then
+                break
+            fi
+            sleep 5
+        done
+    done < <(jq -r '.transforms[]?.id' <<< "$transforms")
+}
 
 {%- for PACKAGE in SUPPORTED_PACKAGES %}
 if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && LATEST_VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
@@ -17,10 +36,25 @@ if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && L
     if [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then
         echo "{{ PACKAGE }} integration version $INSTALLED_VERSION is already at the reported latest version $LATEST_VERSION, skipping upgrade."
     else
-        echo "Upgrading {{ PACKAGE }} package to version $LATEST_VERSION..."
+{%- if PACKAGE == 'elasticsearch' %}
+        cleanup_elasticsearch_fleet_transforms
+{%- endif %}
+        echo "Upgrading {{ PACKAGE }} package from $INSTALLED_VERSION to version $LATEST_VERSION..."
         if ! elastic_fleet_package_install "{{ PACKAGE }}" "$LATEST_VERSION"; then
             PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
             PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
+        # check that package has upgraded to the expected version after install command
+        elif ! LATEST_VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
+            echo "ERROR: Failed to get latest version information for integration {{ PACKAGE }} after upgrade attempt"
+            PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
+            PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
+        elif INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then
+            echo "{{ PACKAGE }} integration upgraded to version $LATEST_VERSION."
+            PKG_UPGRADED=$((PKG_UPGRADED + 1))
+        else
+            echo "ERROR: {{ PACKAGE }} integration still at ${INSTALLED_VERSION:-unknown}; expected $LATEST_VERSION"
+            PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
+            PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
         fi
     fi
 else
@@ -30,6 +64,11 @@ else
 fi
 {%- endfor %}
 
+if [ $PKG_UPGRADED -gt 0 ]; then
+    echo "Elasticsearch template statefiles cleared after $PKG_UPGRADED package upgrade(s), so templates can reload."
+    rm -f /opt/so/state/estemplates.txt /opt/so/state/addon_estemplates.txt
+fi
+
 if [ $PKG_LOAD_FAILURES -gt 0 ]; then
     echo "ERROR: Failed to upgrade $PKG_LOAD_FAILURES package(s):"
     for PKG in "${PKG_LOAD_FAILURES_NAMES[@]}"; do

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -202,26 +202,9 @@ fi
 ### Finalization ###
 
 # Query for Enrollment Tokens for default policies
-if ENDPOINTSENROLLMENTOKEN_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
+ENDPOINTSENROLLMENTOKEN=$(elastic_fleet_active_enrollment_token "endpoints-initial") || exit 1
-    ENDPOINTSENROLLMENTOKEN=$(echo "$ENDPOINTSENROLLMENTOKEN_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
+GRIDNODESENROLLMENTOKENGENERAL=$(elastic_fleet_active_enrollment_token "so-grid-nodes_general") || exit 1
-else
+GRIDNODESENROLLMENTOKENHEAVY=$(elastic_fleet_active_enrollment_token "so-grid-nodes_heavy") || exit 1
-    echo -e "\nFailed to query for Endpoints enrollment token"
-    exit 1
-fi
-
-if GRIDNODESENROLLMENTOKENGENERAL_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
-    GRIDNODESENROLLMENTOKENGENERAL=$(echo "$GRIDNODESENROLLMENTOKENGENERAL_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_general")) | .api_key')
-else
-    echo -e "\nFailed to query for Grid nodes - General enrollment token"
-    exit 1
-fi
-
-if GRIDNODESENROLLMENTOKENHEAVY_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
-    GRIDNODESENROLLMENTOKENHEAVY=$(echo "$GRIDNODESENROLLMENTOKENHEAVY_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_heavy")) | .api_key')
-else
-    echo -e "\nFailed to query for Grid nodes - Heavy enrollment token"
-    exit 1
-fi
 
 # Store needed data in minion pillar
 pillar_file=/opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls

salt/elasticsearch/ca.sls

@@ -1,49 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states or sls in allowed_states %}
-{%   from 'vars/globals.map.jinja' import GLOBALS %}
-
-# Move our new CA over so Elastic and Logstash can use SSL with the internal CA
-catrustdir:
-  file.directory:
-    - name: /opt/so/conf/ca
-    - user: 939
-    - group: 939
-    - makedirs: True
-
-{%   if GLOBALS.is_manager %}
-# We have to add the Manager CA to the CA list
-catrustscript:
-  cmd.script:
-    - source: salt://elasticsearch/tools/sbin_jinja/so-catrust
-    - template: jinja
-    - cwd: /opt/so
-    - defaults:
-        GLOBALS: {{ GLOBALS }}
-{%   endif %}
-
-elasticsearch_cacerts:
-  file.managed:
-    - name: /opt/so/conf/ca/cacerts
-    - source: salt://elasticsearch/cacerts
-    - user: 939
-    - group: 939
-
-elasticsearch_capems:
-  file.managed:
-    - name: /opt/so/conf/ca/tls-ca-bundle.pem
-    - source: salt://elasticsearch/tls-ca-bundle.pem
-    - user: 939
-    - group: 939
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/elasticsearch/cluster.sls

@@ -98,6 +98,13 @@ so-es-cluster-settings:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
       - http: wait_for_so-elasticsearch
+
+so-elasticsearch-system-indices-patch:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-system-indices-patch
+    - require:
+      - http: wait_for_so-elasticsearch
+      - file: so-elasticsearch-system-indices-patch-script
 {%  endif %}
 
 # heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)

salt/elasticsearch/config.sls

@@ -32,24 +32,35 @@ elasticsearch:
     - gid: 930
     - home: /opt/so/conf/elasticsearch
     - createhome: False
+    - shell: /sbin/nologin
 
 elasticsearch_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://elasticsearch/tools/sbin
-    - user: 930
+    - user: root
-    - group: 939
+    - group: root
     - file_mode: 755
     - exclude_pat:
       - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state
+      - so-elasticsearch-system-indices-patch
+    - show_changes: False
+
+so-elasticsearch-system-indices-patch-script:
+  file.managed:
+    - name: /usr/sbin/so-elasticsearch-system-indices-patch
+    - source: salt://elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch
+    - user: root
+    - group: root
+    - mode: 755
     - show_changes: False
 
 elasticsearch_sbin_jinja:
   file.recurse:
     - name: /usr/sbin
     - source: salt://elasticsearch/tools/sbin_jinja
-    - user: 939
+    - user: root
-    - group: 939 
+    - group: root
     - file_mode: 755
     - template: jinja
     - exclude_pat:
@@ -62,8 +73,8 @@ so-elasticsearch-ilm-policy-load-script:
   file.managed:
     - name: /usr/sbin/so-elasticsearch-ilm-policy-load
     - source: salt://elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load
-    - user: 930
+    - user: root
-    - group: 939
+    - group: root
     - mode: 754
     - template: jinja
     - defaults:
@@ -74,8 +85,8 @@ so-elasticsearch-pipelines-script:
   file.managed:
     - name: /usr/sbin/so-elasticsearch-pipelines
     - source: salt://elasticsearch/tools/sbin/so-elasticsearch-pipelines
-    - user: 930
+    - user: root
-    - group: 939
+    - group: root
     - mode: 754
     - show_changes: False
 

salt/elasticsearch/defaults.yaml

@@ -1,7 +1,7 @@
 elasticsearch:
   enabled: false
   esheap: '600m'
-  version: 9.3.7
+  version: 9.4.8
   index_clean: true
   data_retention_method: DLM
   vm:
@@ -1160,6 +1160,7 @@ elasticsearch:
         - so-fleet_agent_id_verification-1
         - so-logs-mappings
         - so-logs-settings
+        - detections-alerts-mappings
         data_stream:
           allow_custom_routing: false
           hidden: false
@@ -3309,6 +3310,7 @@ elasticsearch:
         composed_of:
         - event-mappings
         - logs-system.security@package
+        - so-fleet_system.security_caseless-1
         - logs-system.security@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -3454,6 +3456,720 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+    so-metrics-system_x_core:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.core@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.core@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.core@custom
+        index_patterns:
+        - metrics-system.core-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.core-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_cpu:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.cpu@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.cpu@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.cpu@custom
+        index_patterns:
+        - metrics-system.cpu-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.cpu-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_diskio:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.diskio@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.diskio@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.diskio@custom
+        index_patterns:
+        - metrics-system.diskio-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.diskio-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_filesystem:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.filesystem@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.filesystem@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.filesystem@custom
+        index_patterns:
+        - metrics-system.filesystem-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.filesystem-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_fsstat:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.fsstat@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.fsstat@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.fsstat@custom
+        index_patterns:
+        - metrics-system.fsstat-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.fsstat-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_load:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.load@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.load@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.load@custom
+        index_patterns:
+        - metrics-system.load-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.load-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_memory:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.memory@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.memory@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.memory@custom
+        index_patterns:
+        - metrics-system.memory-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.memory-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_network:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.network@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.network@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.network@custom
+        index_patterns:
+        - metrics-system.network-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.network-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_ntp:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@settings
+        - metrics-system.ntp@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.ntp@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.ntp@custom
+        index_patterns:
+        - metrics-system.ntp-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.ntp-logs
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_process:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.process@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.process@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.process@custom
+        index_patterns:
+        - metrics-system.process-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.process-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_process_x_summary:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.process.summary@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.process.summary@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.process.summary@custom
+        index_patterns:
+        - metrics-system.process.summary-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.process.summary-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_socket_summary:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.socket_summary@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.socket_summary@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.socket_summary@custom
+        index_patterns:
+        - metrics-system.socket_summary-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.socket_summary-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-system_x_uptime:
+      data_stream_lifecycle:
+        data_retention: 90d
+      index_template:
+        composed_of:
+        - metrics@tsdb-settings
+        - metrics-system.uptime@package
+        - metrics@custom
+        - system@custom
+        - metrics-system.uptime@custom
+        - ecs@mappings
+        - so-fleet_integrations.ip_mappings-1
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        ignore_missing_component_templates:
+        - metrics@custom
+        - system@custom
+        - metrics-system.uptime@custom
+        index_patterns:
+        - metrics-system.uptime-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-system.uptime-logs
+              mode: time_series
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
     so-logs-windows_x_forwarded:
       index_sorting: false
       data_stream_lifecycle:
@@ -3461,6 +4177,7 @@ elasticsearch:
       index_template:
         composed_of:
         - logs-windows.forwarded@package
+        - so-fleet_process_caseless-1
         - logs-windows.forwarded@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -3510,6 +4227,7 @@ elasticsearch:
       index_template:
         composed_of:
         - logs-windows.powershell@package
+        - so-fleet_process_caseless-1
         - logs-windows.powershell@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -3559,6 +4277,7 @@ elasticsearch:
       index_template:
         composed_of:
         - logs-windows.powershell_operational@package
+        - so-fleet_process_caseless-1
         - logs-windows.powershell_operational@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -3608,6 +4327,7 @@ elasticsearch:
       index_template:
         composed_of:
         - logs-windows.sysmon_operational@package
+        - so-fleet_process_caseless-1
         - logs-windows.sysmon_operational@custom
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1

salt/elasticsearch/enabled.sls

@@ -6,6 +6,7 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
@@ -13,7 +14,6 @@
 
 include:
   - ca
-  - elasticsearch.ca
   - elasticsearch.ssl
   - elasticsearch.config
   - elasticsearch.sostatus
@@ -63,15 +63,14 @@ so-elasticsearch:
       - {{ BINDING }}
       {% endfor %}
     - binds:
+      - {{ CATRUST.dir }}
       - /opt/so/conf/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
       - /opt/so/conf/elasticsearch/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro
       - /nsm/elasticsearch:/usr/share/elasticsearch/data:rw
       - /opt/so/log/elasticsearch:/var/log/elasticsearch:rw
-      - /opt/so/conf/ca/cacerts:/usr/share/elasticsearch/jdk/lib/security/cacerts:ro
       - /etc/pki/tls/certs/intca.crt:/usr/share/elasticsearch/config/ca.crt:ro
       - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro
       - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro
-      - /etc/pki/elasticsearch.p12:/usr/share/elasticsearch/config/elasticsearch.p12:ro
       - /opt/so/conf/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles:ro
       - /opt/so/conf/elasticsearch/users:/usr/share/elasticsearch/config/users:ro
       {% if ELASTICSEARCHMERGED.config.path.get('repo', False) %}
@@ -85,21 +84,19 @@ so-elasticsearch:
         {% endfor %}
       {% endif %}
     - watch:
+      - cmd: ca_trust_extract
       - file: trusttheca
       - x509: elasticsearch_crt
       - x509: elasticsearch_key
-      - file: elasticsearch_cacerts
       - file: esyml
     - require:
       - file: trusttheca
       - x509: elasticsearch_crt
       - x509: elasticsearch_key
-      - file: elasticsearch_cacerts
       - file: esyml
       - file: eslog4jfile
       - file: nsmesdir
       - file: eslogdir
-      - file: elasticp12perms
       - cmd: auth_users_roles_inode
       - cmd: auth_users_inode
 

salt/elasticsearch/files/ingest/global@custom

@@ -1,5 +1,5 @@
 {
-    "version": 3,
+    "version": 4,
     "_meta": {
         "managed_by": "securityonion",
         "managed": true
@@ -99,7 +99,7 @@
         },
         {
             "set": {
-                "if": "ctx.tags != null && ctx.tags.contains('import')",
+                "if": "ctx.tags != null && ctx.tags.contains('import') && ctx._index != null && ctx._index.startsWith('logs-import-')",
                 "override": true,
                 "field": "data_stream.dataset",
                 "value": "import"
@@ -107,7 +107,7 @@
         },
         {
             "set": {
-                "if": "ctx.tags != null && ctx.tags.contains('import')",
+                "if": "ctx.tags != null && ctx.tags.contains('import') && ctx._index != null && ctx._index.startsWith('logs-import-')",
                 "override": true,
                 "field": "data_stream.namespace",
                 "value": "so"
@@ -177,6 +177,32 @@
                 "description": "Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"
             }
         },
+        {
+            "trim": {
+                "if": "ctx.event?.dataset == 'system.auth'",
+                "description": "Grok leaves a leading space on 'invalid user' names (elastic/integrations#12174) and, before 2.23.2, sudo padding",
+                "field": "user.name",
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        },
+        {
+            "trim": {
+                "if": "ctx.event?.dataset == 'system.auth'",
+                "description": "Appended from the untrimmed user.name",
+                "field": "related.user",
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        },
+        {
+            "script": {
+                "if": "ctx.event?.dataset == 'system.auth' && ctx.related?.user instanceof List",
+                "description": "Dedupe after trimming",
+                "source": "ctx.related.user = new ArrayList(new LinkedHashSet(ctx.related.user));",
+                "ignore_failure": true
+            }
+        },
         {
             "script": {
                 "description": "Snapshot event.ingested into _tmp.event_ingested_pre_fleet before .fleet_final_pipeline-1 overwrites it with ES ingest time",

salt/elasticsearch/files/ingest/import.evtx

@@ -0,0 +1,31 @@
+{
+  "description" : "import.evtx: normalize imported EVTX and reroute to logs-<dataset>-import",
+  "processors" : [
+    { "script":  {
+        "description": "Host from the event, not the importing node",
+        "lang": "painless",
+        "source": "Map host = ['os': ['type': 'windows', 'family': 'windows', 'platform': 'windows']]; def cn = ctx.winlog?.computer_name; if (cn != null && cn.toString().length() > 0) { String name = cn.toString(); int dot = name.indexOf('.'); if (dot > 0) { name = name.substring(0, dot); } host.put('hostname', name); host.put('name', name.toLowerCase()); } ctx.host = host;"
+    } },
+    { "script":  {
+        "description": "String event IDs, as Winlogbeat sends",
+        "lang": "painless",
+        "source": "if (ctx.winlog?.event_id != null) { ctx.winlog.event_id = ctx.winlog.event_id.toString(); } if (ctx.event?.code != null) { ctx.event.code = ctx.event.code.toString(); }"
+    } },
+    { "script":  {
+        "description": "Unnamed <Data> to param1..N, as Winlogbeat",
+        "lang": "painless",
+        "if": "ctx.winlog?.event_data?.Data instanceof Map && ctx.winlog.event_data.Data['#text'] != null",
+        "source": "def t = ctx.winlog.event_data.Data['#text']; List vals = t instanceof List ? t : [t]; for (int i = 0; i < vals.size(); i++) { ctx.winlog.event_data['param' + (i + 1)] = vals.get(i); } ctx.winlog.event_data.remove('Data');"
+    } },
+    { "script":  {
+        "description": "String values and LF line endings, as Winlogbeat",
+        "lang": "painless",
+        "if": "ctx.winlog?.event_data instanceof Map || ctx.winlog?.user_data instanceof Map",
+        "source": "String lf = String.valueOf((char) 10); String crlf = String.valueOf((char) 13) + lf; for (def key : ['event_data', 'user_data']) { def m = ctx.winlog[key]; if (!(m instanceof Map)) { continue; } for (def e : m.entrySet()) { def v = e.getValue(); if (v instanceof String) { e.setValue(v.replace(crlf, lf)); } else if (v instanceof Number || v instanceof Boolean) { e.setValue(v.toString()); } } }"
+    } },
+    { "set":     { "description": "event.kind, as Winlogbeat", "field": "event.kind", "value": "event", "override": false } },
+    { "set":     { "field": "data_stream.dataset", "copy_from": "event.dataset", "override": true, "ignore_empty_value": true } },
+    { "set":     { "field": "data_stream.namespace", "value": "import", "override": true } },
+    { "reroute": { "dataset": "{{data_stream.dataset}}", "namespace": "{{data_stream.namespace}}" } }
+  ]
+}

salt/elasticsearch/files/ingest/suricata.fileinfo

@@ -5,7 +5,8 @@
     { "rename": 	{ "field": "message2.proto", 			"target_field": "network.transport",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.fileinfo.filename", 	"target_field": "file.name",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.fileinfo.gaps", 		"target_field": "file.bytes.missing",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.fileinfo.gaps", 		"target_field": "suricata.fileinfo.gaps",	"ignore_missing": true 	} },
+    { "set":		{ "if": "ctx.suricata?.fileinfo?.gaps == false", 	"field": "file.bytes.missing",	"value": 0 		} },
     { "rename": 	{ "field": "message2.fileinfo.magic", 		"target_field": "file.mime_type",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.fileinfo.md5", 		"target_field": "hash.md5",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.fileinfo.sha1", 		"target_field": "hash.sha1",		"ignore_missing": true 	} },

salt/elasticsearch/files/log4j2.properties

@@ -20,7 +20,8 @@ appender.rolling.strategy.type = DefaultRolloverStrategy
 appender.rolling.strategy.action.type = Delete
 appender.rolling.strategy.action.basepath = /var/log/elasticsearch
 appender.rolling.strategy.action.condition.type = IfFileName
-appender.rolling.strategy.action.condition.glob = *.log.gz
+# age delete regular securityonion.log.gz and gc.log.NN files
+appender.rolling.strategy.action.condition.regex = (?:.*[.]log[.]gz|gc[.]log[.][0-9]+)
 appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified
 appender.rolling.strategy.action.condition.nested_condition.age = 7D
 

salt/elasticsearch/soc_elasticsearch.yaml

@@ -64,6 +64,43 @@ elasticsearch:
               flood_stage:
                 description: The max percentage of used disk space that will cause the node to take protective actions, such as blocking incoming events.
                 helpLink: elasticsearch
+      lifecycle:
+        default:
+          rollover:
+            description: This property accepts a key value pair formatted string and configures the conditions that would trigger a data stream to rollover when it has lifecycle configured.
+            forcedType: string
+            regex: ^max_age=(auto|[1-9][0-9]*[hd]),max_primary_shard_size=[1-9][0-9]*gb,min_docs=(0|[1-9][0-9]*),max_primary_shard_docs=[1-9][0-9]*$
+            regexFailureMessage: Must be in the format of "max_age=auto|<number><h|d>,max_primary_shard_size=<number>gb,min_docs=<number>,max_primary_shard_docs=<number>".
+            advanced: True
+            global: True
+    data_streams:
+      lifecycle:
+        poll_interval:
+          description: How often Elasticsearch checks what the next action is for all data streams with a built-in lifecycle.
+          forcedType: string
+          regex: "^[1-9][0-9]*[mhd]$"
+          regexFailureMessage: Must be a number followed by m, h, or d.
+          advanced: True
+          global: true
+          helpLink: elasticsearch
+        target:
+          merge:
+            policy:
+              merge_factor:
+                description: Data stream lifecycle implements tail merging by updating the Lucene merge policy factor for the target backing index. The merge factor is both the number of segments that should be merged together, and the maximum number of segments that we expect to find.
+                forcedType: int
+                regex: "^[1-9][0-9]*$"
+                advanced: True
+                global: true
+                helpLink: elasticsearch
+              floor_segment:
+                description: Data stream lifecycle implements tail merging by updating the Lucene merge policy floor segment for the target backing index. This floor segment size is a way to prevent indices from having a long tail of very small segments.
+                forcedType: string
+                regex: "^[1-9][0-9]*[MG]B$"
+                regexFailureMessage: Must be a number followed by MB or GB, such as 100MB.
+                advanced: True
+                global: true
+                helpLink: elasticsearch
     action:
       destructive_requires_name:
         description: Requires explicit index names when deleting indices. Prevents accidental deletion of indices via wildcard patterns.
@@ -645,6 +682,9 @@ elasticsearch:
             global: True
             advanced: True
             helpLink: elasticsearch
+    so-assistant-chat: *dataStreamSettings
+    so-assistant-session: *dataStreamSettings
+    so-elastic-agent-monitor: *dataStreamSettings
     so-logs-soc: *dataStreamSettings
     so-logs-system_x_auth: *dataStreamSettings
     so-logs-system_x_syslog: *dataStreamSettings
@@ -667,7 +707,10 @@ elasticsearch:
     so-logs-elastic_agent_x_auditbeat: *dataStreamSettings
     so-logs-elastic_agent_x_cloudbeat: *dataStreamSettings
     so-logs-elastic_agent_x_endpoint_security: *dataStreamSettings
+    so-logs-endpoint_x_actions: *dataStreamSettings
+    so-logs-endpoint_x_action_x_responses: *dataStreamSettings
     so-logs-endpoint_x_alerts: *dataStreamSettings
+    so-logs-endpoint_x_diagnostic_x_collection: *dataStreamSettings
     so-logs-endpoint_x_events_x_api: *dataStreamSettings
     so-logs-endpoint_x_events_x_file: *dataStreamSettings
     so-logs-endpoint_x_events_x_library: *dataStreamSettings
@@ -675,6 +718,7 @@ elasticsearch:
     so-logs-endpoint_x_events_x_process: *dataStreamSettings
     so-logs-endpoint_x_events_x_registry: *dataStreamSettings
     so-logs-endpoint_x_events_x_security: *dataStreamSettings
+    so-logs-endpoint_x_heartbeat: *dataStreamSettings
     so-logs-elastic_agent_x_filebeat: *dataStreamSettings
     so-logs-elastic_agent_x_fleet_server: *dataStreamSettings
     so-logs-elastic_agent_x_heartbeat: *dataStreamSettings
@@ -690,6 +734,19 @@ elasticsearch:
     so-metrics-vsphere_x_datastore: *dataStreamSettings
     so-metrics-vsphere_x_host: *dataStreamSettings
     so-metrics-vsphere_x_virtualmachine: *dataStreamSettings
+    so-metrics-system_x_core: *dataStreamSettings
+    so-metrics-system_x_cpu: *dataStreamSettings
+    so-metrics-system_x_diskio: *dataStreamSettings
+    so-metrics-system_x_filesystem: *dataStreamSettings
+    so-metrics-system_x_fsstat: *dataStreamSettings
+    so-metrics-system_x_load: *dataStreamSettings
+    so-metrics-system_x_memory: *dataStreamSettings
+    so-metrics-system_x_network: *dataStreamSettings
+    so-metrics-system_x_ntp: *dataStreamSettings
+    so-metrics-system_x_process: *dataStreamSettings
+    so-metrics-system_x_process_x_summary: *dataStreamSettings
+    so-metrics-system_x_socket_summary: *dataStreamSettings
+    so-metrics-system_x_uptime: *dataStreamSettings
     so-common: *dataStreamSettings
     so-endgame: *dataStreamSettings
     so-idh: *dataStreamSettings
@@ -880,17 +937,6 @@ elasticsearch:
                   global: True
                   advanced: True
                   helpLink: elasticsearch
-              rollover:
-                max_age:
-                  description: Maximum age of index.  Once an index reaches this limit, it will be rolled over into a new index.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-                max_primary_shard_size:
-                  description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
               shrink:
                 method:
                   description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
@@ -987,8 +1033,6 @@ elasticsearch:
             helpLink: elasticsearch
     sos-backup: *indexSettings
     so-detection: *indexSettings
-    so-assistant-chat: *indexSettings
-    so-assistant-session: *indexSettings
     so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings
       index_sorting:
         description: Sorts the index by event time, at the cost of additional processing resource consumption.

salt/elasticsearch/ssl.sls

@@ -38,10 +38,6 @@ elasticsearch_crt:
     - retry:
         attempts: 5
         interval: 30
-  cmd.run:
-    - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12 -nodes -passout pass:"
-    - onchanges:
-      - x509: /etc/pki/elasticsearch.key
 
 elastickeyperms:
   file.managed:
@@ -50,13 +46,6 @@ elastickeyperms:
     - mode: 640
     - group: 930
     
-elasticp12perms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/elasticsearch.p12
-    - mode: 640
-    - group: 930
-
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/elasticsearch/template.map.jinja

@@ -109,9 +109,15 @@
 {%     if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %}
 {%       do settings.index_template.template.settings.index.pop('sort') %}
 {%     endif %}
-{%     if DATA_RETENTION_METHOD == 'DLM' and settings.index_template.data_stream is defined and settings.data_stream_lifecycle is defined %}
+{%     if DATA_RETENTION_METHOD == 'DLM' and settings.index_template.data_stream is defined %}
-{%       if settings.data_stream_lifecycle.data_retention is defined and settings.data_stream_lifecycle.data_retention %}
+{#       Addon defaults are generated without data_stream_lifecycle, so fall back to global defaults. #}
-{%         do settings.index_template.template.update({'lifecycle': {'data_retention': settings.data_stream_lifecycle.data_retention}}) %}
+{%       if settings.data_stream_lifecycle is defined %}
+{%         set DATA_STREAM_LIFECYCLE = settings.data_stream_lifecycle %}
+{%       else %}
+{%         set DATA_STREAM_LIFECYCLE = DEFAULT_GLOBAL_OVERRIDES.data_stream_lifecycle %}
+{%       endif %}
+{%       if DATA_STREAM_LIFECYCLE.data_retention is defined and DATA_STREAM_LIFECYCLE.data_retention %}
+{%         do settings.index_template.template.update({'lifecycle': {'data_retention': DATA_STREAM_LIFECYCLE.data_retention}}) %}
 {%       else %}
 {%         do settings.index_template.template.update({'lifecycle': {}}) %}
 {%       endif %}

salt/elasticsearch/templates/component/elastic-agent/so-fleet_process_caseless-1.json

@@ -0,0 +1,123 @@
+{
+  "_meta": {
+    "managed_by": "security_onion",
+    "managed": true,
+    "description": "Adds .caseless for Lucene queries. Restates each field's package type and .text."
+  },
+  "template": {
+    "mappings": {
+      "properties": {
+        "process": {
+          "properties": {
+            "executable": {
+              "type": "keyword",
+              "ignore_above": 1024,
+              "fields": {
+                "caseless": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "normalizer": "lowercase"
+                },
+                "text": {
+                  "type": "match_only_text"
+                }
+              }
+            },
+            "name": {
+              "type": "keyword",
+              "ignore_above": 1024,
+              "fields": {
+                "caseless": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "normalizer": "lowercase"
+                },
+                "text": {
+                  "type": "match_only_text"
+                }
+              }
+            },
+            "command_line": {
+              "type": "wildcard",
+              "ignore_above": 1024,
+              "fields": {
+                "caseless": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "normalizer": "lowercase"
+                },
+                "text": {
+                  "type": "match_only_text"
+                }
+              }
+            },
+            "parent": {
+              "properties": {
+                "executable": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "fields": {
+                    "caseless": {
+                      "type": "keyword",
+                      "ignore_above": 1024,
+                      "normalizer": "lowercase"
+                    },
+                    "text": {
+                      "type": "match_only_text"
+                    }
+                  }
+                },
+                "name": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "fields": {
+                    "caseless": {
+                      "type": "keyword",
+                      "ignore_above": 1024,
+                      "normalizer": "lowercase"
+                    },
+                    "text": {
+                      "type": "match_only_text"
+                    }
+                  }
+                },
+                "command_line": {
+                  "type": "wildcard",
+                  "ignore_above": 1024,
+                  "fields": {
+                    "caseless": {
+                      "type": "keyword",
+                      "ignore_above": 1024,
+                      "normalizer": "lowercase"
+                    },
+                    "text": {
+                      "type": "match_only_text"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
+        "file": {
+          "properties": {
+            "path": {
+              "type": "keyword",
+              "ignore_above": 1024,
+              "fields": {
+                "caseless": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "normalizer": "lowercase"
+                },
+                "text": {
+                  "type": "match_only_text"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/elastic-agent/so-fleet_system.security_caseless-1.json

@@ -0,0 +1,80 @@
+{
+  "_meta": {
+    "managed_by": "security_onion",
+    "managed": true,
+    "description": "Adds .caseless for Lucene queries. Keeps each field's existing keyword type."
+  },
+  "template": {
+    "mappings": {
+      "properties": {
+        "process": {
+          "properties": {
+            "command_line": {
+              "type": "keyword",
+              "ignore_above": 1024,
+              "fields": {
+                "caseless": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "normalizer": "lowercase"
+                }
+              }
+            },
+            "parent": {
+              "properties": {
+                "executable": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "fields": {
+                    "caseless": {
+                      "type": "keyword",
+                      "ignore_above": 1024,
+                      "normalizer": "lowercase"
+                    }
+                  }
+                },
+                "name": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "fields": {
+                    "caseless": {
+                      "type": "keyword",
+                      "ignore_above": 1024,
+                      "normalizer": "lowercase"
+                    }
+                  }
+                },
+                "command_line": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "fields": {
+                    "caseless": {
+                      "type": "keyword",
+                      "ignore_above": 1024,
+                      "normalizer": "lowercase"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
+        "file": {
+          "properties": {
+            "path": {
+              "type": "keyword",
+              "ignore_above": 1024,
+              "fields": {
+                "caseless": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "normalizer": "lowercase"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/so/detection-mappings.json

@@ -154,6 +154,20 @@
             }
           }
         }
+      },
+      "runtime": {
+        "so_detection.sigmaKind": {
+          "type": "keyword",
+          "script": {
+            "source": "if (doc['so_detection.language'].size() == 0 || doc['so_detection.language'].value != 'sigma') { return; } def d = params._source.so_detection; if (d == null || d.content == null) { return; } String c = d.content; String NL = String.valueOf((char) 10); emit(c.startsWith('correlation:') || c.contains(NL + 'correlation:') ? 'correlation' : 'single');"
+          }
+        },
+        "so_detection.sigmaCorrelationType": {
+          "type": "keyword",
+          "script": {
+            "source": "if (doc['so_detection.language'].size() == 0 || doc['so_detection.language'].value != 'sigma') { return; } def d = params._source.so_detection; if (d == null || d.content == null) { return; } String c = d.content; String NL = String.valueOf((char) 10); int i = c.startsWith('correlation:') ? 0 : c.indexOf(NL + 'correlation:'); if (i < 0) { return; } i = c.indexOf(NL, i + 1); while (i >= 0 && i + 1 < c.length()) { int e = c.indexOf(NL, i + 1); String line = e < 0 ? c.substring(i + 1) : c.substring(i + 1, e); if (line.length() > 0 && !Character.isWhitespace(line.charAt(0)) && line.charAt(0) != (char) 35) { return; } String t = line.trim(); if (t.startsWith('type:')) { String v = t.substring(5).trim(); int h = v.indexOf(' ' + String.valueOf((char) 35)); if (h >= 0) { v = v.substring(0, h).trim(); } if (v.length() > 1 && (v.charAt(0) == (char) 39 || v.charAt(0) == (char) 34)) { v = v.substring(1, v.length() - 1); } emit(v); return; } i = e; }"
+          }
+        }
       }
     }
   },

salt/elasticsearch/templates/component/so/detections-alerts-mappings.json

@@ -0,0 +1,149 @@
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "tags": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "sigma_level": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "rule": {
+          "properties": {
+            "name": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "uuid": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "category": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "product": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "service": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "correlation": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
+        },
+        "event": {
+          "properties": {
+            "severity": {
+              "type": "long"
+            },
+            "severity_label": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "module": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "dataset": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "kind": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "reason": {
+              "type": "match_only_text",
+              "fields": {
+                "keyword": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                }
+              }
+            },
+            "original": {
+              "type": "keyword",
+              "index": false,
+              "doc_values": false
+            }
+          }
+        },
+        "event_data": {
+          "properties": {
+            "@timestamp": {
+              "type": "date"
+            },
+            "window_start": {
+              "type": "date"
+            },
+            "event_count": {
+              "type": "long"
+            },
+            "value_count": {
+              "type": "long"
+            },
+            "event_type_count": {
+              "type": "long"
+            },
+            "value_sum": {
+              "type": "double"
+            },
+            "value_avg": {
+              "type": "double"
+            },
+            "value_percentile": {
+              "type": "double"
+            },
+            "value_median": {
+              "type": "double"
+            }
+          }
+        },
+        "labels": {
+          "properties": {
+            "correlation_group_by": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "correlation_group": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
+        },
+        "related": {
+          "properties": {
+            "ip": {
+              "type": "ip"
+            },
+            "user": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "hosts": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
+        },
+        "error": {
+          "properties": {
+            "message": {
+              "type": "match_only_text"
+            }
+          }
+        }
+      }
+    }
+  },
+  "_meta": {
+    "description": "Fields written by the ElastAlert SecurityOnionESAlerter to logs-detections.alerts-so"
+  }
+}

salt/elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch

@@ -0,0 +1,203 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+set -eo pipefail
+
+SETTINGS='{"index":{"auto_expand_replicas":"0-1"}}'
+KIBANA_PASSWORD=
+INDEX_PATTERNS=(
+  '.entity_analytics.risk_score.lookup-*'
+  '.entity_analytics.watchlists.*'
+  '.entity_analytics.monitoring.users-*'
+  '.entity_analytics.entity-leads-*'
+  '.asset-criticality.asset-criticality-*'
+  '.workflows-executions'
+  '.workflows-step-executions'
+  '.entities.v2.latest.security_*'
+  '.entities.v2.history.security_*'
+  'risk-score.risk-score-latest-*'
+  '.metrics-endpoint.metadata_united_*'
+  '.metrics-endpoint.metadata_current_*'
+)
+DATA_STREAM_PATTERNS=(
+  '.entities.v2.updates.security_*'
+  'risk-score.risk-score-*'
+  '.rule-events'
+  '.alert-actions'
+)
+TEMPLATE_PATTERNS=(
+  'entities_v2_latest_security_default_index_template'
+  'entities_v2_history_security_default_index_template'
+  '.entities_v2_updates_security_default_index_template'
+  '.risk-score.risk-score-default-index-template'
+  '.rule-events'
+  '.alert-actions'
+  '.metrics-endpoint.metadata_united_default-template'
+  '.metrics-endpoint.metadata_current_default-template'
+)
+
+query_es() {
+  if so-elasticsearch-query "$@" --fail --retry 3 --retry-delay 5; then
+    return 0
+  fi
+
+  # retry failed attempts with so_kibana user (system managed indices reject so_elastic user)
+  local query_path="$1"
+  shift
+
+  if [[ -z "$KIBANA_PASSWORD" ]]; then
+    KIBANA_PASSWORD=$(salt-call pillar.get elasticsearch:auth:users:so_kibana_user:pass --out=newline_values_only)
+  fi
+  [[ -n "$KIBANA_PASSWORD" ]] || return 1
+
+  echo "Retrying ${query_path} as so_kibana." >&2
+  curl -K /opt/so/conf/elasticsearch/curl.config --user "so_kibana:${KIBANA_PASSWORD}" \
+    -s -k -L --fail --retry 3 --retry-delay 5 -H 'Content-Type: application/json' "https://localhost:9200/${query_path}" "$@"
+}
+
+# add auto_expand_replicas=0-1 to given index
+set_auto_expand_replicas() {
+  local index="$1"
+
+  echo "Setting auto_expand_replicas to 0-1 on ${index}."
+  query_es "${index}/_settings" -XPUT -d "$SETTINGS" >/dev/null
+}
+
+# resolve index patterns and find each index with an unassigned replica
+unassigned_replicas() {
+  local pattern="$1"
+  local resolved_indices response index
+
+  if ! resolved_indices=$(query_es "_resolve/index/${pattern}?expand_wildcards=all" 2>/dev/null); then
+    return 0
+  fi
+
+  while read -r index; do
+    if ! response=$(query_es "_cat/shards/${index}?format=json&h=index,prirep,state" 2>/dev/null); then
+      continue
+    fi
+    jq -r '.[]? | objects | select(.prirep == "r" and .state == "UNASSIGNED") | .index' <<<"$response"
+  done < <(jq -r '.indices[]?.name' <<<"$resolved_indices")
+}
+
+data_stream_indices() {
+  local pattern="$1"
+  local response
+
+  if ! response=$(query_es "_data_stream/${pattern}?expand_wildcards=all" 2>/dev/null); then
+    return 0
+  fi
+  jq -r '.data_streams[]?.indices[]?.index_name' <<<"$response"
+}
+
+update_system_indices() {
+  local pattern="$1"
+  local index
+
+  while read -r index; do
+    [[ -n "$index" ]] && set_auto_expand_replicas "$index"
+  done < <(unassigned_replicas "$pattern")
+}
+
+# update data stream backing indices with unassigned replicas
+update_system_ds() {
+  local pattern="$1"
+  local index
+
+  while read -r index; do
+    while read -r unassigned_index; do
+      [[ -n "$unassigned_index" ]] && set_auto_expand_replicas "$unassigned_index"
+    done < <(unassigned_replicas "$index")
+  done < <(data_stream_indices "$pattern")
+}
+
+has_unassigned_replicas() {
+  local pattern="$1"
+  local index
+
+  index=$(unassigned_replicas "$pattern" | sed -n '1p')
+  [[ -n "$index" ]]
+}
+
+data_stream_has_unassigned_replicas() {
+  local pattern="$1"
+  local index
+  while read -r index; do
+    has_unassigned_replicas "$index" && return 0
+  done < <(data_stream_indices "$pattern")
+
+  return 1
+}
+
+needs_patch() {
+  local pattern
+  for pattern in "${INDEX_PATTERNS[@]}"; do
+    has_unassigned_replicas "$pattern" && return 0
+  done
+  for pattern in "${DATA_STREAM_PATTERNS[@]}"; do
+    data_stream_has_unassigned_replicas "$pattern" && return 0
+  done
+
+  return 1
+}
+
+# get index templates, update with auto_expand_replicas=0-1, and PUT back. Keeping mappings/settings/aliases in-place
+update_system_templates() {
+  local pattern="$1"
+  local templates name response template auto_expand_replicas
+
+  if ! templates=$(query_es "_index_template/${pattern}" 2>/dev/null); then
+    return 0
+  fi
+  while read -r name; do
+    response=$(query_es "_index_template/${name}")
+    template=$(jq -c '.index_templates[0].index_template' <<<"$response")
+    auto_expand_replicas=$(jq -r '.template.settings["index.auto_expand_replicas"] // .template.settings.index.auto_expand_replicas // empty' <<<"$template")
+    [[ "$auto_expand_replicas" == "0-1" ]] && continue
+
+    template=$(jq '
+            if (.template.settings.index | type) == "object" then
+                .template.settings.index.auto_expand_replicas = "0-1"
+            else
+                .template.settings["index.auto_expand_replicas"] = "0-1"
+            end
+            | del(.created_date_millis, .modified_date_millis)
+        ' <<<"$template")
+    echo "Setting auto_expand_replicas to 0-1 on index template ${name}."
+    query_es "_index_template/${name}" -XPUT -d "$template" >/dev/null
+  done < <(jq -r '.index_templates[]?.name' <<<"$templates")
+}
+
+if [[ "${1:-}" == "--check" ]]; then
+  needs_patch
+  exit $?
+fi
+
+if [[ $# -ne 0 ]]; then
+  echo "Usage: $0 [--check]" >&2
+  exit 1
+fi
+
+patched=false
+for pattern in "${INDEX_PATTERNS[@]}"; do
+  if has_unassigned_replicas "$pattern"; then
+    update_system_indices "$pattern"
+    patched=true
+  fi
+done
+
+for pattern in "${DATA_STREAM_PATTERNS[@]}"; do
+  if data_stream_has_unassigned_replicas "$pattern"; then
+    update_system_ds "$pattern"
+    patched=true
+  fi
+done
+
+if [[ "$patched" == true ]]; then
+  for pattern in "${TEMPLATE_PATTERNS[@]}"; do
+    update_system_templates "$pattern"
+  done
+fi

salt/elasticsearch/tools/sbin_jinja/so-catrust

@@ -1,26 +0,0 @@
-#!/bin/bash
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-get_elastic_agent_vars
-
-# Exit on errors, since all lines must succeed
-set -e
-
-# Check to see if we have extracted the ca cert.
-if [ ! -f /opt/so/saltstack/local/salt/elasticsearch/cacerts ]; then
-    docker run -v /etc/pki/ca.crt:/etc/ssl/ca.crt --name so-elasticsearchca --user root --entrypoint jdk/bin/keytool {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:$ELASTIC_AGENT_TARBALL_VERSION -keystore /usr/share/elasticsearch/jdk/lib/security/cacerts -alias SOSCA -import -file /etc/ssl/ca.crt -storepass changeit -noprompt
-    # Make sure symbolic links are followed when copying from container
-    docker cp -L so-elasticsearchca:/usr/share/elasticsearch/jdk/lib/security/cacerts /opt/so/saltstack/local/salt/elasticsearch/cacerts
-    docker cp -L so-elasticsearchca:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /opt/so/saltstack/local/salt/elasticsearch/tls-ca-bundle.pem
-    docker rm so-elasticsearchca
-    echo "" >> /opt/so/saltstack/local/salt/elasticsearch/tls-ca-bundle.pem
-    echo "sosca" >> /opt/so/saltstack/local/salt/elasticsearch/tls-ca-bundle.pem
-    cat /etc/pki/ca.crt >> /opt/so/saltstack/local/salt/elasticsearch/tls-ca-bundle.pem
-else
-    exit 0
-fi

salt/firewall/iptables.jinja

@@ -4,11 +4,19 @@
 {%- set role = GLOBALS.role.split('-')[1] %}
 {%- from 'firewall/containers.map.jinja' import NODE_CONTAINERS %}
 
+{%- set NODE_NETWORKS = [] %}
+{%- for NETNAME, NETWORK in DOCKERMERGED.networks.items() %}
+{%-   if not NETWORK.get('manager_only') or GLOBALS.get('is_manager', False) %}
+{%-     do NODE_NETWORKS.append(NETNAME) %}
+{%-   endif %}
+{%- endfor %}
+
 {%- set PR = [] %}
 {%- set D1 = [] %}
 {%- set D2 = [] %}
 {%- for container in NODE_CONTAINERS %}
 {%-   set IP = DOCKERMERGED.containers[container].ip %}
+{%-   set BRIDGE = DOCKERMERGED.containers[container].network %}
 {%-   if DOCKERMERGED.containers[container].port_bindings is defined %}
 {%-     for binding in DOCKERMERGED.containers[container].port_bindings %}
 {#-       cant split int so we convert to string #}
@@ -35,11 +43,11 @@
 {%-         endif %}
 {%-         do PR.append("-A POSTROUTING -s " ~ DOCKERMERGED.containers[container].ip ~ "/32 -d " ~ DOCKERMERGED.containers[container].ip ~ "/32 -p " ~  proto ~ " -m " ~ proto  ~ " --dport " ~ containerPort ~ " -j MASQUERADE") %}
 {%-         if bindip | length and bindip != '0.0.0.0' %}
-{%-           do D1.append("-A DOCKER -d " ~ bindip ~ "/32 ! -i sobridge -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
+{%-           do D1.append("-A DOCKER -d " ~ bindip ~ "/32 ! -i " ~ BRIDGE ~ " -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
 {%-         else %}
-{%-           do D1.append("-A DOCKER ! -i sobridge -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
+{%-           do D1.append("-A DOCKER ! -i " ~ BRIDGE ~ " -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
 {%-         endif %}
-{%-         do D2.append("-A DOCKER -d " ~ DOCKERMERGED.containers[container].ip ~ "/32 ! -i sobridge -o sobridge -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ containerPort ~ " -j ACCEPT") %}
+{%-         do D2.append("-A DOCKER -d " ~ DOCKERMERGED.containers[container].ip ~ "/32 ! -i " ~ BRIDGE ~ " -o " ~ BRIDGE ~ " -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ containerPort ~ " -j ACCEPT") %}
 {%-     endfor %}
 {%-   endif %}
 {%- endfor %}
@@ -52,11 +60,15 @@
 :DOCKER - [0:0]
 -A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER
 -A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER
--A POSTROUTING -s {{DOCKERMERGED.range}} ! -o sobridge -j MASQUERADE
+{%- for NETNAME in NODE_NETWORKS %}
+-A POSTROUTING -s {{ DOCKERMERGED.networks[NETNAME].range }} ! -o {{ NETNAME }} -j MASQUERADE
+{%- endfor %}
 {%- for rule in PR %}
 {{ rule }}
 {%- endfor %}
--A DOCKER -i sobridge -j RETURN
+{%- for NETNAME in NODE_NETWORKS %}
+-A DOCKER -i {{ NETNAME }} -j RETURN
+{%- endfor %}
 {%- for rule in D1 %}
 {{ rule }}
 {%- endfor %}
@@ -97,10 +109,12 @@ COMMIT
 {%- endif %}
 -A FORWARD -j DOCKER-USER
 -A FORWARD -j DOCKER-ISOLATION-STAGE-1
--A FORWARD -o sobridge -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+{%- for NETNAME in NODE_NETWORKS %}
--A FORWARD -o sobridge -j DOCKER
+-A FORWARD -o {{ NETNAME }} -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
--A FORWARD -i sobridge ! -o sobridge -j ACCEPT
+-A FORWARD -o {{ NETNAME }} -j DOCKER
--A FORWARD -i sobridge -o sobridge -j ACCEPT
+-A FORWARD -i {{ NETNAME }} ! -o {{ NETNAME }} -j ACCEPT
+-A FORWARD -i {{ NETNAME }} -o {{ NETNAME }} -j ACCEPT
+{%- endfor %}
 -A FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
 -A FORWARD -i lo -j ACCEPT
 -A FORWARD -m conntrack --ctstate INVALID -j DROP
@@ -112,13 +126,18 @@ COMMIT
 {%- for rule in D2 %}
 {{ rule }}
 {%- endfor %}
-
+{% for NETNAME in NODE_NETWORKS %}
--A DOCKER-ISOLATION-STAGE-1 -i sobridge ! -o sobridge -j DOCKER-ISOLATION-STAGE-2
+-A DOCKER-ISOLATION-STAGE-1 -i {{ NETNAME }} ! -o {{ NETNAME }} -j DOCKER-ISOLATION-STAGE-2
+{%- endfor %}
 -A DOCKER-ISOLATION-STAGE-1 -j RETURN
--A DOCKER-ISOLATION-STAGE-2 -o sobridge -j DROP
+{%- for NETNAME in NODE_NETWORKS %}
+-A DOCKER-ISOLATION-STAGE-2 -o {{ NETNAME }} -j DROP
+{%- endfor %}
 -A DOCKER-ISOLATION-STAGE-2 -j RETURN
--A DOCKER-USER ! -i sobridge -o sobridge -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+{%- for NETNAME in NODE_NETWORKS %}
--A DOCKER-USER ! -i sobridge -o sobridge -j LOGGING
+-A DOCKER-USER ! -i {{ NETNAME }} -o {{ NETNAME }} -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+-A DOCKER-USER ! -i {{ NETNAME }} -o {{ NETNAME }} -j LOGGING
+{%- endfor %}
 -A DOCKER-USER -j RETURN
 -A LOGGING -m limit --limit 2/min -j LOG --log-prefix "IPTables-dropped: "
 -A LOGGING -j DROP

salt/firewall/map.jinja

@@ -4,8 +4,12 @@
 
 {# add our ip to self #}
 {% do FIREWALL_DEFAULT.firewall.hostgroups.self.append(GLOBALS.node_ip) %}
-{# add dockernet range #}
+{# add dockernet ranges #}
-{% do FIREWALL_DEFAULT.firewall.hostgroups.dockernet.append(DOCKERMERGED.range) %}
+{% for NETNAME, NETWORK in DOCKERMERGED.networks.items() %}
+{%   if not NETWORK.get('manager_only') or GLOBALS.get('is_manager', False) %}
+{%     do FIREWALL_DEFAULT.firewall.hostgroups.dockernet.append(NETWORK.range) %}
+{%   endif %}
+{% endfor %}
 
 {% if GLOBALS.role == 'so-idh' %}
 {%   from 'idh/opencanary_config.map.jinja' import IDH_PORTGROUPS %}

salt/hydra/enabled.sls

@@ -13,9 +13,11 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   if 'api' in salt['pillar.get']('features', []) %}
 
 include:
+  - ca.trust
   - hydra.config
   - hydra.sostatus
 
@@ -26,9 +28,10 @@ so-hydra:
     - hostname: hydra
     - name: so-hydra
     - networks:
-      - sobridge:
+      - soauth:
-        - ipv4_address: {{ DOCKERMERGED.containers['so-hydra'].ip }}
+        - ipv4_address: {{ DOCKERMERGED.containers['so-hydra'].ips['soauth'] }}
     - binds:
+      - {{ CATRUST.dir }}
       - /opt/so/conf/hydra/:/hydra-conf:ro
       - /opt/so/log/hydra/:/hydra-log:rw
       - /nsm/hydra/db:/hydra-data:rw
@@ -60,6 +63,7 @@ so-hydra:
     {%   endfor %}
     {% endif %}
     - watch:
+      - cmd: ca_trust_extract
       - file: hydraconfig
     - require:
       - file: hydraconfig
@@ -73,7 +77,7 @@ delete_so-hydra_so-status.disabled:
 
 wait_for_hydra:
   http.wait_for_successful_query:
-    - name: 'http://{{ GLOBALS.manager }}:4444/health/alive'
+    - name: 'http://{{ DOCKERMERGED.containers['so-hydra'].ips['soauth'] }}:4444/health/alive'
     - ssl: True
     - verify_ssl: False
     - status:

salt/hypervisor/init.sls

@@ -21,12 +21,16 @@ hypervisor_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://hypervisor/tools/sbin
+    - user: root
+    - group: root
     - file_mode: 744
 
 hypervisor_sbin_jinja:
   file.recurse:
     - name: /usr/sbin
     - source: salt://hypervisor/tools/sbin_jinja
+    - user: root
+    - group: root
     - template: jinja
     - file_mode: 744
 

salt/idh/config.sls

@@ -86,8 +86,8 @@ idh_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://idh/tools/sbin
-    - user: 939
+    - user: root
-    - group: 939
+    - group: root
     - file_mode: 755
 
 #idh_sbin_jinja:

salt/idh/enabled.sls

@@ -6,9 +6,11 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 
 include:
+  - ca.trust
   - idh.config
   - idh.sostatus
 
@@ -20,6 +22,7 @@ so-idh:
     - detach: True
     - network_mode: host
     - binds:
+      - {{ CATRUST.dir }}
       - /nsm/idh:/var/tmp:rw
       - /opt/so/conf/idh/http-skins:/opt/opencanary/http-skins:ro
       - /opt/so/conf/idh/opencanary.conf:/etc/opencanaryd/opencanary.conf:ro
@@ -34,12 +37,13 @@ so-idh:
       - {{ XTRAHOST }}
       {% endfor %}
     {% endif %}
-    {% if DOCKERMERGED.containers['so-idh'].extra_env %}
     - environment:
-      {% for XTRAENV in DOCKERMERGED.containers['so-idh'].extra_env %}
+      - REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt
+      {% if DOCKERMERGED.containers['so-idh'].extra_env %}
+        {% for XTRAENV in DOCKERMERGED.containers['so-idh'].extra_env %}
       - {{ XTRAENV }}
-      {% endfor %}
+        {% endfor %}
-    {% endif %}
+      {% endif %}
     {% if DOCKERMERGED.containers['so-idh'].ulimits %}
     - ulimits:
     {%   for ULIMIT in DOCKERMERGED.containers['so-idh'].ulimits %}
@@ -47,6 +51,7 @@ so-idh:
     {%   endfor %}
     {% endif %}
     - watch:
+      - cmd: ca_trust_extract
       - file: opencanary_config
     - require:
       - file: opencanary_config

salt/influxdb/config.sls

@@ -41,8 +41,8 @@ influxdb_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://influxdb/tools/sbin
-    - user: 939
+    - user: root
-    - group: 939
+    - group: root
     - file_mode: 755
 
 #influxdb_sbin_jinja:

salt/influxdb/enabled.sls

@@ -6,11 +6,13 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   set PASSWORD = salt['pillar.get']('secrets:influx_pass') %}
 {%   set TOKEN = salt['pillar.get']('influxdb:token') %}
 
 include:
+  - ca.trust
   - influxdb.ssl
   - influxdb.config
   - influxdb.sostatus
@@ -38,6 +40,7 @@ so-influxdb:
         {% endfor %}
       {% endif %}
     - binds:
+      - {{ CATRUST.pem }}:/etc/ssl/certs/ca-certificates.crt:ro
       - /opt/so/log/influxdb/:/log:rw
       - /opt/so/conf/influxdb/config.yaml:/conf/config.yaml:ro
       - /opt/so/conf/influxdb/etc:/etc/influxdb2:rw
@@ -66,6 +69,7 @@ so-influxdb:
     {%   endfor %}
     {% endif %}
     - watch:
+      - cmd: ca_trust_extract
       - file: influxdbconf
       - x509: influxdb_key
       - x509: influxdb_crt
@@ -94,9 +98,11 @@ metrics_link_file:
       - docker_container: so-influxdb
 
 # Install cron job to determine size of influxdb for telegraf
+# telegraf reads this while the cron rewrites it, so write aside and rename rather than
+# truncating in place. tgraflogdir recurses ownership, so the temp file is chowned to match
 get_influxdb_size:
   cron.present:
-    - name: 'du -s -k /nsm/influxdb | cut -f1 > /opt/so/log/telegraf/influxdb_size.log 2>&1'
+    - name: 'du -s -k /nsm/influxdb | cut -f1 > /opt/so/log/telegraf/influxdb_size.log.tmp 2>&1; chown 939:939 /opt/so/log/telegraf/influxdb_size.log.tmp; mv -f /opt/so/log/telegraf/influxdb_size.log.tmp /opt/so/log/telegraf/influxdb_size.log'
     - identifier: get_influxdb_size
     - user: root
     - minute: '*/1'

salt/kafka/config.sls

@@ -21,6 +21,7 @@ kafka_user:
     - gid: 960
     - home: /opt/so/conf/kafka
     - createhome: False
+    - shell: /sbin/nologin
 
 kafka_home_dir:
   file.absent:
@@ -30,16 +31,16 @@ kafka_sbin_tools:
   file.recurse:
     - name: /usr/sbin
     - source: salt://kafka/tools/sbin
-    - user: 960
+    - user: root
-    - group: 960
+    - group: root
     - file_mode: 755
 
 kafka_sbin_jinja_tools:
   file.recurse:
     - name: /usr/sbin
     - source: salt://kafka/tools/sbin_jinja
-    - user: 960
+    - user: root
-    - group: 960
+    - group: root
     - file_mode: 755
     - template: jinja
     - defaults:

salt/kafka/enabled.sls

@@ -12,12 +12,14 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   set KAFKANODES = salt['pillar.get']('kafka:nodes') %}
 {%   set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %}
 {%   if 'gmd' in salt['pillar.get']('features', []) %}
 
 include:
+  - ca.trust
   - kafka.ca
   - kafka.config
   - kafka.ssl
@@ -51,6 +53,7 @@ so-kafka:
       - {{ BINDING }}
       {% endfor %}
     - binds:
+      - {{ CATRUST.java }}:/opt/java/openjdk/lib/security/cacerts:ro
       - /etc/pki/kafka.p12:/etc/pki/kafka.p12:ro
       - /opt/so/conf/kafka/kafka-truststore.jks:/etc/pki/kafka-truststore.jks:ro
       - /nsm/kafka/data/:/nsm/kafka/data/:rw
@@ -61,6 +64,11 @@ so-kafka:
       {% if KAFKA_EXTERNAL_ACCESS %}
       - /opt/so/conf/kafka/kafka_server_jaas.conf:/opt/kafka/config/kafka_server_jaas.conf:ro
       {% endif %}
+      {% if DOCKERMERGED.containers['so-kafka'].custom_bind_mounts %}
+        {% for BIND in DOCKERMERGED.containers['so-kafka'].custom_bind_mounts %}
+      - {{ BIND }}
+        {% endfor %}
+      {% endif %}
     {% if DOCKERMERGED.containers['so-kafka'].ulimits %}
     - ulimits:
     {%   for ULIMIT in DOCKERMERGED.containers['so-kafka'].ulimits %}
@@ -68,6 +76,7 @@ so-kafka:
     {%   endfor %}
     {% endif %}
     - watch:
+      - cmd: ca_trust_extract
       {% for sc in ['server', 'client'] %}
       - file: kafka_kraft_{{sc}}_properties
       {% endfor %}

salt/kibana/config.sls

@@ -22,6 +22,7 @@ kibana:
     - gid: 932
     - home: /opt/so/conf/kibana
     - createhome: False
+    - shell: /sbin/nologin
 
 # Drop the correct nginx config based on role
 
@@ -36,16 +37,16 @@ kibana_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://kibana/tools/sbin
-    - user: 932
+    - user: root
-    - group: 939
+    - group: root
     - file_mode: 755
 
 kibana_sbin_jinja:
   file.recurse:
     - name: /usr/sbin
     - source: salt://kibana/tools/sbin_jinja
-    - user: 932
+    - user: root
-    - group: 939 
+    - group: root
     - file_mode: 755
     - template: jinja
     - defaults:

salt/kibana/defaults.yaml

@@ -22,7 +22,7 @@ kibana:
           - default
           - file
     migrations:
-      discardCorruptObjects: "9.3.7"
+      discardCorruptObjects: "9.4.8"
     telemetry:
       enabled: False
     xpack:

salt/kibana/enabled.sls

@@ -8,8 +8,10 @@
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 
 include:
+  - ca.trust
   - kibana.config
   - kibana.sostatus
 
@@ -27,6 +29,7 @@ so-kibana:
       - ELASTICSEARCH_HOST={{ GLOBALS.manager }}
       - ELASTICSEARCH_PORT=9200
       - MANAGER={{ GLOBALS.manager }}
+      - NODE_OPTIONS=--use-openssl-ca
       {% if DOCKERMERGED.containers['so-kibana'].extra_env %}
         {% for XTRAENV in DOCKERMERGED.containers['so-kibana'].extra_env %}
       - {{ XTRAENV }}
@@ -40,6 +43,7 @@ so-kibana:
       {% endfor %}
     {% endif %}
     - binds:
+      - {{ CATRUST.dir }}
       - /opt/so/conf/kibana/etc:/usr/share/kibana/config:rw
       - /opt/so/log/kibana:/var/log/kibana:rw
       - /opt/so/conf/kibana/customdashboards:/usr/share/kibana/custdashboards:ro
@@ -60,6 +64,7 @@ so-kibana:
     {%   endfor %}
     {% endif %}
     - watch:
+      - cmd: ca_trust_extract
       - file: kibanaconfig
 
 wait_for_so-kibana:

salt/kibana/files/saved_objects.ndjson

@@ -1,5 +1,5 @@
 {"attributes":{"fieldFormatMap":"{\"match_body.source_ip\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'\\\"{{value}}\\\"')),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"match_body.destination_ip\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'\\\"{{value}}\\\"')),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}}}","fields":"[{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"alert_info.slack_username_override\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"alert_info.slack_username_override.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"alert_info.slack_webhook_url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"alert_info.slack_webhook_url.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"alert_info.type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"alert_info.type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"alert_sent\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"alert_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"data.rule\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"data.rule.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"endtime\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"exponent\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"hits\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"match_body.@timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"match_body.@version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"match_body.@version.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"match_body.signature_info\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"match_body.signature_info.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"match_body._id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"match_body._id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"match_body._index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\Line truncated
-{"attributes":{"fieldFormatMap":"{\"_id\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"https://PLACEHOLDER/#/hunt?q=_id%3A%22{{value}}%22\",\"labelTemplate\":\"Hunt and optionally pivot to PCAP/Cases\"}},\"uid\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:{{value}})),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"source_ip\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'\\\"{{value}}\\\"')),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"destination_ip\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'\\\"{{value}}\\\"')),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"source_port\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'source_port:\\\"{{value}}\\\" OR destination_port:\\\"{{value}}\\\"')),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"destination_port\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'source_port:\\\"{{value}}\\\" OR destination_port:\\\"{{value}}\\\"')),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"fuid\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:{{value}})),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"resp_fuids\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:{{value}})),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"orig_fuids\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:{{value}})),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"sid\":{\"id\":\"number\",\"params\":{\"pattern\":\"0\"}},\"port\":{\"id\":\"number\",\"params\":{\"pattern\":\"0.[000]\"}},\"query\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:{{value}})),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"query.keyword\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:{{value}})),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"server_name\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:{{value}})),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"server_name.keyword\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:OLine truncated
+{"attributes":{"fieldFormatMap":"{\"_id\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"https://PLACEHOLDER/#/hunt?q=_id%3A%22{{value}}%22\",\"labelTemplate\":\"Hunt and optionally pivot to PCAP/Cases\"}},\"uid\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:{{value}})),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"source_ip\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'\\\"{{value}}\\\"')),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"destination_ip\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'\\\"{{value}}\\\"')),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"source_port\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'source_port:\\\"{{value}}\\\" OR destination_port:\\\"{{value}}\\\"')),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"destination_port\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'source_port:\\\"{{value}}\\\" OR destination_port:\\\"{{value}}\\\"')),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"fuid\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:{{value}})),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"resp_fuids\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:{{value}})),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"orig_fuids\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:{{value}})),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"sid\":{\"id\":\"number\",\"params\":{\"pattern\":\"0\"}},\"port\":{\"id\":\"number\",\"params\":{\"pattern\":\"0.[000]\"}},\"query\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:{{value}})),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"query.keyword\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:{{value}})),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"server_name\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(_source),index:'*:logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:{{value}})),sort:!('@timestamp',desc))\",\"labelTemplate\":\"{{value}}\"}},\"server_name.keyword\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"/kibana/app/dashboards#/view/68563ed0-34bf-11e7-9b32-bb903919ead9?_g=(refreshInterval:(display:OLine truncated
 {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"Security Onion - Network Data","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Security Onion - Network Data\",\"type\":\"markdown\",\"aggs\":[],\"params\":{\"fontSize\":12,\"openLinksInNewTab\":false,\"markdown\":\"[Home](/kibana/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645)     \\n \\n**Datasets**    \\n[Connections](/kibana/app/dashboards#/view/0cc628b0-6e9f-11ea-9266-1fd14ca6af34) |  [DCE/RPC](/kibana/app/dashboards#/view/9e882df0-72c5-11ea-8dd2-9d8795a1200b) |\\n[DHCP](/kibana/app/dashboards#/view/80625c10-96dd-11ea-814e-bb515e873c2c)    \\n[DNP3](/kibana/app/dashboards#/view/b1f52180-755a-11ea-9565-7315f4ee5cac)  |  [DNS](/kibana/app/dashboards#/view/55ac6bf0-6ec4-11ea-9266-1fd14ca6af34) |\\n[FTP](/kibana/app/dashboards#/view/739bfad0-755a-11ea-9565-7315f4ee5cac) |\\n[HTTP](/kibana/app/dashboards#/view/44e9c820-6eb1-11ea-9266-1fd14ca6af34) | [Intel](/kibana/app/dashboards#/view/85b529a0-0e5a-11eb-a255-e1e8e85e3571) | [IRC](/kibana/app/dashboards#/view/38523560-75ba-11ea-9565-7315f4ee5cac)  |\\n[Kerberos](/kibana/app/dashboards#/view/b207ab90-75bc-11ea-9565-7315f4ee5cac)   \\n[Modbus](/kibana/app/dashboards#/view/886a7b90-75bd-11ea-9565-7315f4ee5cac) | \\n[MySQL](/kibana/app/dashboards#/view/c3ced6d0-75be-11ea-9565-7315f4ee5cac)  | \\n[NTLM](/kibana/app/dashboards#/view/558292e0-75c1-11ea-9565-7315f4ee5cac)  | \\n[PE](/kibana/app/dashboards#/view/94b55b90-c761-11ea-bebb-37c5ab5894ea) |\\n[RADIUS](/kibana/app/dashboards#/view/b9769e60-75c4-11ea-9565-7315f4ee5cac) | [RDP](/kibana/app/dashboards#/view/5b743150-75c5-11ea-9565-7315f4ee5cac) | \\n[RFB](/kibana/app/dashboards#/view/c8b3c360-75c6-11ea-9565-7315f4ee5cac) | [SIP](/kibana/app/dashboards#/view/dd98e260-75c6-11ea-9565-7315f4ee5cac)     \\n[SMB](/kibana/app/dashboards#/view/f24d7b80-75c6-11ea-9565-7315f4ee5cac) | [SMTP](/kibana/app/dashboards#/view/00304500-75e7-11ea-9565-7315f4ee5cac) | [SNMP](/kibana/app/dashboards#/view/96522610-75e8-11ea-9565-7315f4ee5cac) | \\n[SSH](/kibana/app/dashboards#/view/9dfd77e0-75eb-11ea-9565-7315f4ee5cac) | [SSL](/kibana/app/dashboards#/view/efae8de0-75eb-11ea-9565-7315f4ee5cac) | [Syslog](/kibana/app/dashboards#/view/66499a20-75ed-11ea-9565-7315f4ee5cac) | [Tunnels](/kibana/app/dashboards#/view/c962dd60-75ed-11ea-9565-7315f4ee5cac) | [X.509](/kibana/app/dashboards#/view/2e0865f0-75ee-11ea-9565-7315f4ee5cac)   \\n\"}}"},"coreMigrationVersion":"8.7.1","created_at":"2023-07-20T15:26:57.180Z","id":"7f822930-6ea4-11ea-9266-1fd14ca6af34","migrationVersion":{"visualization":"8.5.0"},"references":[],"sort":[1689866817180,4251],"type":"visualization","updated_at":"2023-07-20T15:26:57.180Z","version":"WzQwNzgsMV0="}
 {"attributes":{"allowNoIndex":true,"fieldFormatMap":"{\"Ransomware.child_processes.pid\":{\"id\":\"string\"},\"Ransomware.pid\":{\"id\":\"string\"},\"Responses.@timestamp\":{\"id\":\"string\"},\"Target.process.parent.pgid\":{\"id\":\"string\"},\"Target.process.parent.pid\":{\"id\":\"string\"},\"Target.process.parent.ppid\":{\"id\":\"string\"},\"Target.process.parent.thread.id\":{\"id\":\"string\"},\"Target.process.pgid\":{\"id\":\"string\"},\"Target.process.pid\":{\"id\":\"string\"},\"Target.process.ppid\":{\"id\":\"string\"},\"Target.process.thread.id\":{\"id\":\"string\"},\"event.sequence\":{\"id\":\"string\"},\"event.severity\":{\"id\":\"string\"},\"process.parent.pgid\":{\"id\":\"string\"},\"process.parent.pid\":{\"id\":\"string\"},\"process.parent.ppid\":{\"id\":\"string\"},\"process.parent.thread.id\":{\"id\":\"string\"},\"process.pgid\":{\"id\":\"string\"},\"process.pid\":{\"id\":\"string\"},\"process.ppid\":{\"id\":\"string\"},\"process.thread.id\":{\"id\":\"string\"},\"threat.enrichments.indicator.file.elf.header.entrypoint\":{\"id\":\"string\"},\"threat.enrichments.indicator.file.elf.sections.chi2\":{\"id\":\"number\"},\"threat.enrichments.indicator.file.elf.sections.entropy\":{\"id\":\"number\"},\"threat.enrichments.indicator.file.elf.sections.physical_size\":{\"id\":\"bytes\"},\"threat.enrichments.indicator.file.elf.sections.virtual_address\":{\"id\":\"string\"},\"threat.enrichments.indicator.file.elf.sections.virtual_size\":{\"id\":\"string\"},\"threat.enrichments.indicator.url.port\":{\"id\":\"string\"},\"threat.indicator.file.elf.header.entrypoint\":{\"id\":\"string\"},\"threat.indicator.file.elf.sections.chi2\":{\"id\":\"number\"},\"threat.indicator.file.elf.sections.entropy\":{\"id\":\"number\"},\"threat.indicator.file.elf.sections.physical_size\":{\"id\":\"bytes\"},\"threat.indicator.file.elf.sections.virtual_address\":{\"id\":\"string\"},\"threat.indicator.file.elf.sections.virtual_size\":{\"id\":\"string\"},\"threat.indicator.url.port\":{\"id\":\"string\"},\"destination.bytes\":{\"id\":\"bytes\"},\"destination.port\":{\"id\":\"string\"},\"http.request.body.bytes\":{\"id\":\"bytes\"},\"http.request.bytes\":{\"id\":\"bytes\"},\"http.response.body.bytes\":{\"id\":\"bytes\"},\"http.response.bytes\":{\"id\":\"bytes\"},\"http.response.status_code\":{\"id\":\"string\"},\"network.bytes\":{\"id\":\"bytes\"},\"source.bytes\":{\"id\":\"bytes\"},\"source.port\":{\"id\":\"string\"}}","fields":"[{\"name\":\"cloud.account.id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cloud.availability_zone\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cloud.instance.id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cloud.instance.name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cloud.machine.type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cloud.provider\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cloud.region\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cloud.project.id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"cloud.image.id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"container.id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"container.image.name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"container.labels\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"container.name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host.architecture\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host.domain\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host.hosLine truncated
 {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Security Onion - All Logs","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Security Onion - All Logs\",\"type\":\"metric\",\"params\":{\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"type\":\"range\",\"from\":0,\"to\":10000}],\"labels\":{\"show\":true},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":29}},\"dimensions\":{\"metrics\":[{\"type\":\"vis_dimension\",\"accessor\":0,\"format\":{\"id\":\"number\",\"params\":{}}}]},\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}]}"},"coreMigrationVersion":"8.7.1","created_at":"2023-07-20T15:26:57.180Z","id":"d04b5130-6e99-11ea-9266-1fd14ca6af34","migrationVersion":{"visualization":"8.5.0"},"references":[{"id":"logs-*","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1689866817180,4254],"type":"visualization","updated_at":"2023-07-20T15:26:57.180Z","version":"WzQwODAsMV0="}

salt/kibana/tools/sbin_jinja/so-kibana-space-defaults

@@ -9,5 +9,5 @@ SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http:
 # Disable certain Features from showing up in the Kibana UI
 echo
 echo "Setting up default Kibana Space:"
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV1","entityManager","streams","cloudConnect","slo"]} ' >> /opt/so/log/kibana/misc.log
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","securitySolutionCasesV3","inventory","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV4","securitySolutionAlertsV1","entityManager","slo","streams","anonymization","searchInferenceEndpoints","cloudConnect","queryActivity","automatic_import","stackAlerts","monitoring","dataQuality","actions"]} ' >> /opt/so/log/kibana/misc.log
 echo

salt/kratos/config.sls

@@ -27,6 +27,7 @@ kratos:
     - uid: 928
     - gid: 928
     - home: /opt/so/conf/kratos
+    - shell: /sbin/nologin
 
 kratosdir:
   file.directory:

salt/kratos/enabled.sls

@@ -7,8 +7,10 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 
 include:
+  - ca.trust
   - kratos.config
   - kratos.sostatus
 
@@ -19,9 +21,10 @@ so-kratos:
     - hostname: kratos
     - name: so-kratos
     - networks:
-      - sobridge:
+      - soauth:
-        - ipv4_address: {{ DOCKERMERGED.containers['so-kratos'].ip }}
+        - ipv4_address: {{ DOCKERMERGED.containers['so-kratos'].ips['soauth'] }}
     - binds:
+      - {{ CATRUST.dir }}
       - /opt/so/conf/kratos/:/kratos-conf:ro
       - /opt/so/log/kratos/:/kratos-log:rw
       - /nsm/kratos/db:/kratos-data:rw
@@ -53,6 +56,7 @@ so-kratos:
     {%   endfor %}
     {% endif %}
     - watch:
+      - cmd: ca_trust_extract
       - file: kratosschema
       - file: kratosconfig
     - require:
@@ -71,7 +75,7 @@ delete_so-kratos_so-status.disabled:
 
 wait_for_kratos:
   http.wait_for_successful_query:
-    - name: 'http://{{ GLOBALS.manager }}:4434/'
+    - name: 'http://{{ DOCKERMERGED.containers['so-kratos'].ips['soauth'] }}:4434/'
     - ssl: True
     - verify_ssl: False
     - status:

salt/libvirt/64962/init.sls

@@ -6,6 +6,8 @@ so-fix-salt-ldap_script:
   file.managed:
     - name: /usr/sbin/so-fix-salt-ldap.py
     - source: salt://libvirt/64962/scripts/so-fix-salt-ldap.py
+    - user: root
+    - group: root
     - mode: 744
 
 fix-salt-ldap:

salt/logrotate/defaults.yaml

@@ -220,6 +220,36 @@ logrotate:
       - extension .log
       - dateext
       - dateyesterday
+    /opt/so/log/salt/virtual_node_manager:
+      - daily
+      - rotate 14
+      - missingok
+      - copytruncate
+      - compress
+      - create
+      - extension .log
+      - dateext
+      - dateyesterday
+    /opt/so/log/salt/so-salt-cloud:
+      - daily
+      - rotate 14
+      - missingok
+      - copytruncate
+      - compress
+      - create
+      - extension .log
+      - dateext
+      - dateyesterday
+    /opt/so/log/salt/so-soup-grid-highstate:
+      - daily
+      - rotate 14
+      - missingok
+      - copytruncate
+      - compress
+      - create
+      - extension .log
+      - dateext
+      - dateyesterday
     /nsm/idh/*_x_log:
       - daily
       - rotate 14

salt/logrotate/soc_logrotate.yaml

@@ -140,6 +140,27 @@ logrotate:
       multiline: True
       global: True
       forcedType: "[]string"
+    "/opt/so/log/salt/virtual_node_manager":
+      description: List of logrotate options for this file.
+      title: /opt/so/log/salt/virtual_node_manager
+      advanced: True
+      multiline: True
+      global: True
+      forcedType: "[]string"
+    "/opt/so/log/salt/so-salt-cloud":
+      description: List of logrotate options for this file.
+      title: /opt/so/log/salt/so-salt-cloud
+      advanced: True
+      multiline: True
+      global: True
+      forcedType: "[]string"
+    "/opt/so/log/salt/so-soup-grid-highstate":
+      description: List of logrotate options for this file.
+      title: /opt/so/log/salt/so-soup-grid-highstate
+      advanced: True
+      multiline: True
+      global: True
+      forcedType: "[]string"
     "/nsm/idh/*_x_log":
       description: List of logrotate options for this file.
       title: /nsm/idh/*.log

salt/logstash/config.sls

@@ -35,13 +35,14 @@ logstash:
     - uid: 931
     - gid: 931
     - home: /opt/so/conf/logstash
+    - shell: /sbin/nologin
 
 logstash_sbin:
   file.recurse:
     - name: /usr/sbin
     - source: salt://logstash/tools/sbin
-    - user: 931
+    - user: root
-    - group: 939
+    - group: root
     - file_mode: 755
 
 #logstash_sbin_jinja:
@@ -81,6 +82,14 @@ ls_custom_pipeline_conf_{{assigned_pipeline}}_{{pipeline}}:
 
 
 {% for assigned_pipeline in ASSIGNED_PIPELINES %}
+{# a blank per-pipeline setting falls back to the global logstash.yml value #}
+{% set PARSED_OVERRIDES = LOGSTASH_MERGED.get('pipeline_settings', {}).get(assigned_pipeline, {}) %}
+{% if PARSED_OVERRIDES is not mapping %}
+{%   do salt.log.warning('logstash: ignoring malformed pipeline_settings for pipeline ' ~ assigned_pipeline ~ '; expected a set of settings') %}
+{% endif %}
+{% set PIPELINE_OVERRIDES = PARSED_OVERRIDES if PARSED_OVERRIDES is mapping else {} %}
+{% set THREADS = PIPELINE_OVERRIDES.get('pipeline_x_workers') or LOGSTASH_MERGED.config.pipeline_x_workers %}
+{% set BATCH = PIPELINE_OVERRIDES.get('pipeline_x_batch_x_size') or LOGSTASH_MERGED.config.pipeline_x_batch_x_size %}
     {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %}
 ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}:
   file.managed:
@@ -92,8 +101,8 @@ ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_")
         GLOBALS: {{ GLOBALS }}
         ES_USER: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') }}"
         ES_PASS: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') }}"
-        THREADS: {{ LOGSTASH_MERGED.config.pipeline_x_workers }}
+        THREADS: {{ THREADS }}
-        BATCH: {{ LOGSTASH_MERGED.config.pipeline_x_batch_x_size }}
+        BATCH: {{ BATCH }}
       {% else %}
     - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1]}}
       {% endif %}
@@ -125,6 +134,14 @@ lspipelinesyml:
     - defaults:
         ASSIGNED_PIPELINES: {{ ASSIGNED_PIPELINES }}
 
+lslog4j2:
+  file.managed:
+    - name: /opt/so/conf/logstash/etc/log4j2.properties
+    - source: salt://logstash/etc/log4j2.properties.jinja
+    - template: jinja
+    - user: 931
+    - group: 939
+
 lsetcsync:
   file.recurse:
     - name: /opt/so/conf/logstash/etc
@@ -133,7 +150,11 @@ lsetcsync:
     - group: 939
     - template: jinja
     - clean: True
-    - exclude_pat: pipelines*
+{#- both names are matched: the .jinja source so the recurse does not copy it verbatim,
+    and the rendered file so clean: True does not delete what lslog4j2 wrote #}
+    - exclude_pat:
+      - pipelines*
+      - log4j2.properties*
     - defaults:
         LOGSTASH_MERGED: {{ LOGSTASH_MERGED }}
 

salt/logstash/defaults.yaml

@@ -42,6 +42,11 @@ logstash:
     custom2: []
     custom3: []
     custom4: []
+    custom5: []
+    custom6: []
+    custom7: []
+    custom8: []
+    custom9: []
   pipeline_config:
     custom001: |-
       filter {
@@ -60,10 +65,405 @@ logstash:
     custom008: PLACEHOLDER
     custom009: PLACEHOLDER
     custom010: PLACEHOLDER
+  pipeline_settings:
+    fleet:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    manager:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    receiver:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    search:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom0:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom1:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom2:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom3:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom4:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom5:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom6:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom7:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom8:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom9:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
   settings:
     lsheap: 500m
   config:
     api_x_http_x_host: 0.0.0.0
+    log_x_level: info
+    log_x_format: plain
     path_x_logs: /var/log/logstash
     pipeline_x_workers: 1
     pipeline_x_batch_x_size: 125

salt/logstash/enabled.sls

@@ -6,6 +6,7 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'logstash/map.jinja' import LOGSTASH_MERGED %}
 {%   from 'logstash/map.jinja' import LOGSTASH_NODES %}
@@ -13,9 +14,6 @@
 
 include:
   - ca
-{%   if GLOBALS.role not in ['so-receiver','so-fleet'] %}
-  - elasticsearch.ca
-{%   endif %}
 {# Kafka ca runs on nodes that can run logstash for Kafka input / output. Only when Kafka is global pipeline #}
 {%   if GLOBALS.role in ['so-searchnode', 'so-manager', 'so-managerhype', 'so-managersearch', 'so-receiver', 'so-standalone'] and GLOBALS.pipeline == 'KAFKA' %}
   - kafka.ca
@@ -58,6 +56,8 @@ so-logstash:
       - {{ BINDING }}
       {% endfor %}
     - binds:
+      - {{ CATRUST.dir }}
+      - {{ CATRUST.java }}:/usr/share/logstash/jdk/lib/security/cacerts:ro
       - /opt/so/conf/elasticsearch/templates/:/templates/:ro
       - /opt/so/conf/logstash/etc/:/usr/share/logstash/config/:ro
       - /opt/so/conf/logstash/pipelines:/usr/share/logstash/pipelines:ro
@@ -78,10 +78,6 @@ so-logstash:
       - /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro
           {% endif %}
       {% endif %}
-      {% if GLOBALS.role not in ['so-receiver','so-fleet'] %}
-      - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro
-      - /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro
-      {% endif %}
       {% if GLOBALS.pipeline == "KAFKA" and GLOBALS.role in ['so-manager', 'so-managerhype', 'so-managersearch', 'so-standalone', 'so-searchnode'] %}
       - /etc/pki/kafka-logstash.p12:/usr/share/logstash/kafka-logstash.p12:ro
       - /opt/so/conf/kafka/kafka-truststore.jks:/etc/pki/kafka-truststore.jks:ro
@@ -104,12 +100,11 @@ so-logstash:
     {%   endfor %}
     {% endif %}
     - watch:
+      - cmd: ca_trust_extract
       - file: lsetcsync
+      - file: lslog4j2
+      - file: lspipelinesyml
       - file: trusttheca
-      {% if GLOBALS.is_manager %}
-      - file: elasticsearch_cacerts
-      - file: elasticsearch_capems
-      {% endif %}
       {% if GLOBALS.is_manager or GLOBALS.role in ['so-fleet', 'so-heavynode', 'so-receiver'] %}
       - x509: etc_elasticfleet_logstash_crt
       - x509: etc_elasticfleet_logstash_key
@@ -131,10 +126,6 @@ so-logstash:
       {% endif %}
     - require:
       - file: trusttheca
-      {% if GLOBALS.is_manager %}
-      - file: elasticsearch_cacerts
-      - file: elasticsearch_capems
-      {% endif %}
       {% if GLOBALS.is_manager or GLOBALS.role in ['so-fleet', 'so-heavynode', 'so-receiver'] %}
       - x509: etc_elasticfleet_logstash_crt
       - x509: etc_elasticfleet_logstash_key

salt/logstash/etc/log4j2.properties.jinja

@@ -1,3 +1,4 @@
+{%- from 'logstash/map.jinja' import LOGSTASH_MERGED -%}
 status = error
 name = LogstashPropertiesConfig
 
@@ -16,8 +17,14 @@ name = LogstashPropertiesConfig
 appender.rolling.type = RollingFile
 appender.rolling.name = rolling
 appender.rolling.fileName = /var/log/logstash/logstash.log
+{%- if LOGSTASH_MERGED.config.get('log_x_format', 'plain') == 'json' %}
+appender.rolling.layout.type = JSONLayout
+appender.rolling.layout.compact = true
+appender.rolling.layout.eventEol = true
+{%- else %}
 appender.rolling.layout.type = PatternLayout
 appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %.10000m%n
+{%- endif %}
 appender.rolling.filePattern = /var/log/logstash/logstash-%d{yyyy-MM-dd}.log.gz
 appender.rolling.policies.type = Policies
 appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
@@ -32,7 +39,5 @@ appender.rolling.strategy.action.condition.type = IfFileName
 appender.rolling.strategy.action.condition.glob = *.gz
 appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified
 appender.rolling.strategy.action.condition.nested_condition.age = 7D
-rootLogger.level = info
+rootLogger.level = ${sys:ls.log.level}
 rootLogger.appenderRef.rolling.ref = rolling
-#rootLogger.level = ${sys:ls.log.level}
-#rootLogger.appenderRef.console.ref = ${sys:ls.log.format}_console

salt/logstash/etc/pipelines.yml.jinja

@@ -1,4 +1,17 @@
+{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
+{%- set PIPELINE_SETTINGS = LOGSTASH_MERGED.get('pipeline_settings', {}) %}
 {%- for assigned_pipeline in ASSIGNED_PIPELINES %}
 - pipeline.id: {{ assigned_pipeline }}
   path.config: "/usr/share/logstash/pipelines/{{ assigned_pipeline }}/"
+{%-   set extra = PIPELINE_SETTINGS.get(assigned_pipeline, {}) %}
+{%-   if extra is mapping %}
+{#-     values are emitted unquoted so yaml re-infers the type logstash expects:
+        4 as an integer, false as a boolean, 1024mb and auto as strings #}
+{%-     for key, value in extra | dictsort %}
+{%-       set rendered = key | replace('_x_', '.') %}
+{%-       if value not in ['', None] and rendered not in ['pipeline.id', 'path.config'] %}
+  {{ rendered }}: {{ value }}
+{%-       endif %}
+{%-     endfor %}
+{%-   endif %}
 {% endfor -%}