add logrotate for virtual_node_manager and so-salt-cloud

.github/DISCUSSION_TEMPLATE/3-0.yml

@@ -11,8 +11,6 @@ body:
         -
         - 3.0.0
         - 3.1.0
-        - 3.2.0
-        - 3.3.0
         - Other (please provide detail below)
     validations:
       required: true

.github/workflows/pythontest.yml

@@ -5,7 +5,6 @@ on:
     paths:
       - "salt/sensoroni/files/analyzers/**"
       - "salt/manager/tools/sbin/**"
-      - "salt/_beacons/**"
 
 jobs:
   build:
@@ -15,7 +14,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.14"]
-        python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin", "salt/_beacons"]
+        python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin"]
 
     steps:
     - uses: actions/checkout@v3

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 3.2.0-20260729 ISO image released on 2026/07/29
+### 3.0.0-20260331 ISO image released on 2026/03/31
 
 
 ### Download and Verify
 
-3.2.0-20260729 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-3.2.0-20260729.iso
+3.0.0-20260331 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
  
-MD5: B1E10F46DF872B655C29325DF965A4DB  
-SHA1: 0F3C7ED80F6D326B7A993C2F899B986320C01BF9  
-SHA256: 7465163C1D1ADFCDC3935530EAFB312E987C016941ADC11841B214553314D1FF  
+MD5: ECD318A1662A6FDE0EF213F5A9BD4B07  
+SHA1: E55BE314440CCF3392DC0B06BC5E270B43176D9C  
+SHA256: 7FC47405E335CBE5C2B6C51FE7AC60248F35CBE504907B8B5A33822B23F8F4D5  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.2.0-20260729.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.2.0-20260729.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-3.2.0-20260729.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-3.2.0-20260729.iso.sig securityonion-3.2.0-20260729.iso
+gpg --verify securityonion-3.0.0-20260331.iso.sig securityonion-3.0.0-20260331.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Tue 28 Jul 2026 06:17:34 PM EDT using RSA key ID FE507013
+gpg: Signature made Mon 30 Mar 2026 06:22:14 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

HOTFIX

@@ -1 +0,0 @@
-

VERSION

@@ -1 +1 @@
-3.3.0
+3.1.0

pillar/data/addtotab.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+# This script adds sensors/nodes/etc to the nodes tab
+default_salt_dir=/opt/so/saltstack/default
+local_salt_dir=/opt/so/saltstack/local
+TYPE=$1
+NAME=$2
+IPADDRESS=$3
+CPUS=$4
+GUID=$5
+MANINT=$6
+ROOTFS=$7
+NSM=$8
+MONINT=$9
+#NODETYPE=$10
+#HOTNAME=$11
+
+echo "Seeing if this host is already in here. If so delete it"
+if grep -q $NAME "$local_salt_dir/pillar/data/$TYPE.sls"; then
+  echo "Node Already Present - Let's re-add it"
+  awk -v blah="  $NAME:" 'BEGIN{ print_flag=1 }
+{
+    if( $0 ~ blah )
+    {
+       print_flag=0;
+       next
+    }
+    if( $0 ~ /^  [a-zA-Z0-9]+:$/ )
+    {
+        print_flag=1;
+    }
+    if ( print_flag == 1 )
+        print $0
+
+} ' $local_salt_dir/pillar/data/$TYPE.sls > $local_salt_dir/pillar/data/tmp.$TYPE.sls
+mv $local_salt_dir/pillar/data/tmp.$TYPE.sls $local_salt_dir/pillar/data/$TYPE.sls
+echo "Deleted $NAME from the tab. Now adding it in again with updated info"
+fi
+echo "  $NAME:" >> $local_salt_dir/pillar/data/$TYPE.sls
+echo "    ip: $IPADDRESS" >> $local_salt_dir/pillar/data/$TYPE.sls
+echo "    manint: $MANINT" >> $local_salt_dir/pillar/data/$TYPE.sls
+echo "    totalcpus: $CPUS" >> $local_salt_dir/pillar/data/$TYPE.sls
+echo "    guid: $GUID" >> $local_salt_dir/pillar/data/$TYPE.sls
+echo "    rootfs: $ROOTFS" >> $local_salt_dir/pillar/data/$TYPE.sls
+echo "    nsmfs: $NSM" >> $local_salt_dir/pillar/data/$TYPE.sls
+if [ $TYPE == 'sensorstab' ]; then
+  echo "    monint: bond0" >> $local_salt_dir/pillar/data/$TYPE.sls
+fi
+if [ $TYPE == 'evaltab' ] || [ $TYPE == 'standalonetab' ]; then
+  echo "    monint: bond0" >> $local_salt_dir/pillar/data/$TYPE.sls
+  if [ ! $10 ]; then
+    salt-call state.apply utility queue=True
+  fi
+fi
+if [ $TYPE == 'nodestab' ]; then
+  salt-call state.apply elasticsearch queue=True
+#  echo "    nodetype: $NODETYPE" >> $local_salt_dir/pillar/data/$TYPE.sls
+#  echo "    hotname: $HOTNAME" >> $local_salt_dir/pillar/data/$TYPE.sls
+fi

pillar/top.sls

@@ -3,8 +3,6 @@ base:
     - ca
     - global.soc_global
     - global.adv_global
-    - salt.soc_salt
-    - salt.adv_salt
     - docker.soc_docker
     - docker.adv_docker
     - influxdb.token

salt/_beacons/postgres_pillar_beacon.py

@@ -1,142 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Custom salt beacon that watches the SOC audit_settings table in postgres for
-# new settings changes and emits a beacon event per new row. This replaces the
-# inotify watch on /opt/so/saltstack/local/pillar -- instead of monitoring pillar
-# files on disk, we monitor the securityonion.audit_settings table that SOC writes to.
-#
-# Detection is poll-based with a monotonic `id` watermark persisted to
-# WATERMARK_FILE: each pass selects rows with id greater than the last id seen,
-# which makes it self-healing (a missed poll simply catches up on the next one).
-#
-# Each emitted event carries setting_id and node_id; the push_pillar reactor maps
-# setting_id -> app via pillar_push_map.yaml and writes a push intent, after which
-# the existing so-push-drainer / orch.push_batch pipeline takes over unchanged.
-
-import logging
-import os
-import subprocess
-
-log = logging.getLogger(__name__)
-
-WATERMARK_FILE = '/opt/so/state/postgres_pillar_beacon_watch.id'
-CONTAINER = 'so-postgres'
-DATABASE = 'securityonion'
-
-# Unaligned, tuples-only psql output with a field separator that cannot appear in
-# an id/setting_id/node_id, so we can split each row reliably.
-FIELD_SEP = '\x1f'
-
-
-def __virtual__():
-    return True
-
-
-def validate(config):
-    return True, 'valid'
-
-
-def _read_watermark():
-    # Returns the last processed id, or None if the watermark has not been seeded.
-    try:
-        with open(WATERMARK_FILE, 'r') as f:
-            return int((f.read() or '').strip())
-    except (IOError, ValueError):
-        return None
-
-
-def _write_watermark(value):
-    try:
-        os.makedirs(os.path.dirname(WATERMARK_FILE), exist_ok=True)
-        tmp = WATERMARK_FILE + '.tmp'
-        with open(tmp, 'w') as f:
-            f.write(str(int(value)))
-        os.rename(tmp, WATERMARK_FILE)
-    except OSError:
-        log.exception('postgres_pillar_beacon: failed to persist watermark to %s', WATERMARK_FILE)
-
-
-def _query(sql):
-    # Run a query against securityonion inside the so-postgres container over the unix
-    # socket (trust auth, no password). Returns stdout on success, or None on any
-    # failure so the caller can no-op and retry on the next interval.
-    cmd = [
-        'docker', 'exec', CONTAINER,
-        'psql', '-U', 'postgres', '-d', DATABASE,
-        '-tA', '-F', FIELD_SEP, '-c', sql,
-    ]
-    try:
-        result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
-    except subprocess.TimeoutExpired:
-        log.warning('postgres_pillar_beacon: psql timed out')
-        return None
-    except Exception:
-        log.exception('postgres_pillar_beacon: failed to exec psql')
-        return None
-    if result.returncode != 0:
-        log.warning('postgres_pillar_beacon: psql failed (rc=%s): %s',
-                    result.returncode, (result.stderr or '').strip())
-        return None
-    return result.stdout
-
-
-def beacon(config):  # noqa: C901
-    retval = []
-
-    watermark = _read_watermark()
-
-    # First run / missing watermark: seed to the current MAX(id) and emit nothing
-    # so we never replay the entire settings history into a fleetwide push.
-    if watermark is None:
-        seed = _query('SELECT COALESCE(MAX(id), 0) FROM audit_settings;')
-        if seed is None:
-            return retval  # postgres not ready yet; retry next interval
-        try:
-            _write_watermark(int((seed or '0').strip() or 0))
-        except ValueError:
-            log.warning('postgres_pillar_beacon: could not parse MAX(id) seed: %r', seed)
-        return retval
-
-    rows = _query(
-        "SELECT id, setting_id, COALESCE(node_id, '') FROM audit_settings "
-        "WHERE id > %d ORDER BY id;" % watermark
-    )
-    if rows is None:
-        return retval
-
-    max_id = watermark
-    for line in rows.splitlines():
-        # Do NOT str.strip() the whole line: Python treats the \x1f field
-        # separator (and \x1c-\x1e) as whitespace, so stripping would eat an
-        # empty trailing node_id field and make the row look malformed.
-        if not line.strip():
-            continue
-        parts = line.split(FIELD_SEP)
-        if len(parts) < 3:
-            log.warning('postgres_pillar_beacon: skipping malformed row: %r', line)
-            continue
-        try:
-            row_id = int(parts[0])
-        except ValueError:
-            log.warning('postgres_pillar_beacon: skipping row with non-int id: %r', line)
-            continue
-        setting_id = parts[1]
-        node_id = parts[2]
-        retval.append({
-            'tag': 'audit_settings',
-            'id': row_id,
-            'setting_id': setting_id,
-            'node_id': node_id,
-        })
-        if row_id > max_id:
-            max_id = row_id
-
-    if max_id > watermark:
-        _write_watermark(max_id)
-        log.info('postgres_pillar_beacon: emitted %d change(s), watermark %d -> %d',
-                 len(retval), watermark, max_id)
-
-    return retval

salt/_beacons/postgres_pillar_beacon_test.py

@@ -1,165 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-import os
-import shutil
-import subprocess
-import tempfile
-import unittest
-from unittest.mock import patch
-
-import postgres_pillar_beacon
-
-
-class TestPostgresPillarBeacon(unittest.TestCase):
-
-    def setUp(self):
-        # Point WATERMARK_FILE at a throwaway dir so the real read/write helpers
-        # (and their os.makedirs/os.rename) run against actual files, then clean
-        # it all up in tearDown.
-        self.tmpdir = tempfile.mkdtemp()
-        self.watermark = os.path.join(self.tmpdir, 'state', 'watch.id')
-        patcher = patch.object(postgres_pillar_beacon, 'WATERMARK_FILE', self.watermark)
-        patcher.start()
-        self.addCleanup(patcher.stop)
-
-    def tearDown(self):
-        shutil.rmtree(self.tmpdir, ignore_errors=True)
-
-    # -- trivial contract -------------------------------------------------
-
-    def test_virtual_returns_true(self):
-        self.assertTrue(postgres_pillar_beacon.__virtual__())
-
-    def test_validate_returns_valid(self):
-        self.assertEqual(postgres_pillar_beacon.validate({}), (True, 'valid'))
-
-    # -- _read_watermark --------------------------------------------------
-
-    def test_read_watermark_valid(self):
-        postgres_pillar_beacon._write_watermark(42)
-        self.assertEqual(postgres_pillar_beacon._read_watermark(), 42)
-
-    def test_read_watermark_missing_file_returns_none(self):
-        # tmp watermark file was never created
-        self.assertIsNone(postgres_pillar_beacon._read_watermark())
-
-    def test_read_watermark_garbage_returns_none(self):
-        os.makedirs(os.path.dirname(self.watermark), exist_ok=True)
-        with open(self.watermark, 'w') as f:
-            f.write('nope')
-        self.assertIsNone(postgres_pillar_beacon._read_watermark())
-
-    # -- _write_watermark -------------------------------------------------
-
-    def test_write_watermark_round_trip(self):
-        postgres_pillar_beacon._write_watermark(7)
-        with open(self.watermark) as f:
-            self.assertEqual(f.read(), '7')
-
-    def test_write_watermark_swallows_oserror(self):
-        with patch.object(postgres_pillar_beacon.os, 'makedirs', side_effect=OSError):
-            # Must not raise; failure is logged and the beacon retries next pass.
-            postgres_pillar_beacon._write_watermark(5)
-        self.assertFalse(os.path.exists(self.watermark))
-
-    # -- _query -----------------------------------------------------------
-
-    def test_query_success_returns_stdout_and_builds_argv(self):
-        completed = subprocess.CompletedProcess(args=[], returncode=0, stdout='rows', stderr='')
-        with patch.object(postgres_pillar_beacon.subprocess, 'run', return_value=completed) as mock_run:
-            result = postgres_pillar_beacon._query('SELECT 1;')
-        self.assertEqual(result, 'rows')
-        argv = mock_run.call_args[0][0]
-        self.assertEqual(argv[:5], ['docker', 'exec', 'so-postgres', 'psql', '-U'])
-        self.assertIn('SELECT 1;', argv)
-        self.assertFalse(mock_run.call_args[1].get('shell', False))
-
-    def test_query_timeout_returns_none(self):
-        with patch.object(postgres_pillar_beacon.subprocess, 'run',
-                          side_effect=subprocess.TimeoutExpired(cmd='psql', timeout=30)):
-            self.assertIsNone(postgres_pillar_beacon._query('SELECT 1;'))
-
-    def test_query_generic_exception_returns_none(self):
-        with patch.object(postgres_pillar_beacon.subprocess, 'run', side_effect=Exception('boom')):
-            self.assertIsNone(postgres_pillar_beacon._query('SELECT 1;'))
-
-    def test_query_nonzero_returncode_returns_none(self):
-        completed = subprocess.CompletedProcess(args=[], returncode=1, stdout='', stderr='bad')
-        with patch.object(postgres_pillar_beacon.subprocess, 'run', return_value=completed):
-            self.assertIsNone(postgres_pillar_beacon._query('SELECT 1;'))
-
-    # -- beacon: first run / seeding --------------------------------------
-
-    def test_beacon_seeds_when_postgres_not_ready(self):
-        with patch.object(postgres_pillar_beacon, '_read_watermark', return_value=None), \
-             patch.object(postgres_pillar_beacon, '_query', return_value=None), \
-             patch.object(postgres_pillar_beacon, '_write_watermark') as mock_write:
-            self.assertEqual(postgres_pillar_beacon.beacon({}), [])
-        mock_write.assert_not_called()
-
-    def test_beacon_seeds_to_max_id_and_emits_nothing(self):
-        with patch.object(postgres_pillar_beacon, '_read_watermark', return_value=None), \
-             patch.object(postgres_pillar_beacon, '_query', return_value='7\n'), \
-             patch.object(postgres_pillar_beacon, '_write_watermark') as mock_write:
-            self.assertEqual(postgres_pillar_beacon.beacon({}), [])
-        mock_write.assert_called_once_with(7)
-
-    def test_beacon_seed_unparseable_is_swallowed(self):
-        with patch.object(postgres_pillar_beacon, '_read_watermark', return_value=None), \
-             patch.object(postgres_pillar_beacon, '_query', return_value='abc'), \
-             patch.object(postgres_pillar_beacon, '_write_watermark') as mock_write:
-            self.assertEqual(postgres_pillar_beacon.beacon({}), [])
-        mock_write.assert_not_called()
-
-    # -- beacon: steady state ---------------------------------------------
-
-    def test_beacon_query_failure_returns_empty(self):
-        with patch.object(postgres_pillar_beacon, '_read_watermark', return_value=10), \
-             patch.object(postgres_pillar_beacon, '_query', return_value=None), \
-             patch.object(postgres_pillar_beacon, '_write_watermark') as mock_write:
-            self.assertEqual(postgres_pillar_beacon.beacon({}), [])
-        mock_write.assert_not_called()
-
-    def test_beacon_emits_events_and_advances_watermark(self):
-        sep = postgres_pillar_beacon.FIELD_SEP
-        rows = '11%s5%snode1\n12%s6%s\n' % (sep, sep, sep, sep)
-        with patch.object(postgres_pillar_beacon, '_read_watermark', return_value=10), \
-             patch.object(postgres_pillar_beacon, '_query', return_value=rows), \
-             patch.object(postgres_pillar_beacon, '_write_watermark') as mock_write:
-            result = postgres_pillar_beacon.beacon({})
-        self.assertEqual(result, [
-            {'tag': 'audit_settings', 'id': 11, 'setting_id': '5', 'node_id': 'node1'},
-            {'tag': 'audit_settings', 'id': 12, 'setting_id': '6', 'node_id': ''},
-        ])
-        mock_write.assert_called_once_with(12)
-
-    def test_beacon_skips_malformed_blank_and_noninteger_rows(self):
-        sep = postgres_pillar_beacon.FIELD_SEP
-        rows = (
-            '\n'                       # blank line -> skipped
-            '13%s7\n'                  # too few fields -> skipped
-            'abc%s8%snodeX\n'          # non-integer id -> skipped
-            '14%s9%snodeY\n'           # the one good row
-        ) % (sep, sep, sep, sep, sep)
-        with patch.object(postgres_pillar_beacon, '_read_watermark', return_value=10), \
-             patch.object(postgres_pillar_beacon, '_query', return_value=rows), \
-             patch.object(postgres_pillar_beacon, '_write_watermark') as mock_write:
-            result = postgres_pillar_beacon.beacon({})
-        self.assertEqual(result, [
-            {'tag': 'audit_settings', 'id': 14, 'setting_id': '9', 'node_id': 'nodeY'},
-        ])
-        mock_write.assert_called_once_with(14)
-
-    def test_beacon_no_new_rows_does_not_advance_watermark(self):
-        with patch.object(postgres_pillar_beacon, '_read_watermark', return_value=10), \
-             patch.object(postgres_pillar_beacon, '_query', return_value=''), \
-             patch.object(postgres_pillar_beacon, '_write_watermark') as mock_write:
-            self.assertEqual(postgres_pillar_beacon.beacon({}), [])
-        mock_write.assert_not_called()
-
-
-if __name__ == '__main__':
-    unittest.main()

salt/_beacons/rules_beacon.py

@@ -1,139 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Custom salt beacon that watches the suricata/strelka rule directories for changes
-# and emits a beacon event per changed directory. This replaces the stock salt
-# `inotify` beacon, which leaks a kernel inotify instance every time the minion
-# rebuilds the beacon loader's __context__ (orphaning the old pyinotify.Notifier
-# without closing it) until fs.inotify.max_user_instances is exhausted and the
-# beacon dies with EMFILE. Polling holds zero inotify instances, so the leak is
-# impossible, and it keeps firing during state runs (no blackout).
-#
-# Detection is poll-based with a per-directory fingerprint persisted to
-# WATERMARK_DIR: each pass walks the directory and hashes every file's
-# (relpath, st_mtime_ns, st_size), which catches content writes, additions,
-# moves, and deletions. A change in the digest emits one event; an unchanged
-# digest emits nothing. This makes it self-healing (a missed poll simply catches
-# up on the next one).
-#
-# Each emitted event carries the watched directory path under the configured tag
-# (e.g. salt/beacon/<minion>/rules_beacon/suricata); the push_suricata / push_strelka
-# reactors write a push intent, after which the existing so-push-drainer /
-# orch.push_batch pipeline takes over unchanged.
-
-import hashlib
-import logging
-import os
-import re
-
-log = logging.getLogger(__name__)
-
-WATERMARK_DIR = '/opt/so/state'
-
-# Temp/editor files that should not trigger a push. Mirrors the exclude regexes
-# the inotify beacon used. Matched against the full pathname.
-EXCLUDES = [
-    re.compile(r'\.sw[a-z]$'),
-    re.compile(r'~$'),
-    re.compile(r'/4913$'),
-    re.compile(r'/\.#'),
-]
-
-
-def __virtual__():
-    return True
-
-
-def validate(config):
-    return True, 'valid'
-
-
-def _paths_from_config(config):
-    # The beacon config arrives as a list of single-key dicts (salt beacon style).
-    # Merge it and return the {dir: tag} mapping under the 'paths' key.
-    merged = {}
-    if isinstance(config, list):
-        for item in config:
-            if isinstance(item, dict):
-                merged.update(item)
-    elif isinstance(config, dict):
-        merged = config
-    paths = merged.get('paths', {})
-    return paths if isinstance(paths, dict) else {}
-
-
-def _excluded(pathname):
-    for pattern in EXCLUDES:
-        if pattern.search(pathname):
-            return True
-    return False
-
-
-def _fingerprint(directory):
-    # Stat-only walk; hash each file's (relpath, mtime_ns, size). Returns a hex
-    # digest, or the digest of an empty tree if the directory does not exist.
-    h = hashlib.sha1()
-    if os.path.isdir(directory):
-        entries = []
-        for root, _dirs, files in os.walk(directory):
-            for name in files:
-                full = os.path.join(root, name)
-                if _excluded(full):
-                    continue
-                try:
-                    st = os.stat(full)
-                except OSError:
-                    continue
-                rel = os.path.relpath(full, directory)
-                entries.append('%s\0%d\0%d' % (rel, st.st_mtime_ns, st.st_size))
-        for line in sorted(entries):
-            h.update(line.encode('utf-8', 'surrogateescape'))
-            h.update(b'\n')
-    return h.hexdigest()
-
-
-def _watermark_file(tag):
-    return os.path.join(WATERMARK_DIR, 'rules_beacon_%s.hash' % tag)
-
-
-def _read_watermark(tag):
-    try:
-        with open(_watermark_file(tag), 'r') as f:
-            return (f.read() or '').strip() or None
-    except IOError:
-        return None
-
-
-def _write_watermark(tag, digest):
-    path = _watermark_file(tag)
-    try:
-        os.makedirs(WATERMARK_DIR, exist_ok=True)
-        tmp = path + '.tmp'
-        with open(tmp, 'w') as f:
-            f.write(digest)
-        os.rename(tmp, path)
-    except OSError:
-        log.exception('rules_beacon: failed to persist watermark to %s', path)
-
-
-def beacon(config):
-    retval = []
-
-    for directory, tag in _paths_from_config(config).items():
-        digest = _fingerprint(directory)
-        previous = _read_watermark(tag)
-
-        # First run / missing watermark: seed the digest and emit nothing so a
-        # fresh host does not fire a spurious fleetwide push.
-        if previous is None:
-            _write_watermark(tag, digest)
-            continue
-
-        if digest != previous:
-            _write_watermark(tag, digest)
-            retval.append({'tag': tag, 'path': directory})
-            log.info('rules_beacon: change detected in %s, emitting %s', directory, tag)
-
-    return retval

salt/_beacons/rules_beacon_test.py

@@ -1,172 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-import hashlib
-import os
-import shutil
-import tempfile
-import unittest
-from unittest.mock import patch
-
-import rules_beacon
-
-
-class TestRulesBeacon(unittest.TestCase):
-
-    def setUp(self):
-        # Isolate all on-disk state (watermarks and the dirs we fingerprint) in a
-        # throwaway tree, and point WATERMARK_DIR at it so the real read/write
-        # helpers run against actual files.
-        self.tmpdir = tempfile.mkdtemp()
-        self.state = os.path.join(self.tmpdir, 'state')
-        patcher = patch.object(rules_beacon, 'WATERMARK_DIR', self.state)
-        patcher.start()
-        self.addCleanup(patcher.stop)
-
-    def tearDown(self):
-        shutil.rmtree(self.tmpdir, ignore_errors=True)
-
-    def _make_dir(self, name, files=None):
-        path = os.path.join(self.tmpdir, name)
-        os.makedirs(path, exist_ok=True)
-        for fname, content in (files or {}).items():
-            with open(os.path.join(path, fname), 'w') as f:
-                f.write(content)
-        return path
-
-    # -- trivial contract -------------------------------------------------
-
-    def test_virtual_returns_true(self):
-        self.assertTrue(rules_beacon.__virtual__())
-
-    def test_validate_returns_valid(self):
-        self.assertEqual(rules_beacon.validate({}), (True, 'valid'))
-
-    # -- _paths_from_config -----------------------------------------------
-
-    def test_paths_from_config_list_of_dicts(self):
-        config = [{'interval': 10}, {'paths': {'/a': 'suricata', '/b': 'strelka'}}]
-        self.assertEqual(
-            rules_beacon._paths_from_config(config),
-            {'/a': 'suricata', '/b': 'strelka'},
-        )
-
-    def test_paths_from_config_plain_dict(self):
-        self.assertEqual(
-            rules_beacon._paths_from_config({'paths': {'/a': 'suricata'}}),
-            {'/a': 'suricata'},
-        )
-
-    def test_paths_from_config_skips_non_dict_items(self):
-        self.assertEqual(rules_beacon._paths_from_config(['bogus', 42]), {})
-
-    def test_paths_from_config_paths_not_a_dict(self):
-        self.assertEqual(rules_beacon._paths_from_config({'paths': 'nope'}), {})
-
-    def test_paths_from_config_unexpected_type(self):
-        self.assertEqual(rules_beacon._paths_from_config('nonsense'), {})
-
-    # -- _excluded --------------------------------------------------------
-
-    def test_excluded_matches_temp_and_editor_files(self):
-        for pathname in ('/rules/foo.swp', '/rules/foo~', '/rules/4913', '/rules/.#foo'):
-            self.assertTrue(rules_beacon._excluded(pathname), pathname)
-
-    def test_excluded_allows_real_rule_files(self):
-        self.assertFalse(rules_beacon._excluded('/rules/suricata.rules'))
-
-    # -- _fingerprint -----------------------------------------------------
-
-    def test_fingerprint_missing_dir_is_empty_tree_digest(self):
-        missing = os.path.join(self.tmpdir, 'does-not-exist')
-        self.assertEqual(rules_beacon._fingerprint(missing), hashlib.sha1().hexdigest())
-
-    def test_fingerprint_changes_when_content_changes(self):
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        before = rules_beacon._fingerprint(d)
-        with open(os.path.join(d, 'a.rules'), 'w') as f:
-            f.write('alert tcp any any -> any any')  # different size
-        self.assertNotEqual(rules_beacon._fingerprint(d), before)
-
-    def test_fingerprint_ignores_excluded_files(self):
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        before = rules_beacon._fingerprint(d)
-        with open(os.path.join(d, 'a.rules.swp'), 'w') as f:
-            f.write('editor swap')
-        self.assertEqual(rules_beacon._fingerprint(d), before)
-
-    def test_fingerprint_skips_unstatable_entries(self):
-        # A dangling symlink appears in os.walk's file list but os.stat raises
-        # OSError, exercising the except-continue path.
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        good = rules_beacon._fingerprint(d)
-        os.symlink(os.path.join(d, 'missing-target'), os.path.join(d, 'broken.link'))
-        self.assertEqual(rules_beacon._fingerprint(d), good)
-
-    # -- _read_watermark / _write_watermark -------------------------------
-
-    def test_watermark_round_trip(self):
-        rules_beacon._write_watermark('suricata', 'deadbeef')
-        self.assertEqual(rules_beacon._read_watermark('suricata'), 'deadbeef')
-
-    def test_read_watermark_missing_returns_none(self):
-        self.assertIsNone(rules_beacon._read_watermark('suricata'))
-
-    def test_read_watermark_empty_file_returns_none(self):
-        os.makedirs(self.state, exist_ok=True)
-        with open(rules_beacon._watermark_file('suricata'), 'w') as f:
-            f.write('')
-        self.assertIsNone(rules_beacon._read_watermark('suricata'))
-
-    def test_write_watermark_swallows_oserror(self):
-        with patch.object(rules_beacon.os, 'makedirs', side_effect=OSError):
-            rules_beacon._write_watermark('suricata', 'deadbeef')
-        self.assertIsNone(rules_beacon._read_watermark('suricata'))
-
-    # -- beacon -----------------------------------------------------------
-
-    def _config(self, mapping):
-        return [{'paths': mapping}]
-
-    def test_beacon_seeds_first_run_and_emits_nothing(self):
-        with patch.object(rules_beacon, '_fingerprint', return_value='hash1'), \
-             patch.object(rules_beacon, '_read_watermark', return_value=None), \
-             patch.object(rules_beacon, '_write_watermark') as mock_write:
-            result = rules_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
-        self.assertEqual(result, [])
-        mock_write.assert_called_once_with('suricata', 'hash1')
-
-    def test_beacon_emits_on_change(self):
-        with patch.object(rules_beacon, '_fingerprint', return_value='newhash'), \
-             patch.object(rules_beacon, '_read_watermark', return_value='oldhash'), \
-             patch.object(rules_beacon, '_write_watermark') as mock_write:
-            result = rules_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
-        self.assertEqual(result, [{'tag': 'suricata', 'path': '/rules/suricata'}])
-        mock_write.assert_called_once_with('suricata', 'newhash')
-
-    def test_beacon_no_change_emits_nothing(self):
-        with patch.object(rules_beacon, '_fingerprint', return_value='samehash'), \
-             patch.object(rules_beacon, '_read_watermark', return_value='samehash'), \
-             patch.object(rules_beacon, '_write_watermark') as mock_write:
-            result = rules_beacon.beacon(self._config({'/rules/suricata': 'suricata'}))
-        self.assertEqual(result, [])
-        mock_write.assert_not_called()
-
-    def test_beacon_end_to_end_with_real_files(self):
-        # Exercise the full stack (real fingerprint + real watermark files) across
-        # two poll passes: first seeds silently, second fires after a write.
-        d = self._make_dir('rules', {'a.rules': 'alert'})
-        config = self._config({d: 'suricata'})
-
-        self.assertEqual(rules_beacon.beacon(config), [])  # seed pass
-        self.assertEqual(rules_beacon.beacon(config), [])  # unchanged pass
-
-        with open(os.path.join(d, 'b.rules'), 'w') as f:
-            f.write('alert tcp any any -> any any')
-        self.assertEqual(rules_beacon.beacon(config), [{'tag': 'suricata', 'path': d}])
-
-
-if __name__ == '__main__':
-    unittest.main()

salt/_beacons/zeek.py

@@ -3,30 +3,31 @@ import logging
 
 def status():
 
-    cmd = "runuser -l zeek -c '/opt/zeek/bin/zeekctl status'"
-    retval = __salt__['docker.run']('so-zeek', cmd)  # noqa: F821
-    logging.info('zeekctl_module: zeekctl.status retval: %s' % retval)
+ cmd = "runuser -l zeek -c '/opt/zeek/bin/zeekctl status'"
+ retval = __salt__['docker.run']('so-zeek', cmd)
+ logging.info('zeekctl_module: zeekctl.status retval: %s' % retval)
 
-    return retval
+ return retval
 
 
 def beacon(config):
 
-    retval = []
+  retval = []
 
-    is_enabled = __salt__['healthcheck.is_enabled']()  # noqa: F821
-    logging.info('zeek_beacon: healthcheck_is_enabled: %s' % is_enabled)
+  is_enabled = __salt__['healthcheck.is_enabled']()
+  logging.info('zeek_beacon: healthcheck_is_enabled: %s' % is_enabled)
 
-    if is_enabled:
-        zeekstatus = status().lower().split(' ')
-        logging.info('zeek_beacon: zeekctl.status: %s' % str(zeekstatus))
-        if 'stopped' in zeekstatus or 'crashed' in zeekstatus or 'error' in zeekstatus or 'error:' in zeekstatus:
-            zeek_restart = True
-        else:
-            zeek_restart = False
+  if is_enabled:
+    zeekstatus = status().lower().split(' ')
+    logging.info('zeek_beacon: zeekctl.status: %s' % str(zeekstatus))
+    if 'stopped' in zeekstatus or 'crashed' in zeekstatus or 'error' in zeekstatus or 'error:' in zeekstatus:
+     zeek_restart = True
+    else:
+     zeek_restart = False
 
-        __salt__['telegraf.send']('healthcheck zeek_restart=%s' % str(zeek_restart))  # noqa: F821
-        retval.append({'zeek_restart': zeek_restart})
-        logging.info('zeek_beacon: retval: %s' % str(retval))
+    __salt__['telegraf.send']('healthcheck zeek_restart=%s' % str(zeek_restart))
+    retval.append({'zeek_restart': zeek_restart})
+    logging.info('zeek_beacon: retval: %s' % str(retval))
+
+  return retval
 
-    return retval

salt/_beacons/zeek_test.py

@@ -1,59 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-import unittest
-from unittest.mock import MagicMock
-
-import zeek
-
-ZEEKCTL_CMD = "runuser -l zeek -c '/opt/zeek/bin/zeekctl status'"
-
-
-class TestZeekBeacon(unittest.TestCase):
-
-    def setUp(self):
-        # zeek.py relies on the __salt__ dunder that Salt injects at load time.
-        # Nothing defines it under test, so we attach a dict of mock loader
-        # functions to the module and remove it again afterwards.
-        self.salt = {
-            'docker.run': MagicMock(return_value='Zeek is running'),
-            'healthcheck.is_enabled': MagicMock(return_value=True),
-            'telegraf.send': MagicMock(),
-        }
-        zeek.__salt__ = self.salt
-        self.addCleanup(lambda: delattr(zeek, '__salt__'))
-
-    # -- status -----------------------------------------------------------
-
-    def test_status_runs_zeekctl_and_returns_output(self):
-        self.salt['docker.run'].return_value = 'Zeek is running'
-        result = zeek.status()
-        self.assertEqual(result, 'Zeek is running')
-        self.salt['docker.run'].assert_called_once_with('so-zeek', ZEEKCTL_CMD)
-
-    # -- beacon -----------------------------------------------------------
-
-    def test_beacon_disabled_returns_empty_and_skips_telegraf(self):
-        self.salt['healthcheck.is_enabled'].return_value = False
-        self.assertEqual(zeek.beacon({}), [])
-        self.salt['telegraf.send'].assert_not_called()
-
-    def test_beacon_running_reports_no_restart(self):
-        self.salt['docker.run'].return_value = 'Zeek is running'
-        self.assertEqual(zeek.beacon({}), [{'zeek_restart': False}])
-        self.salt['telegraf.send'].assert_called_once_with('healthcheck zeek_restart=False')
-
-    def test_beacon_unhealthy_status_triggers_restart(self):
-        # Each of these status tokens should flag a restart (the or-chain in beacon).
-        for status_text in ('Zeek is stopped', 'Zeek crashed', 'Zeek error state', 'Zeek error:'):
-            with self.subTest(status=status_text):
-                self.salt['docker.run'].return_value = status_text
-                self.salt['telegraf.send'].reset_mock()
-                self.assertEqual(zeek.beacon({}), [{'zeek_restart': True}])
-                self.salt['telegraf.send'].assert_called_once_with('healthcheck zeek_restart=True')
-
-
-if __name__ == '__main__':
-    unittest.main()

salt/allowed_states.map.jinja

@@ -37,7 +37,8 @@
     'elasticfleet',
     'elasticfleet.manager',
     'elasticsearch.cluster',
-    'elastic-fleet-package-registry'
+    'elastic-fleet-package-registry',
+    'utility'
 ] %}
 
 {% set sensor_states = [

salt/backup/tools/sbin/so-config-backup.jinja

@@ -25,11 +25,9 @@ if [ ! -f $BACKUPFILE ]; then
   # Create empty backup file
   tar -cf $BACKUPFILE -T /dev/null
 
-  # Loop through all paths defined in global.sls, and append them to backup file if they exist
+  # Loop through all paths defined in global.sls, and append them to backup file
   {%- for LOCATION in BACKUPLOCATIONS %}
-  if [[ -d {{ LOCATION }} || -f {{ LOCATION }} ]]; then
-    tar -rf $BACKUPFILE "${EXCLUSIONS[@]}" {{ LOCATION }}
-  fi
+  tar -rf $BACKUPFILE "${EXCLUSIONS[@]}" {{ LOCATION }}
   {%- endfor %}
 
 fi

salt/common/init.sls

@@ -130,17 +130,6 @@ common_sbin:
       - so-pcap-import
 {% endif %}
 
-# Pin physical NIC names by MAC (run-once) so a kernel upgrade can't renumber the
-# interfaces SO binds by name. The marker keeps it a one-time setup; an admin can
-# pre-create the marker to opt out.
-pin_nic_names:
-  cmd.run:
-    - name: /usr/sbin/so-nic-pin
-    - unless: 'test -e /opt/so/state/nic_names_pinned'
-    - require:
-      - file: common_sbin
-      - file: statedir
-
 common_sbin_jinja:
   file.recurse:
     - name: /usr/sbin

salt/common/tools/sbin/so-common

@@ -142,11 +142,6 @@ check_elastic_license() {
 	fi  
 }
 
-check_elasticsearch_responsive() {
-    retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" ||
-        fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot."
-}
-
 check_salt_master_status() {
 	local count=0
     local attempts="${1:- 10}"
@@ -291,20 +286,6 @@ download_and_verify() {
 	fi
 }
 
-# check if container with name is running and optionally stop it
-docker_check_running() {
-    # show running containers, only names
-    if docker ps --format '{{.Names}}' | grep -q "^so-${1}$"; then
-        if [[ "$2" == "--stop" ]]; then
-            docker stop "so-${1}"
-        fi
-
-        return 0
-    else
-        return 1
-    fi
-}
-
 elastic_license() {
 
 read -r -d '' message <<- EOM
@@ -602,6 +583,42 @@ run_check_net_err() {
 	fi
 }
 
+wait_for_salt_minion() {
+    local minion="$1"
+    local max_wait="${2:-30}"
+    local interval="${3:-2}"
+    local logfile="${4:-'/dev/stdout'}"
+    local elapsed=0
+
+	echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Waiting for salt-minion '$minion' to be ready..."
+
+    while [ $elapsed -lt $max_wait ]; do
+        # Check if service is running
+		echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Check if salt-minion service is running"
+        if ! systemctl is-active --quiet salt-minion; then
+            echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion service not running (elapsed: ${elapsed}s)"
+            sleep $interval
+            elapsed=$((elapsed + interval))
+            continue
+        fi
+		echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion service is running"
+
+        # Check if minion responds to ping
+		echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Check if $minion responds to ping"
+        if salt "$minion" test.ping --timeout=3 --out=json 2>> "$logfile" | grep -q "true"; then
+            echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion '$minion' is connected and ready!"
+            return 0
+        fi
+        
+        echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Waiting... (${elapsed}s / ${max_wait}s)"
+        sleep $interval
+        elapsed=$((elapsed + interval))
+    done
+
+    echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - ERROR: salt-minion '$minion' not ready after $max_wait seconds"
+    return 1
+}
+
 salt_minion_count() {
 	local MINIONDIR="/opt/so/saltstack/local/pillar/minions"
 	MINIONCOUNT=$(ls -la $MINIONDIR/*.sls | grep -v adv_ | wc -l)
@@ -666,7 +683,7 @@ systemctl_func() {
 
 	echo ""
 	echo "${echo_action^}ing $service_name service at $(date +"%T.%6N")"
-    systemctl $action $service_name && echo "Successfully ${echo_action}ed $service_name at $(date +"%T.%6N")." || echo "Failed to $action $service_name at $(date +"%T.%6N")."
+    systemctl $action $service_name && echo "Successfully ${echo_action}ed $service_name." || echo "Failed to $action $service_name."
 	echo ""
 }
 

salt/common/tools/sbin/so-kernel-upgrade

@@ -1,243 +0,0 @@
-#!/bin/bash
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-#
-# so-kernel-upgrade — install the UEK8 (6.x) kernel and make it the boot default.
-#
-# Security Onion is moving off the EL9 stock kernel (RHCK, 5.14) and UEK7 (5.15) onto UEK8
-# (6.x). Three things have to happen, and the tool has to drive each one:
-#
-#   1. Populate. The manager mirrors the UEK8 packages into /nsm/kernelrepo via so-repo-sync,
-#      and serves them to the grid over https://<manager>/kernelrepo. Until that sync runs the
-#      repo is valid but EMPTY -- dnf resolves it happily and installs nothing, with no error.
-#   2. Install. A node on RHCK has no kernel-uek* package at all, so there is nothing for
-#      'dnf update' to upgrade. A node on UEK7 does have kernel-uek installed, so
-#      'dnf install kernel-uek' reports "Nothing to do" and exits 0 without installing 6.x.
-#      Both cases need an explicit install of the UEK8 NEVRA.
-#   3. Boot it. Whether a newly installed UEK8 kernel becomes the boot default depends on the
-#      RUNNING kernel's flavor. kernel-install/grubby (with UPDATEDEFAULT=yes) only auto-promote
-#      within the running kernel's flavor lineage:
-#        - From UEK7 (5.x, kernel-uek) the install stays in the kernel-uek lineage and IS
-#          auto-promoted, so no grubby change is needed -- just make sure the repo is populated
-#          and install UEK8.
-#        - From the stock EL9 kernel (RHCK, 5.14, no UEK) it is a flavor CROSS that is NOT
-#          auto-promoted, so the box keeps booting RHCK until grubby is told otherwise.
-#      This tool inspects the running kernel and only runs 'grubby --set-default' for RHCK.
-#
-# Every one of those failure modes is silent by default. This tool handles each case and fails
-# loudly when it cannot, rather than reporting success while changing nothing.
-#
-# Manager vs minion: only the manager owns /nsm/kernelrepo, so only the manager can populate
-# it. If the repo is empty here, a manager runs so-repo-sync itself; a minion has no way to
-# fix it and exits non-zero telling the admin to sync the manager first.
-#
-# Idempotent: an already-installed, already-default UEK8 kernel is left alone. It only sets
-# the boot default; it does NOT reboot -- the admin reboots the node on their own schedule.
-
-. /usr/sbin/so-common
-
-# Client-side repo id (what dnf enables on this node, from repo/client/oracle.sls) vs the
-# reposync-side section in repodownload.conf that the manager mirrors from (mirrors the
-# securityonion/securityonionsync split for the main repo).
-KERNEL_REPO="securityonionkernel"
-KERNEL_REPO_SYNC="securityonionkernelsync"
-KERNEL_PKG="kernel-uek"
-KERNEL_REPO_DIR="/nsm/kernelrepo"
-REPOSYNC_CONF="/opt/so/conf/reposync/repodownload.conf"
-GLOBAL_PILLAR="/opt/so/saltstack/local/pillar/global/soc_global.sls"
-
-log() { echo "[so-kernel-upgrade] $*"; }
-die() { echo "[so-kernel-upgrade] ERROR: $*" >&2; exit 1; }
-
-command -v grubby >/dev/null 2>&1 || die "grubby not found"
-command -v dnf >/dev/null 2>&1 || die "dnf not found"
-
-ARCH="$(rpm -E '%{_arch}')"
-
-is_airgap() {
-    [ -f "$GLOBAL_PILLAR" ] && grep -q 'airgap: *[Tt]rue' "$GLOBAL_PILLAR"
-}
-
-# Newest installed UEK8 (6.x) kernel known to the bootloader. UEK8 vmlinuz paths look like
-# /boot/vmlinuz-6.12.0-204.92.4.2.el9uek.x86_64; UEK7 (5.15) and RHCK (5.14) won't match.
-find_uek8() {
-    grubby --info=ALL 2>/dev/null \
-        | sed -n 's/^kernel="\(.*\)"$/\1/p' \
-        | grep -E '/vmlinuz-6\.[0-9]+.*uek' \
-        | sort -V | tail -1
-}
-
-# Classify the RUNNING kernel (uname -r) -- this, not what's installed, is what decides whether
-# a UEK8 install auto-promotes to the boot default:
-#   uek8  6.x UEK    already on the target line; nothing to do
-#   uek7  5.x UEK    a UEK8 install stays in the kernel-uek lineage and auto-promotes (no grubby)
-#   rhck  5.14 EL9   crossing into the UEK flavor does NOT auto-promote (needs grubby --set-default)
-running_flavor() {
-    case "$(uname -r)" in
-        6.*uek*) echo uek8 ;;
-        *uek*)   echo uek7 ;;
-        *)       echo rhck ;;
-    esac
-}
-
-# Newest UEK8 kernel-uek NEVRA offered by the kernel repo, empty if the repo has none.
-# Restricted to the kernel repo so a UEK7 kernel-uek in the main repo can't be picked up,
-# and filtered to 6.x so we never "succeed" by reinstalling the 5.15 we already have.
-uek8_available() {
-    dnf -q repoquery --disablerepo='*' --enablerepo="$KERNEL_REPO" \
-        --arch="$ARCH" --latest-limit=1 \
-        --qf '%{name}-%{evr}.%{arch}\n' "$KERNEL_PKG" 2>/dev/null \
-        | grep -E "^${KERNEL_PKG}-6\." | tail -1
-}
-
-kernelrepo_rpm_count() {
-    find "$KERNEL_REPO_DIR" -maxdepth 1 -name '*.rpm' 2>/dev/null | wc -l
-}
-
-# The kernel repo starts life as valid-but-empty (kernelrepo_init_empty in
-# salt/manager/init.sls) and is filled by so-repo-sync. During a soup, so-repo-sync runs
-# BEFORE the highstate deploys the [securityonionkernelsync] section into repodownload.conf, so
-# the first kernel-aware soup leaves the repo empty until the next nightly sync.
-sync_kernel_repo() {
-    if is_airgap; then
-        log "airgap install: $KERNEL_REPO_DIR is populated from the airgap ISO, not by so-repo-sync."
-        return 1
-    fi
-    if ! grep -q "^\[${KERNEL_REPO_SYNC}\]" "$REPOSYNC_CONF" 2>/dev/null; then
-        log "$REPOSYNC_CONF has no [${KERNEL_REPO_SYNC}] section -- run a highstate to deploy it."
-        return 1
-    fi
-
-    log "populating $KERNEL_REPO_DIR with so-repo-sync (mirrors upstream; can take several minutes)"
-    su socore -c '/usr/sbin/so-repo-sync' || { log "so-repo-sync failed"; return 1; }
-
-    dnf -q clean expire-cache >/dev/null 2>&1
-    return 0
-}
-
-# Make the kernel repo actually able to serve a UEK8 package, or fail trying.
-ensure_kernel_repo() {
-    # The repo is assigned by the repo.client highstate, and only once NICs are pinned by MAC
-    # (/opt/so/state/nic_names_pinned) so the kernel swap can't renumber interfaces SO binds
-    # by name. skip_if_unavailable=1 means a broken repo is silently ignored, so check first.
-    if ! dnf -q repolist --enabled 2>/dev/null | awk '{print $1}' | grep -qx "$KERNEL_REPO"; then
-        log "repo '$KERNEL_REPO' is not enabled on this node."
-        log "Run a highstate first; the repo is skipped until /opt/so/state/nic_names_pinned"
-        log "exists (run so-nic-pin) and this node's salt matches the version this release ships."
-        die "kernel repo unavailable"
-    fi
-
-    [ -n "$(uek8_available)" ] && return 0
-
-    log "repo '$KERNEL_REPO' is enabled but offers no UEK8 $KERNEL_PKG package"
-
-    if ! is_manager_node; then
-        log "This is a minion; it consumes the kernel repo from the manager and cannot populate it."
-        log "On the manager, run:  su socore -c /usr/sbin/so-repo-sync"
-        log "then re-run this script here."
-        die "manager's kernel repo is empty"
-    fi
-
-    log "this is a manager and $KERNEL_REPO_DIR holds $(kernelrepo_rpm_count) rpm(s)"
-    sync_kernel_repo || die "could not populate $KERNEL_REPO_DIR"
-
-    [ -n "$(uek8_available)" ] \
-        || die "so-repo-sync completed but $KERNEL_REPO still offers no UEK8 $KERNEL_PKG"
-}
-
-reboot_notice() {
-    [ "$(uname -r)" = "$(basename "$1" | sed 's/^vmlinuz-//')" ] \
-        || log "REBOOT REQUIRED to start using the UEK8 kernel (currently running $(uname -r))."
-}
-
-# Keep future kernel updates on the UEK line rather than falling back to RHCK. Oracle ships
-# /etc/sysconfig/kernel; only rewrite it when it's actually pointing somewhere else.
-set_default_kernel_conf() {
-    if [ -f /etc/sysconfig/kernel ] && ! grep -q '^DEFAULTKERNEL=kernel-uek-core$' /etc/sysconfig/kernel; then
-        log "setting DEFAULTKERNEL=kernel-uek-core in /etc/sysconfig/kernel"
-        sed -i 's/^DEFAULTKERNEL=.*/DEFAULTKERNEL=kernel-uek-core/' /etc/sysconfig/kernel
-    fi
-}
-
-# Make sure a UEK8 kernel is installed, leaving its boot entry in INSTALLED_UEK8. If one is
-# already present we leave the repo alone -- it may be disabled or empty and we don't need it
-# just to flip the boot default. Otherwise install the explicit NEVRA, not the bare package
-# name: on a UEK7 node 'dnf install kernel-uek' sees 5.15 already present, prints "Nothing to
-# do" and exits 0 without installing 6.x.
-ensure_uek8_installed() {
-    INSTALLED_UEK8="$(find_uek8)"
-    if [ -n "$INSTALLED_UEK8" ]; then
-        log "UEK8 kernel already installed: $INSTALLED_UEK8"
-        return 0
-    fi
-
-    ensure_kernel_repo
-    local nevra; nevra="$(uek8_available)"
-    log "installing $nevra from $KERNEL_REPO"
-    dnf -y install "$nevra" || die "failed to install $nevra"
-
-    INSTALLED_UEK8="$(find_uek8)"
-    [ -n "$INSTALLED_UEK8" ] || die "$nevra installed but no 6.x UEK boot entry appeared -- check 'grubby --info=ALL'"
-    log "installed UEK8 kernel: $INSTALLED_UEK8"
-}
-
-case "$(running_flavor)" in
-uek8)
-    # Already on the 6.x UEK line. A plain 'dnf update' keeps this node current within the
-    # lineage and auto-promotes newer builds, so there is nothing for this tool to do.
-    log "already running a UEK8 kernel ($(uname -r)); nothing to do."
-    exit 0
-    ;;
-
-uek7)
-    # On a 5.x UEK kernel. Installing UEK8 stays inside the kernel-uek lineage, so dnf/grubby
-    # (UPDATEDEFAULT=yes) auto-promote it and we do NOT touch grubby. A node still on UEK7
-    # usually means the kernel repo was empty when it last updated, so populate it and install.
-    log "running UEK7 kernel ($(uname -r)); the kernel repo was likely not yet populated when"
-    log "this node last updated. Populating it and installing UEK8 -- the update stays on the"
-    log "kernel-uek line, so it becomes the boot default automatically (no grubby change needed)."
-    set_default_kernel_conf
-    ensure_uek8_installed
-
-    now="$(grubby --default-kernel 2>/dev/null)"
-    if [ "$now" = "$INSTALLED_UEK8" ]; then
-        log "boot default auto-promoted to UEK8 kernel: $INSTALLED_UEK8"
-    else
-        log "WARNING: expected the UEK8 kernel to auto-promote but the default is still"
-        log "'${now:-unknown}'. Run 'grubby --set-default=$INSTALLED_UEK8' to force it."
-    fi
-    reboot_notice "$INSTALLED_UEK8"
-    ;;
-
-rhck)
-    # On the stock EL9 kernel (5.14, no UEK installed). Crossing from RHCK into the UEK flavor
-    # does NOT auto-promote -- kernel-install/grubby only auto-promote within the running
-    # kernel's flavor lineage -- so after installing we must set the boot default explicitly.
-    log "running stock EL9 (RHCK) kernel ($(uname -r)); installing UEK8 and setting it as the"
-    log "boot default explicitly (a RHCK->UEK flavor change does not auto-promote)."
-    set_default_kernel_conf
-    ensure_uek8_installed
-    target="$INSTALLED_UEK8"
-
-    current="$(grubby --default-kernel 2>/dev/null)"
-    if [ "$current" = "$target" ]; then
-        log "UEK8 kernel is already the boot default: $target"
-        reboot_notice "$target"
-        exit 0
-    fi
-
-    log "current default kernel: ${current:-unknown}"
-    log "switching boot default to UEK8 kernel: $target"
-    grubby --set-default="$target" || die "grubby --set-default failed for $target"
-
-    # Verify the change actually took before claiming success.
-    now="$(grubby --default-kernel 2>/dev/null)"
-    [ "$now" = "$target" ] || die "default kernel is still '${now:-unknown}' after set-default"
-
-    log "boot default is now $target"
-    reboot_notice "$target"
-    ;;
-esac

salt/common/tools/sbin/so-log-check

@@ -132,8 +132,6 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|HTTP 404: Not Found"          # Salt loops until Kratos returns 200, during startup Kratos may not be ready
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Cancelling deferred write event maybeFenceReplicas because the event queue is now closed" # Kafka controller log during shutdown/restart
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Redis may have been restarted" # Redis likely restarted by salt
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|file already closed"          # Go logging race condition during container restart
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|relation \"audit_settings\" does not exist" # salt checking for changes before SOC starts
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
@@ -154,8 +152,6 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|id.orig_h"                    # false positive (zeek test data)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|emerging-all.rules"           # false positive (error in rulename)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|invalid query input"          # false positive (Invalid user input in hunt query)
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no data available for the requested dates" # false positive (pcap cypress test submits a job with an empty time frame)
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no job processor"             # false positive (same empty-time-frame job on import nodes, where no pcap processor runs)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|example"                      # false positive (example test data)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200"                   # false positive (request successful, contained error string in content)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|app_layer.error"              # false positive (suricata 7) in stats.log e.g. app_layer.error.imap.parser | Total | 0
@@ -171,11 +167,6 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error while parsing document for index \[.ds-logs-kratos-so-.*object mapping for \[file\]" # false positive (mapping error occuring BEFORE kratos index has rolled over in 2.4.210)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|No such container"            # false positive (telegraf trying to run stats on an old container)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|passwords do not match"       # false positive (automated hydra test)
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Request did not pass preprocessing" # expected WARN log lines indicating invalid auth header
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Missing or invalid authorization header for bearer token" # expected WARN log lines indicating invalid auth header
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unexpected authorization header" # expected WARN log lines indicating invalid auth header
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Missing ory_kratos_session cookie" # expected WARN log lines indicating invalid auth header
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Static assets preprocessor only supports GET and HEAD requests" # expected WARN log lines indicating invalid auth header
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
@@ -238,7 +229,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|from NIC checksum offloading" # zeek reporter.log
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal"           # docker container getting recycled
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-(tychon|aws_billing|microsoft_defender_endpoint|armis|o365_metrics|microsoft_sentinel|snyk|cyera|island_browser).*user so_kibana lacks the required permissions \[(logs|metrics)-\1" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user. This error should not be seen on fresh ES 9.3.3 installs or after SO 3.1.0 with soups addition of check_transform_health_and_reauthorize()
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown"             # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1
 fi
 

salt/common/tools/sbin/so-nic-pin

@@ -1,76 +0,0 @@
-#!/bin/bash
-#
-# so-nic-pin — pin physical NIC names by permanent MAC via classic by-MAC udev
-#              rules, so a kernel upgrade can't renumber them.
-#
-# Security Onion binds its management and monitor interfaces BY NAME in pillar
-# (host:mainint, sensor:mainint, and bond0 is built on a specific physical NIC).
-# A kernel upgrade can change the kernel/systemd-udevd predictable-naming output
-# and renumber those NICs (e.g. enp1s0 -> enp2s0), which breaks the grid: the
-# pillar references a name that no longer exists and bond/bridge bring-up fails.
-#
-# This writes /etc/udev/rules.d/70-persistent-net.rules pinning each PHYSICAL NIC
-# to its CURRENT name by its PERMANENT MAC, freezing the names across future kernel
-# changes. It only writes the rules file; it does NOT live-trigger a rename (the
-# rules apply on the next boot/kernel, and a live rename would be disruptive).
-#
-# Run-once: gated by the drop file /opt/so/state/nic_names_pinned. If the marker is
-# present the script does nothing, so an admin can pre-create it to opt out. Invoked
-# from the common state on every highstate; the marker keeps it a one-time setup.
-
-NET_RULES_FILE="/etc/udev/rules.d/70-persistent-net.rules"
-MARKER="/opt/so/state/nic_names_pinned"
-
-log() { echo -e "[so-nic-pin] $*"; }
-
-# Echo "<name> <permanent-mac>" for every PHYSICAL NIC. A physical NIC is backed by a
-# real device (has device/driver), which excludes bond0/sobridge/docker0/veth*/lo whose
-# MACs are dynamic and must never be pinned. The PERMANENT MAC is used (ethtool -P, with
-# fallbacks), not the current one: an enslaved bond member's current MAC is rewritten to
-# the bond's, so matching on it would be wrong/ambiguous.
-physical_nics() {
-    local path n mac
-    for path in /sys/class/net/*; do
-        n="${path##*/}"
-        [ "$n" = "lo" ] && continue
-        [ -e "${path}/device/driver" ] || continue          # real device only
-        mac="$(ethtool -P "$n" 2>/dev/null | awk '/Permanent address/{print $NF}')"
-        case "$mac" in ""|00:00:00:00:00:00) mac="$(cat "${path}/bonding_slave/perm_hwaddr" 2>/dev/null)" ;; esac
-        case "$mac" in ""|00:00:00:00:00:00) mac="$(cat "${path}/address" 2>/dev/null)" ;; esac
-        case "$mac" in ""|00:00:00:00:00:00) continue ;; esac
-        echo "$n $mac"
-    done
-}
-
-# Turn "<name> <mac>" lines on stdin into classic by-MAC persistent-net udev rules.
-render_net_rules() {
-    echo "# Generated by so-nic-pin: pin NIC names by MAC so kernel upgrades can't renumber them."
-    echo "# Security Onion binds its management/monitor interfaces by name; do not hand-edit."
-    local n mac
-    while read -r n mac; do
-        [ -n "$n" ] || continue
-        printf 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="%s", NAME="%s"\n' \
-            "$mac" "$n"
-    done
-}
-
-[ "$(id -u)" -eq 0 ] || exit 0                   # salt runs us as root; bail quietly otherwise
-[ -e "${MARKER}" ] && exit 0                      # run-once guard (mirrors the state's unless)
-
-nics="$(physical_nics)"
-if [ -z "${nics}" ]; then
-    log "no physical NICs detected — nothing to pin (will retry on next highstate)"
-    exit 0                                         # do NOT drop the marker; let it retry later
-fi
-
-log "pinning physical NICs by permanent MAC:"
-echo "${nics}" | sed 's/^/    /'
-
-[ -f "${NET_RULES_FILE}" ] && cp -f "${NET_RULES_FILE}" "${NET_RULES_FILE}.bak"
-echo "${nics}" | render_net_rules > "${NET_RULES_FILE}" || {
-    log "ERROR: failed to write ${NET_RULES_FILE}"
-    exit 1
-}
-
-mkdir -p "$(dirname "${MARKER}")" && touch "${MARKER}"
-log "wrote ${NET_RULES_FILE} ($(grep -c '^SUBSYSTEM' "${NET_RULES_FILE}") NIC(s) pinned); dropped ${MARKER}"

salt/common/tools/sbin/so-restart

@@ -5,41 +5,27 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
+
+
+# Usage: so-restart  kibana | playbook 
+
 . /usr/sbin/so-common
 
-usage() {
-    echo "Usage: $0 <component> [args]"
-    echo ""
-    echo "Supported args:"
-    echo "  --force | -f          Force stop all Salt jobs before starting component."
-    echo ""
-    echo "Examples:"
-    echo "  $0 kibana             Restart Kibana"
-    echo "  $0 kibana --force     Force stop all Salt jobs before restarting Kibana"
-    exit 1
-}
+if [ $# -ge 1 ]; then
 
-if [[ $# -lt 1 ]]; then
-    usage
-fi
+        echo $banner
+        printf "Restarting $1...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n"
+        echo $banner
 
-#shellcheck disable=SC2154
-echo "$banner"
-printf "Restarting %s...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n" "$1"
-echo "$banner"
-if [[ "$2" = "--force" ]] || [[ "$2" = "-f" ]]; then
-    printf "\nForce-stopping all Salt jobs before proceeding\n\n"
-    salt-call saltutil.kill_all_jobs
+        if [ "$2" = "--force" ]; then
+                printf "\nForce-stopping all Salt jobs before proceeding\n\n"
+                salt-call saltutil.kill_all_jobs
+        fi
+
+        case $1 in
+                "elastic-fleet") docker stop so-elastic-fleet && docker rm so-elastic-fleet && salt-call state.apply elasticfleet queue=True;;
+                *)  docker stop so-$1 ; docker rm so-$1 ; salt-call state.apply $1 queue=True;;
+        esac
+else
+        echo -e "\nPlease provide an argument by running like so-restart $component, or by using the component-specific script.\nEx. so-restart logstash, or so-logstash-restart\n"
 fi
-case $1 in
-    "elastic-fleet"|"elasticfleet")
-        docker_check_running "elastic-fleet" "--stop"
-        docker rm "so-elastic-fleet" 2> /dev/null
-        salt-call state.apply elasticfleet queue=True
-        ;;
-    *)
-        docker_check_running "$1" "--stop"
-        docker rm "so-${1}" 2> /dev/null
-        salt-call state.apply "$1" queue=True
-        ;;
-esac

salt/common/tools/sbin/so-start

@@ -5,54 +5,27 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-# shellcheck disable=SC1091
+
+
+# Usage: so-start  all | kibana | playbook 
+
 . /usr/sbin/so-common
 
-usage() {
-    echo "Usage: $0 <component> [args]"
-    echo ""
-    echo "Supported args:"
-    echo "  --force | -f          Force stop all Salt jobs before starting component."
-    echo ""
-    echo "Examples:"
-    echo "  $0 kibana             Start Kibana"
-    echo "  $0 kibana --force     Force stop all Salt jobs before starting Kibana"
-    exit 1
-}
+if [ $# -ge 1 ]; then
+	echo $banner
+	printf "Starting $1...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n"
+	echo $banner
 
-if [[ $# -lt 1 ]]; then
-    usage
+	if [ "$2" = "--force" ]; then
+   		printf "\nForce-stopping all Salt jobs before proceeding\n\n"
+   		salt-call saltutil.kill_all_jobs
+	fi
+
+	case $1 in
+   		"all") salt-call state.highstate queue=True;;
+   		"elastic-fleet") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply elasticfleet queue=True; fi ;; 
+   		*) if docker ps | grep -E -q '^so-$1$'; then printf "\n$1 is already running\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply $1 queue=True; fi ;; 
+	esac
+else
+	echo -e "\nPlease provide an argument by running like so-start $component, or by using the component-specific script.\nEx. so-start logstash, or so-logstash-start\n"	
 fi
-
-#shellcheck disable=SC2154
-echo "$banner"
-printf "Starting %s...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n" "$1"
-echo "$banner"
-if [[ "$2" = "--force" ]] || [[ "$2" == "-f" ]]; then
-	printf "\nForce-stopping all Salt jobs before proceeding\n\n"
-	salt-call saltutil.kill_all_jobs
-fi
-
-case "$1" in
-    "all")
-        salt-call state.highstate queue=True
-        ;;
-   	"elastic-fleet"|"elasticfleet")
-        if docker_check_running "elastic-fleet"; then
-            printf "\nso-%s is already running!\n\n" "elastic-fleet"
-            /usr/sbin/so-status
-        else
-            docker rm "so-elastic-fleet" 2> /dev/null
-            salt-call state.apply elasticfleet queue=True
-        fi
-        ;;
-   	*)
-        if docker_check_running "$1"; then
-            printf "\nso-%s is already running\n\n" "$1"
-            /usr/sbin/so-status
-        else
-            docker rm "so-${1}" 2> /dev/null
-            salt-call state.apply "$1" queue=True
-        fi
-        ;;
-esac

salt/common/tools/sbin/so-status

@@ -74,13 +74,13 @@ def output(options, console, code, data):
     summary = { "status_code": code, "containers": data }
     print(json.dumps(summary))
   elif "-q" not in options:
-    if code == 99:
+    if code == 2:
+      console.print(" [bold yellow]:hourglass: [bold white]System appears to be starting. No highstate has completed since the system was restarted.")
+    elif code == 99:
       console.print(" [bold red]:exclamation: [bold white]Installation does not appear to be complete. A highstate has not fully completed.")
     elif code == 100:
       console.print(" [bold red]:exclamation: [bold white]Installation encountered errors.")
     else:
-      if code == 2:
-        console.print(" [bold yellow]:hourglass: [bold white]System appears to be starting. No highstate has completed since the system was restarted. Container status is shown below.")
       table = Table(title = "Security Onion Status", show_edge = False, safe_box = True, box = box.MINIMAL)
       table.add_column("Container", justify="right", style="white", no_wrap=True)
       table.add_column("Status", justify="left", style="green", no_wrap=True)
@@ -154,14 +154,8 @@ def check_status(options, console):
   code = check_installation_status(options, console)
   if code == 0:
     code = check_system_status(options, console)
-  # Containers now start on boot without a highstate, so gather/display their
-  # status even when the system is still "starting" (code 2). Keep the starting
-  # code as the exit/status_code so SOC keeps showing the "restarting" message
-  # on the Grid until a highstate completes.
-  if code == 0 or code == 2:
-    container_code, container_list = check_container_status(options, console)
-    if code == 0:
-      code = container_code
+  if code == 0:
+    code, container_list = check_container_status(options, console)
   output(options, console, code, container_list)
   return code
 
@@ -186,3 +180,4 @@ def main():
 
 if __name__ == "__main__":
   main()
+

salt/common/tools/sbin/so-stop

@@ -5,33 +5,21 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-# shellcheck disable=SC1091
+
+
+# Usage: so-stop kibana | playbook | thehive
+
 . /usr/sbin/so-common
 
-usage() {
-    echo "Usage: $0 <component>"
-    echo ""
-    echo "Examples:"
-    echo "  $0 kibana             Stop Kibana"
-    exit 1
-}
+if [ $# -ge 1 ]; then
+	echo $banner
+	printf "Stopping $1...\n"
+	echo $banner
 
-if [[ $# -lt 1 ]]; then
-    usage
+	case $1 in
+    		*) docker stop so-$1 ; docker rm so-$1 ;;
+	esac
+else
+	echo -e "\nPlease provide an argument by running like so-stop $component, or by using the component-specific script.\nEx. so-stop logstash, or so-logstash-stop\n"	
 fi
 
-
-#shellcheck disable=SC2154
-echo "$banner"
-printf "Stopping %s...\n" "$1"
-echo "$banner"
-case $1 in
-	"elasticfleet"|"elastic-fleet")
-        docker_check_running "elastic-fleet" "--stop"
-        docker rm "so-elastic-fleet" 2> /dev/null
-        ;;
-    *)
-        docker_check_running "$1" "--stop"
-        docker rm "so-${1}" 2> /dev/null
-    ;;
-esac

salt/common/tools/sbin_jinja/so-import-pcap

@@ -63,8 +63,7 @@ function status {
 function pcapinfo() {
   PCAP=$1
   ARGS=$2
-  docker run --rm -v "$PCAP:/input.pcap" --entrypoint capinfos {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} /input.pcap -ae $ARGS |\
-    sed 's/First packet/Earliest packet/g' | sed 's/Last packet/Latest packet/g'
+  docker run --rm -v "$PCAP:/input.pcap" --entrypoint capinfos {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} /input.pcap -ae $ARGS
 }
 
 function pcapfix() {

salt/common/tools/sbin_jinja/so-salt-minion-check

@@ -1,3 +1,5 @@
+{% import_yaml 'salt/minion.defaults.yaml' as SALT_MINION_DEFAULTS -%}
+
 #!/bin/bash
 #
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
@@ -5,7 +7,7 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-{% from 'salt/schedule.map.jinja' import SCHEDULEMERGED %}
+
 
 # this script checks the time the file /opt/so/log/salt/state-apply-test was last modified and restarts the salt-minion service if it is outside a threshold date/time
 # the file is modified via file.touch using a scheduled job healthcheck.salt-minion.state-apply-test that runs a state.apply. 
@@ -18,14 +20,12 @@
 
 QUIET=false
 UPTIME_REQ=1800 #in seconds, how long the box has to be up before considering restarting salt-minion due to /opt/so/log/salt/state-apply-test not being touched
-HIGHSTATE_UPTIME_REQ=900 #in seconds; if the box has been up this long and no highstate has completed since boot, force one
 CURRENT_TIME=$(date +%s)
 SYSTEM_START_TIME=$(date -d "$(</proc/uptime awk '{print $1}') seconds ago" +%s)
 LAST_HIGHSTATE_END=$([ -e "/opt/so/log/salt/lasthighstate" ] && date -r /opt/so/log/salt/lasthighstate +%s || echo 0)
 LAST_HEALTHCHECK_STATE_APPLY=$([ -e "/opt/so/log/salt/state-apply-test" ] && date -r /opt/so/log/salt/state-apply-test +%s || echo 0)
 # SETTING THRESHOLD TO ANYTHING UNDER 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default
-# THRESHOLD is derived from the salt schedule highstate interval + 1 hour, so the minion-check grace period tracks the schedule automatically.
-THRESHOLD=$(( ({{ SCHEDULEMERGED.highstate_interval_minutes }} + 60) * 60 )) #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted
+THRESHOLD={{SALT_MINION_DEFAULTS.salt.minion.check_threshold}} #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted
 THRESHOLD_DATE=$((LAST_HEALTHCHECK_STATE_APPLY+THRESHOLD))
 
 logCmd() {
@@ -77,50 +77,24 @@ done
 
 log "running so-salt-minion-check"
 
-RESTARTED=false
-
-# Check 1 (minion-restart-check): if the minion has stopped applying states (the
-# state-apply-test healthcheck file has gone stale), restart the salt-minion service.
 if [ $CURRENT_TIME -ge $((SYSTEM_START_TIME+$UPTIME_REQ))  ]; then
   if [ $THRESHOLD_DATE -le $CURRENT_TIME ]; then
-    log "[minion-restart-check] salt-minion is unable to apply states; restarting salt-minion" E
-    log "[minion-restart-check] state-apply-test not touched by required date `date -d @$THRESHOLD_DATE`, last touched `date -d @$LAST_HEALTHCHECK_STATE_APPLY`" I
-    log "[minion-restart-check] last highstate completed at `date -d @$LAST_HIGHSTATE_END`" I
-    log "[minion-restart-check] checking if any jobs are running" I
+    log "salt-minion is unable to apply states" E
+    log "/opt/so/log/salt/healthcheck-state-apply not touched by required date: `date -d @$THRESHOLD_DATE`, last touched: `date -d @$LAST_HEALTHCHECK_STATE_APPLY`" I
+    log "last highstate completed at `date -d @$LAST_HIGHSTATE_END`" I
+    log "checking if any jobs are running" I
     logCmd "salt-call --local saltutil.running" I
-    log "[minion-restart-check] ensure salt.minion-state-apply-test is enabled" I
+    log "ensure salt.minion-state-apply-test is enabled" I
     logCmd "salt-call state.enable salt.minion-state-apply-test" I
-    log "[minion-restart-check] ensure highstate is enabled" I
+    log "ensure highstate is enabled" I
     logCmd "salt-call state.enable highstate" I
-    log "[minion-restart-check] killing all salt-minion processes" I
+    log "killing all salt-minion processes" I
     logCmd "pkill -9 -ef /usr/bin/salt-minion" I
-    log "[minion-restart-check] starting salt-minion service" I
+    log "starting salt-minion service" I
     logCmd "systemctl start salt-minion" I
-    log "[minion-restart-check] waiting for salt-minion to become ready, then applying highstate in the background (queued)" I
-    nohup bash -c '/usr/sbin/so-salt-minion-wait; salt-call state.highstate queue=True' >> "/opt/so/log/salt/so-salt-minion-check" 2>&1 &
-    RESTARTED=true
   else
-    log "[minion-restart-check] healthy: state-apply-test last touched `date -d @$LAST_HEALTHCHECK_STATE_APPLY`, must go stale past `date -d @$THRESHOLD_DATE` to trigger a salt-minion restart" I
+    log "/opt/so/log/salt/healthcheck-state-apply last touched: `date -d @$LAST_HEALTHCHECK_STATE_APPLY` must be touched by `date -d @$THRESHOLD_DATE` to avoid salt-minion restart" I
   fi
 else
-  log "[minion-restart-check] skipped: system uptime $((CURRENT_TIME-SYSTEM_START_TIME))s is below the ${UPTIME_REQ}s minimum required before a salt-minion restart" I
-fi
-
-# Check 2 (boot-highstate-check): if the host has been up long enough but no highstate
-# has completed since this boot, force one. This recovers a host whose boot highstate
-# (so-boot-highstate.service) failed or was skipped, even while the minion is otherwise
-# healthy (touching state-apply-test). We deliberately do NOT enable highstate here: if
-# soup has disabled it during an upgrade, Salt will refuse the highstate and we avoid
-# forcing one mid-upgrade.
-if $RESTARTED; then
-  log "[boot-highstate-check] skipped: minion-restart-check already queued a highstate this run" I
-elif [ $CURRENT_TIME -lt $((SYSTEM_START_TIME+HIGHSTATE_UPTIME_REQ)) ]; then
-  log "[boot-highstate-check] skipped: system uptime $((CURRENT_TIME-SYSTEM_START_TIME))s is below the ${HIGHSTATE_UPTIME_REQ}s minimum required before forcing a highstate" I
-elif [ $LAST_HIGHSTATE_END -ge $SYSTEM_START_TIME ]; then
-  log "[boot-highstate-check] healthy: a highstate completed at `date -d @$LAST_HIGHSTATE_END`, after this boot at `date -d @$SYSTEM_START_TIME`" I
-elif salt-call --local saltutil.running 2>/dev/null | grep -q 'state.highstate'; then
-  log "[boot-highstate-check] no highstate has completed since boot, but one is already running; skipping" I
-else
-  log "[boot-highstate-check] no highstate has completed since boot after $((CURRENT_TIME-SYSTEM_START_TIME))s uptime; applying highstate" E
-  nohup bash -c 'salt-call state.highstate -l info queue=True' >> "/opt/so/log/salt/so-salt-minion-check" 2>&1 &
+  log "system uptime only $((CURRENT_TIME-SYSTEM_START_TIME)) seconds does not meet $UPTIME_REQ second requirement." I
 fi

salt/docker_clean/init.sls

@@ -9,8 +9,7 @@
 prune_images:
   cmd.run:
     - name: so-docker-prune
-    - onlyif: command -v /usr/sbin/so-docker-prune >/dev/null 2>&1
-    - order: 9000
+    - order: last
 
 {% else %}
 

salt/elastalert/enabled.sls

@@ -19,7 +19,6 @@ wait_for_elasticsearch:
 so-elastalert:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastalert:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: elastalert
     - name: so-elastalert
     - user: so-elastalert

salt/elastic-fleet-package-registry/enabled.sls

@@ -15,7 +15,6 @@ include:
 so-elastic-fleet-package-registry:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-fleet-package-registry:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - name: so-elastic-fleet-package-registry
     - hostname: Fleet-package-reg-{{ GLOBALS.hostname }}
     - detach: True

salt/elasticagent/enabled.sls

@@ -16,7 +16,6 @@ include:
 so-elastic-agent:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - name: so-elastic-agent
     - hostname: {{ GLOBALS.hostname }}
     - detach: True

salt/elasticfleet/config.sls

@@ -173,7 +173,7 @@ eaoptionalintegrationsdir:
 
 {% for minion in node_data %}
 {% set role = node_data[minion]["role"] %}
-{% if role in [ "eval","fleet","import","manager", "managerhype", "managersearch","standalone" ] %}
+{% if role in [ "eval","fleet","heavynode","import","manager", "managerhype", "managersearch","standalone" ] %}
 {% set optional_integrations = ELASTICFLEETMERGED.optional_integrations %}
 {% set integration_keys = optional_integrations.keys() %}
 fleet_server_integrations_{{ minion }}:

salt/elasticfleet/content-defaults.map.jinja

@@ -9,6 +9,7 @@
 
 {% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
 {% set ADDON_CONTENT_INTEGRATION_DEFAULTS = {} %}
+{% set DEBUG_STUFF = {} %}
 
 {% for pkg in ADDON_CONTENT_PACKAGE_COMPONENTS %}
 {%   if pkg.name in CORE_ESFLEET_PACKAGES %}

salt/elasticfleet/defaults.yaml

@@ -1,5 +1,6 @@
 elasticfleet:
   enabled: False
+  patch_version: 9.3.3+build202604082258 # Elastic Agent specific patch release.
   enable_manager_output: True
   config:
     server:

salt/elasticfleet/enabled.sls

@@ -26,9 +26,7 @@ include:
 wait_for_elasticsearch_elasticfleet:
   cmd.run:
     - name: so-elasticsearch-wait
-{% endif %}
 
-{% if GLOBALS.role == "so-fleet" %}
 # Sync Elastic Agent artifacts to Fleet Node
 elasticagent_syncartifacts:
   file.recurse:
@@ -42,7 +40,6 @@ elasticagent_syncartifacts:
 so-elastic-fleet:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - name: so-elastic-fleet
     - hostname: FleetServer-{{ GLOBALS.hostname }}
     - detach: True
@@ -102,17 +99,6 @@ so-elastic-fleet:
       - file: trusttheca
       - x509: etc_elasticfleet_key
       - x509: etc_elasticfleet_crt
-
-wait_for_so-elastic-fleet:
-  http.wait_for_successful_query:
-    - name: "https://localhost:8220/api/status"
-    - ssl: True
-    - verify_ssl: False
-    - status: 200
-    - wait_for: 300
-    - request_interval: 15
-    - require:
-      - docker_container: so-elastic-fleet
 {%   endif %}
 
 delete_so-elastic-fleet_so-status.disabled:

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -5,7 +5,7 @@
   "package": {
     "name": "endpoint",
     "title": "Elastic Defend",
-    "version": "9.3.1",
+    "version": "9.3.0",
     "requires_root": true
   },
   "enabled": true,

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -29,7 +29,7 @@
               "\\.gz$"
             ],
             "include_files": [],
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.20.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.3\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.20.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.20.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.3\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.15.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.15.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.15.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
             ],

salt/elasticfleet/input-defaults.map.jinja

@@ -9,6 +9,7 @@
 
 {% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
 {% set ADDON_INPUT_INTEGRATION_DEFAULTS = {} %}
+{% set DEBUG_STUFF = {} %}
 
 {% for pkg in ADDON_INPUT_PACKAGE_COMPONENTS %}
 {%   if pkg.name in CORE_ESFLEET_PACKAGES %}
@@ -115,6 +116,7 @@
 
 
 {%         do ADDON_INPUT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
+{%         do DEBUG_STUFF.update({integration_key: "Generating defaults for "+ pkg.name })%}
 {%       endfor %}
 {%     endif %}
 {%   endif %}

salt/elasticfleet/install_agent_grid.sls

@@ -10,15 +10,6 @@
 {% set AGENT_STATUS = salt['service.available']('elastic-agent') %}
 {% set AGENT_EXISTS = salt['file.file_exists']('/opt/Elastic/Agent/elastic-agent') %}
 
-so-elastic-agent-install:
-  file.managed:
-    - name: /usr/sbin/so-elastic-agent-install
-    - source: salt://elasticfleet/tools/sbin/so-elastic-agent-install
-    - user: 947
-    - group: 939
-    - mode: 755
-    - show_changes: False
-
 {% if not AGENT_STATUS or not AGENT_EXISTS %}
 
 pull_agent_installer:
@@ -30,9 +21,11 @@ pull_agent_installer:
 
 run_installer:
   cmd.run:
-    - name: /usr/sbin/so-elastic-agent-install "{{ GRIDNODETOKEN }}"
-    - require:
-      - file: pull_agent_installer
+    - name: ./so-elastic-agent_linux_amd64 -token={{ GRIDNODETOKEN }} -force
+    - cwd: /opt/so
+    - retry:
+        attempts: 3
+        interval: 20
 
 cleanup_agent_installer:
   file.absent:

salt/elasticfleet/manager.sls

@@ -9,20 +9,16 @@
 
 include:
   - elasticfleet.config
-  - kibana.enabled
 
 # If enabled, automatically update Fleet Logstash Outputs
-{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration %}
-{%   if grains.role not in ['so-import', 'so-eval']%}
+{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval'] %}
 so-elastic-fleet-auto-configure-logstash-outputs:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-outputs-update
     - retry:
         attempts: 4
         interval: 30
-    - require:
-      - http: wait_for_so-kibana
-{%   endif %}
+{% endif %}
 
 # If enabled, automatically update Fleet Server URLs & ES Connection
 so-elastic-fleet-auto-configure-server-urls:
@@ -31,9 +27,6 @@ so-elastic-fleet-auto-configure-server-urls:
     - retry:
         attempts: 4
         interval: 30
-    - require:
-      - http: wait_for_so-kibana
-{% endif %}
 
 # Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
 so-elastic-fleet-auto-configure-elasticsearch-urls:
@@ -42,8 +35,6 @@ so-elastic-fleet-auto-configure-elasticsearch-urls:
     - retry:
         attempts: 4
         interval: 30
-    - require:
-      - http: wait_for_so-kibana
 
 so-elastic-fleet-auto-configure-artifact-urls:
   cmd.run:
@@ -51,8 +42,6 @@ so-elastic-fleet-auto-configure-artifact-urls:
     - retry:
         attempts: 4
         interval: 30
-    - require:
-      - http: wait_for_so-kibana
 
 so-elastic-fleet-package-statefile:
   file.managed:
@@ -64,9 +53,9 @@ so-elastic-fleet-package-upgrade:
     - name: /usr/sbin/so-elastic-fleet-package-upgrade
     - retry:
         attempts: 3
-        interval: 30
-    - require:
-      - http: wait_for_so-kibana
+        interval: 10
+    - onchanges:
+      - file: /opt/so/state/elastic_fleet_packages.txt
 
 so-elastic-fleet-integrations:
   cmd.run:
@@ -74,8 +63,6 @@ so-elastic-fleet-integrations:
     - retry:
         attempts: 3
         interval: 10
-    - require:
-      - http: wait_for_so-kibana
 
 so-elastic-agent-grid-upgrade:
   cmd.run:
@@ -83,8 +70,6 @@ so-elastic-agent-grid-upgrade:
     - retry:
         attempts: 12
         interval: 5
-    - require:
-      - http: wait_for_so-kibana
 
 so-elastic-fleet-integration-upgrade:
   cmd.run:
@@ -92,22 +77,16 @@ so-elastic-fleet-integration-upgrade:
     - retry:
         attempts: 3
         interval: 10
-    - require:
-      - http: wait_for_so-kibana
 
 {# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
 so-elastic-fleet-addon-integrations:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
-    - require:
-      - http: wait_for_so-kibana
 
 {% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
 so-elastic-defend-manage-filters-file-watch:
   cmd.run:
     - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log
-    - require:
-      - http: wait_for_so-kibana
     - onchanges:
       - file: elasticdefendcustom
       - file: elasticdefenddisabled

salt/elasticfleet/tools/sbin/so-elastic-agent-install

@@ -1,100 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-elastic-fleet-common
-
-
-# passed in as arg from elasticfleet/install_agent_grid.sls, else pulled from pillar later
-GRIDNODETOKEN="$1"
-LOGFILE="/opt/so/SO-Elastic-Agent_Installer_Health.log"
-
-check_agent_health() {
-    timeout=300
-    interval=10
-    start=$SECONDS
-
-    while (( SECONDS - start < timeout )); do
-        agent_status=$(elastic-agent status 2>&1)
-        echo -e "\n$(date)\n$agent_status\n" >> "$LOGFILE"
-        if echo "$agent_status" | grep -A1 'elastic-agent$' | grep -q 'status: (HEALTHY)'; then
-            return 0
-        fi
-        echo "The Elastic Agent is not yet healthy. Waiting for ${interval} seconds before checking again..."
-        sleep "$interval"
-    done
-
-    echo "The Elastic Agent did not become healthy within ${timeout} seconds"
-    return 1
-}
-
-uninstall_agent() {
-    if command -v elastic-agent >/dev/null 2>&1; then
-        elastic-agent uninstall -f
-    fi
-}
-
-if [[ -z "$GRIDNODETOKEN" ]]; then
-    noderole=$(so-yaml.py get -r /etc/salt/grains role)
-    if [[ "$noderole" == "so-heavynode" ]]; then
-        GRIDNODETOKEN=$(salt-call pillar.get global:fleet_grid_enrollment_token_heavy --out=newline_values_only)
-    else
-        GRIDNODETOKEN=$(salt-call pillar.get global:fleet_grid_enrollment_token_general --out=newline_values_only)
-    fi
-fi
-
-if [[ -z "$GRIDNODETOKEN" ]]; then
-    echo "Unable to determine Elastic Fleet enrollment token. Exiting."
-
-    exit 1
-fi
-
-if [[ ! -x /opt/so/so-elastic-agent_linux_amd64 ]]; then
-    echo "Downloading so-elastic-agent installer... This could take a while if another Salt job is running."
-
-    # When running outside of elasticfleet/install_agent_grid.sls we need to download the installer independently.
-    #  PYTHONWARNINGS="ignore" to avoid messages like the following when running salt-call:
-    #  '/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/base.py:129: TransportWarning: Unclosed transport! <salt.transport.zeromq.RequestClient object at 0x7fc5f0ee7a30>
-    #    File "/bin/salt-call", line 12, in <module>
-    #      sys.exit(salt_call())'
-
-    PYTHONWARNINGS="ignore" salt-call state.single file.managed name=/opt/so/so-elastic-agent_linux_amd64 source=salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64 mode=755 makedirs=True queue=True
-
-fi
-
-if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then
-    attempts=0
-    cd /opt/so/ || exit 1
-
-    truncate -s 0 "$LOGFILE"
-
-    uninstall_agent
-
-    while [[ $attempts -lt 3 ]]; do
-        if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force && echo "Verifying Elastic Agent health..." && check_agent_health; then
-            rm -f /opt/so/so-elastic-agent_linux_amd64
-            elastic-agent status
-
-            exit 0
-        fi
-
-        attempts=$((attempts + 1))
-
-        if [[ $attempts -lt 3 ]]; then
-            echo "Unable to verify Elastic Agent health... Retrying in 20 seconds..."
-            sleep 20
-        fi
-    done
-
-    uninstall_agent
-    rm -f /opt/so/so-elastic-agent_linux_amd64
-    echo "The so-elastic-agent installer failed after 3 attempts. Exiting."
-
-    exit 1
-else
-    echo "Unable to locate so-elastic-agent installer. Exiting."
-
-    exit 1
-fi

salt/elasticfleet/tools/sbin/so-elastic-fleet-common

@@ -30,94 +30,6 @@ fleet_api() {
     curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/${QUERYPATH}" "$@" --retry 3 --retry-delay 10 --fail 2>/dev/null
 }
 
-# Max number of concurrent Fleet write jobs (create/update). Override via env if needed.
-MAX_FLEET_JOBS=${MAX_FLEET_JOBS:-10}
-
-# Block until fewer than MAX_FLEET_JOBS background jobs are running.
-elastic_fleet_throttle() {
-    while (( $(jobs -rp | wc -l) >= MAX_FLEET_JOBS )); do
-        wait -n || true
-    done
-}
-
-# Load every integration JSON in a directory into a single agent policy.
-# The agent policy is fetched ONCE (not per file), and the create/update writes
-# are dispatched as throttled background jobs.
-#   $1 AGENT_POLICY     - the agent policy id/name to load integrations into
-#   $2 DIR              - directory of integration *.json files
-#   $3 LABEL           - human-readable label for log output
-#   $4 SKIP_CREATE_NAME - (optional) integration name to skip when creating (still updated if present)
-# Returns 1 if the policy cannot be fetched or if any integration failed to create/update.
-elastic_fleet_load_integrations_dir() {
-    local AGENT_POLICY=$1
-    local DIR=$2
-    local LABEL=$3
-    local SKIP_CREATE_NAME=$4
-    local POLICY_JSON FAIL_FILE OUT_DIR INTEGRATION NAME ID i
-
-    FAIL_FILE=$(mktemp)
-    # Each job buffers its full output (header + API response) into its own file so the
-    # parent can print them grouped and in submission order after concurrent writes finish.
-    OUT_DIR=$(mktemp -d)
-    i=0
-
-    # Fetch the agent policy a single time; we look up integration ids locally below.
-    if ! POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY"); then
-        echo "Error: Failed to retrieve agent policy '$AGENT_POLICY'."
-        rm -f "$FAIL_FILE"
-        rm -rf "$OUT_DIR"
-        return 1
-    fi
-
-    if ! jq -e '.item.package_policies' <<<"$POLICY_JSON" >/dev/null 2>&1; then
-        echo "Error: Invalid agent policy response for '$AGENT_POLICY'."
-        rm -f "$FAIL_FILE"
-        rm -rf "$OUT_DIR"
-        return 1
-    fi
-
-    for INTEGRATION in "$DIR"/*.json; do
-        [ -e "$INTEGRATION" ] || continue
-        NAME=$(jq -r .name "$INTEGRATION")
-        ID=$(jq -r --arg n "$NAME" '.item.package_policies[]? | select(.name==$n) | .id' <<<"$POLICY_JSON")
-
-        elastic_fleet_throttle
-        {
-            local RESP
-            if [ -n "$ID" ]; then
-                printf "\n\n%s - Updating integration %s\n" "$LABEL" "$NAME"
-                if ! RESP=$(elastic_fleet_integration_update "$ID" "@$INTEGRATION"); then
-                    flock 9; echo "update ${INTEGRATION##*/}" >&9
-                fi
-                printf '%s\n' "$RESP"
-            elif [ -n "$SKIP_CREATE_NAME" ] && [ "$NAME" == "$SKIP_CREATE_NAME" ]; then
-                printf "\n\n%s - Skipping creation of %s\n" "$LABEL" "$NAME"
-            else
-                printf "\n\n%s - Creating integration %s\n" "$LABEL" "$NAME"
-                if ! RESP=$(elastic_fleet_integration_create "@$INTEGRATION"); then
-                    flock 9; echo "create ${INTEGRATION##*/}" >&9
-                fi
-                printf '%s\n' "$RESP"
-            fi
-        } >"$OUT_DIR/$(printf '%03d' "$i")" 9>>"$FAIL_FILE" &
-        i=$((i+1))
-    done
-    wait || true
-
-    # Emit per-integration output grouped and in submission order (glob sorts numerically).
-    cat "$OUT_DIR"/* 2>/dev/null
-    rm -rf "$OUT_DIR"
-
-    local rc=0
-    if [ -s "$FAIL_FILE" ]; then
-        printf "\n%s: failed integrations:\n" "$LABEL"
-        cat "$FAIL_FILE"
-        rc=1
-    fi
-    rm -f "$FAIL_FILE"
-    return $rc
-}
-
 elastic_fleet_integration_check() {
 
     AGENT_POLICY=$1
@@ -134,9 +46,7 @@ elastic_fleet_integration_create() {
 
     JSON_STRING=$1
 
-    # --retry-all-errors so transient 409 conflicts (concurrent writes to the same agent
-    # policy) are retried; curl --retry alone does not retry 409.
-    if ! fleet_api "package_policies" --retry-all-errors -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPOST -d "$JSON_STRING"; then
+    if ! fleet_api "package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPOST -d "$JSON_STRING"; then
         return 1
     fi
 }
@@ -167,9 +77,7 @@ elastic_fleet_integration_update() {
 
     JSON_STRING=$2
 
-    # --retry-all-errors so transient 409 conflicts (concurrent writes to the same agent
-    # policy) are retried; curl --retry alone does not retry 409.
-    if ! fleet_api "package_policies/$UPDATE_ID" --retry-all-errors -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPUT -d "$JSON_STRING"; then
+    if ! fleet_api "package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPUT -d "$JSON_STRING"; then
         return 1
     fi
 }

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -9,45 +9,108 @@
 RETURN_CODE=0
 
 if [ ! -f /opt/so/state/eaintegrations.txt ]; then
+  # First, check for any package upgrades
+  /usr/sbin/so-elastic-fleet-package-upgrade
 
-  # update Fleet Server policies
+  # Second, update Fleet Server policies
   /usr/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server
 
-  # configure Elastic Defend Integration separately
+  # Third, configure Elastic Defend Integration seperately
   /usr/sbin/so-elastic-fleet-integration-policy-elastic-defend
 
-  # Each group fetches its agent policy once and dispatches create/update writes concurrently.
-
   # Initial Endpoints
-  elastic_fleet_load_integrations_dir "endpoints-initial" \
-    /opt/so/conf/elastic-fleet/integrations/endpoints-initial "Initial Endpoints Policy" || RETURN_CODE=1
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/endpoints-initial/*.json; do
+    printf "\n\nInitial Endpoints Policy - Loading $INTEGRATION\n"
+    elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
+    if [ -n "$INTEGRATION_ID" ]; then
+      printf "\n\nIntegration $NAME exists - Updating integration\n"
+      if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+        echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
+    else
+      printf "\n\nIntegration does not exist - Creating integration\n"
+      if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
+    fi
+  done
 
   # Grid Nodes - General
-  elastic_fleet_load_integrations_dir "so-grid-nodes_general" \
-    /opt/so/conf/elastic-fleet/integrations/grid-nodes_general "Grid Nodes Policy_General" || RETURN_CODE=1
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_general/*.json; do
+    printf "\n\nGrid Nodes Policy_General - Loading $INTEGRATION\n"
+    elastic_fleet_integration_check "so-grid-nodes_general" "$INTEGRATION"
+    if [ -n "$INTEGRATION_ID" ]; then
+      printf "\n\nIntegration $NAME exists - Updating integration\n"
+      if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+        echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
+    else
+      printf "\n\nIntegration does not exist - Creating integration\n"
+      if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
+    fi
+  done
 
   # Grid Nodes - Heavy
-  elastic_fleet_load_integrations_dir "so-grid-nodes_heavy" \
-    /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy "Grid Nodes Policy_Heavy" || RETURN_CODE=1
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy/*.json; do
+    printf "\n\nGrid Nodes Policy_Heavy - Loading $INTEGRATION\n"
+    elastic_fleet_integration_check "so-grid-nodes_heavy" "$INTEGRATION"
+    if [ -n "$INTEGRATION_ID" ]; then
+      printf "\n\nIntegration $NAME exists - Updating integration\n"
+      if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+        echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
+    else
+      printf "\n\nIntegration does not exist - Creating integration\n"
+      if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
+    fi
+  done
 
-  # Fleet Server - Optional integrations (adds integration configuration to a given FleetServer_ policy)
-  for FLEET_DIR in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/; do
-    [ -d "$FLEET_DIR" ] || continue
-    INTEGRATIONS=("${FLEET_DIR%/}"/*.json)
-    [ -e "${INTEGRATIONS[0]}" ] || continue
-
-    FLEET_POLICY=$(basename "$FLEET_DIR")
-    elastic_fleet_load_integrations_dir "$FLEET_POLICY" \
-      "${FLEET_DIR%/}" "Fleet Server Policy" "elasticsearch-logs" || RETURN_CODE=1
+  # Fleet Server - Optional integrations
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json; do
+    if ! [ "$INTEGRATION" == "/opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json" ]; then
+      FLEET_POLICY=`echo "$INTEGRATION"| cut -d'/' -f7`
+      printf "\n\nFleet Server Policy - Loading $INTEGRATION\n"
+      elastic_fleet_integration_check "$FLEET_POLICY" "$INTEGRATION"
+      if [ -n "$INTEGRATION_ID" ]; then
+        printf "\n\nIntegration $NAME exists - Updating integration\n"
+        if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+          echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+          RETURN_CODE=1
+          continue
+        fi
+      else
+        printf "\n\nIntegration does not exist - Creating integration\n"
+        if [ "$NAME" != "elasticsearch-logs" ]; then
+          if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+            echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+            RETURN_CODE=1
+            continue
+          fi
+        fi
+      fi
+    fi
   done
 
   # Only create the state file if all policies were created/updated successfully
-  if [[ $RETURN_CODE -eq 0 ]]; then
+  if [[ "$RETURN_CODE" != "1" ]]; then
     touch /opt/so/state/eaintegrations.txt
-  else
-    exit 1
   fi
 else
-  echo "Fleet integration policies already loaded."
-  exit 0
+  exit $RETURN_CODE
 fi

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers

@@ -30,7 +30,7 @@ done
 if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then 
   printf "\nFleet Host URL, Enrollment Token or Elastic Version empty - exiting..."
   printf "\nFleet Host: $FLEETHOST, Enrollment Token: $ENROLLMENTOKEN\n"
-  exit 1
+  exit 
 fi
 
 OSARCH=( "linux-x86_64" "windows-x86_64" "darwin-x86_64" "darwin-aarch64" )
@@ -62,54 +62,31 @@ do
 done 
 
 GOTARGETOS=( "linux" "windows" "darwin" "darwin/arm64" )
+GOARCH="amd64"
 printf "\n### Generating OS packages using the cleaned up tarballs"
-for GOOS in "${GOTARGETOS[@]}"; do
-    GOARCH="amd64"
+for GOOS in "${GOTARGETOS[@]}"
+do
     if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi
     printf "\n\n### Generating $GOOS/$GOARCH Installer...\n"
     docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \
     --mount type=bind,source=/etc/pki/tls/certs/,target=/workspace/files/cert/ \
     --mount type=bind,source=/nsm/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \
-    --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ \
+    --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \
     {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN"  -o /output/so-elastic-agent_${GOOS}_${GOARCH}
     printf "\n### $GOOS/$GOARCH Installer Generated...\n"
 done
 
 printf "\n\n### Generating MSI...\n"
-cp /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe
+cp /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe
 docker run \
---mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ -w /output \
+--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ -w /output \
 {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs
 printf "\n### MSI Generated...\n"
 
-# Verify installers were created
-for GOOS in "${GOTARGETOS[@]}"; do
-    GOARCH="amd64"
-    if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin"; GOARCH="arm64"; fi
-    if [[ ! -f /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_${GOOS}_${GOARCH} ]]; then
-        printf "\n### ERROR: Installer for %s/%s was not generated. Exiting...\n" "$GOOS" "$GOARCH"
-        exit 1
-    fi
-    # After verifying new installer was generated, move it to so_agent-installers directory
-    mv /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_${GOOS}_${GOARCH} /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/
-done
-
-# Verify MSI installer
-if [[ ! -f /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64_msi ]]; then
-    printf "\n### ERROR: Installer MSI was not generated. Exiting...\n"
-    exit 1
-else
-    # After verifying new installer MSI was generated, move it to so_agent-installers directory
-    mv /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64_msi /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/
-fi
-
 printf "\n### Cleaning up temp files \n"
 rm -rf /nsm/elastic-agent-workspace
-rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe
+rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe
 
 printf "\n### Copying so_agent-installers to /nsm/elastic-fleet/ for nginx.\n"
 \cp -vr /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ /nsm/elastic-fleet/
 chmod 644 /nsm/elastic-fleet/so_agent-installers/*
-
-# if we got here all installers have been generated successfully
-exit 0

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade

@@ -23,90 +23,73 @@ if [ $? -ne 0 ]; then
 fi
 
 default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %})
-# JSON array of the default packages, used by the jq filter below.
-default_packages_json=$(printf '%s\n' "${default_packages[@]}" | jq -R . | jq -s '.')
-
-# Output lock (serializes concurrent job output) and failure file (one marker line per
-# failed integration). Mirrors the pattern used by elastic_fleet_load_integrations_dir.
-OUTPUT_LOCK=$(mktemp)
-FAIL_FILE=$(mktemp)
-trap 'rm -f "$OUTPUT_LOCK" "$FAIL_FILE"' EXIT
-
-# Cache of package name -> latest available version, so the same package is only looked up
-# once instead of once per (policy, integration).
-declare -A LATEST_VERSION_CACHE
 
+ERROR=false
 for AGENT_POLICY in $agent_policies; do
-    # Fetch the agent policy a single time; package name/version and integration id are all
-    # extracted locally below instead of re-fetching the same policy per integration.
-    if ! POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY"); then
+    if ! integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY"); then
         # this script upgrades default integration packages, exit 1 and let salt handle retrying
         exit 1
     fi
-
-    # One jq pass emits name/package.name/package.version/id for every eligible integration.
-    # The endpoint/fleet_server skips and the default-package gate are applied here in jq.
-    # $defaults (not $def, a jq reserved keyword) holds the default package list.
-    while IFS=$'\t' read -r INTEGRATION PACKAGE_NAME PACKAGE_VERSION INTEGRATION_ID; do
-        [ -n "$INTEGRATION" ] || continue
-
-        # Look up the latest available version once per package, then memoize it.
-        if [[ -z "${LATEST_VERSION_CACHE[$PACKAGE_NAME]+set}" ]]; then
-            if ! AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME"); then
-                echo "Error: Failed getting latest version for $PACKAGE_NAME"
+    for INTEGRATION in $integrations; do
+        if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then
+            # Get package name so we know what package to look for when checking the current and latest available version
+            if ! PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION"); then
                 exit 1
             fi
-            LATEST_VERSION_CACHE[$PACKAGE_NAME]=$AVAILABLE_VERSION
-        fi
-        AVAILABLE_VERSION=${LATEST_VERSION_CACHE[$PACKAGE_NAME]}
-
-        if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then
-            # Dry run, then (if clean) the actual upgrade, dispatched as a throttled background
-            # job. Each job builds its full log into one block, then flushes it under a single
-            # shared lock (OUTPUT_LOCK) so concurrent jobs never interleave on stdout; a failed
-            # job also appends a marker line to FAIL_FILE while holding that same lock.
-            elastic_fleet_throttle
-            {
-                block=$'\n'"Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..."$'\n'
-                block+="Upgrading $INTEGRATION..."$'\n'"Starting dry run..."$'\n'
-                fail=""
-                if ! DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID"); then
-                    block+="Error: Failed to complete dry run for '$INTEGRATION_ID'."$'\n'
-                    fail="dryrun $INTEGRATION"
-                elif [[ "$(jq .[].hasErrors <<<"$DRYRUN_OUTPUT")" == "false" ]]; then
-                    block+="No errors detected. Proceeding with upgrade..."$'\n'
-                    if ! elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"; then
-                        block+="Error: Upgrade failed for $PACKAGE_NAME with integration ID '$INTEGRATION_ID'."$'\n'
-                        fail="upgrade $INTEGRATION"
+            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+            if [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then
+            {%- endif %}
+                # Get currently installed version of package
+                attempt=0
+                max_attempts=3
+                while [ $attempt -lt $max_attempts ]; do
+                    if PACKAGE_VERSION=$(elastic_fleet_integration_policy_package_version "$AGENT_POLICY" "$INTEGRATION") && AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME"); then
+                        break
                     fi
-                else
-                    block+="Errors detected during dry run for $PACKAGE_NAME policy upgrade..."$'\n'
-                    fail="dryrun-errors $INTEGRATION"
+                    attempt=$((attempt + 1))
+                done
+                if [ $attempt -eq $max_attempts ]; then
+                    echo "Error: Failed getting $PACKAGE_VERSION or $AVAILABLE_VERSION"
+                    exit 1
                 fi
-                {
-                    flock 9
-                    printf '%s' "$block"
-                    [ -n "$fail" ] && printf '%s\n' "$fail" >>"$FAIL_FILE"
-                } 9>>"$OUTPUT_LOCK"
-            } &
+
+                # Get integration ID
+                if ! INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION"); then
+                    exit 1
+                fi
+
+                if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then
+                    # Dry run of the upgrade
+                    echo ""
+                    echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..."
+                    echo "Upgrading $INTEGRATION..."
+                    echo "Starting dry run..."
+                    if ! DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID"); then
+                        exit 1
+                    fi
+                    DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors)
+
+                    # If no errors with dry run, proceed with actual upgrade
+                    if [[ "$DRYRUN_ERRORS" == "false" ]]; then
+                        echo "No errors detected. Proceeding with upgrade..."
+                        if ! elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"; then
+                            echo "Error: Upgrade failed for $PACKAGE_NAME with integration ID '$INTEGRATION_ID'."
+                            ERROR=true
+                            continue
+                        fi
+                    else
+                        echo "Errors detected during dry run for $PACKAGE_NAME policy upgrade..."
+                        ERROR=true
+                        continue
+                    fi
+                fi
+            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+            fi
+            {%- endif %}
         fi
-    done < <(jq -r --argjson defaults "$default_packages_json" '
-        .item.package_policies[]
-        | select(.name != "elastic-defend-endpoints")
-        | select(.name | startswith("fleet_server-") | not)
-        {%- if not AUTO_UPGRADE_INTEGRATIONS %}
-        | select(.package.name | IN($defaults[]))
-        {%- endif %}
-        | [.name, .package.name, .package.version, .id] | @tsv
-    ' <<<"$POLICY_JSON")
+    done
 done
-
-# Barrier: wait for every dispatched dry-run/upgrade job to finish.
-wait
-
-if [ -s "$FAIL_FILE" ]; then
-    printf '\nFailed integration upgrades:\n'
-    cat "$FAIL_FILE"
+if [[ "$ERROR" == "true" ]]; then
     exit 1
 fi
 echo

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load

@@ -16,6 +16,7 @@
 STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt
 INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json
 BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
+BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
 BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json
 INTEGRATION_PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
 INPUT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_input_package_components.json
@@ -28,6 +29,29 @@ PENDING_UPDATE=false
 #   Requiring some level of manual Elastic Stack configuration before installation
 EXCLUDED_INTEGRATIONS=('apm')
 
+version_conversion(){
+    version=$1
+    echo "$version" | awk -F '.' '{ printf("%d%03d%03d\n", $1, $2, $3); }'
+}
+
+compare_versions() {
+    version1=$1
+    version2=$2
+
+    # Convert versions to numbers
+    num1=$(version_conversion "$version1")
+    num2=$(version_conversion "$version2")
+
+    # Compare using bc
+    if (( $(echo "$num1 < $num2" | bc -l) )); then
+        echo "less"
+    elif (( $(echo "$num1 > $num2" | bc -l) )); then
+        echo "greater"
+    else
+        echo "equal"
+    fi
+}
+
 IFS=$'\n'
 agent_policies=$(elastic_fleet_agent_policy_ids)
 if [ $? -ne 0 ]; then
@@ -39,23 +63,23 @@ default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.l
 
 in_use_integrations=()
 
-# Fetch each agent policy once; its package_policies[] already contain both the integration name
-#  and the .package.name, so extract all non-default package names locally in a single jq instead
-#  of re-fetching the same policy per integration.
-default_packages_json=$(printf '%s\n' "${default_packages[@]}" | jq -R . | jq -s '.')
 for AGENT_POLICY in $agent_policies; do
 
-    if ! policy_json=$(fleet_api "agent_policies/$AGENT_POLICY"); then
+    if ! integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY"); then
         # skip the agent policy if we can't get required info, let salt retry. Integrations loaded by this script are non-default integrations.
         echo "Skipping $AGENT_POLICY.. "
         continue
     fi
-    # non-default integrations that are in-use in any policy
-    while IFS= read -r PACKAGE_NAME; do
-        [ -n "$PACKAGE_NAME" ] && in_use_integrations+=("$PACKAGE_NAME")
-    done < <(jq -r --argjson defaults "$default_packages_json" \
-        '.item.package_policies[].package.name | select(. as $n | ($defaults | index($n)) | not)' \
-        <<<"$policy_json")
+    for INTEGRATION in $integrations; do
+        if ! PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION"); then
+            echo  "Not adding $INTEGRATION, couldn't get package name"
+            continue
+        fi
+        # non-default integrations that are in-use in any policy
+        if ! [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then
+            in_use_integrations+=("$PACKAGE_NAME")
+        fi
+    done
 done
 
 if [[ -f $STATE_FILE_SUCCESS  ]]; then
@@ -66,55 +90,72 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
         rm -f $INSTALLED_PACKAGE_LIST
         echo $latest_package_list | jq '{packages: [.items[] | {name: .name, latest_version: .version, installed_version: .installationInfo.version, subscription: .conditions.elastic.subscription }]}' >> $INSTALLED_PACKAGE_LIST
 
-        # Build the bulk install list and the per-package status messages with two jq passes
-        #  instead of a per-package bash loop. The old loop forked ~10 processes per package
-        #  (5 jq + awk/bc for the version compare) and re-parsed/rewrote a growing JSON file on
-        #  every add (O(n^2)). Selection and messages below are identical to that logic.
-        SUB={% if SUB %}true{% else %}false{% endif %}
-        AUTOUP={% if AUTO_UPGRADE_INTEGRATIONS %}true{% else %}false{% endif %}
-        EXCLUDED_JSON=$(printf '%s\n' "${EXCLUDED_INTEGRATIONS[@]}" | jq -R 'select(length>0)' | jq -s '.')
-        INUSE_JSON=$(printf '%s\n' "${in_use_integrations[@]}" | jq -R 'select(length>0)' | jq -s 'unique')
+        while read -r package; do
+            # get package details
+            package_name=$(echo "$package" | jq -r '.name')
+            latest_version=$(echo "$package" | jq -r '.latest_version')
+            installed_version=$(echo "$package" | jq -r '.installed_version')
+            subscription=$(echo "$package" | jq -r '.subscription')
+            bulk_package=$(echo "$package" | jq '{name: .name, version: .latest_version}' )
 
-        # vnum replicates the previous version_conversion (%d%03d%03d of the first three dotted
-        #  fields); needs() replicates the excluded/subscription/installed/upgrade/in-use logic.
-        JQ_DECISION='
-def vnum:
-  [ (split(".")|.[0:3][] | gsub("[^0-9].*";"") | (if .=="" then "0" else . end) | tonumber) ]
-  | (.[0]//0)*1000000 + (.[1]//0)*1000 + (.[2]//0);
-def needs($sub;$autoup;$excluded;$inuse):
-  .name as $n
-  | ($n | IN($excluded[]) | not)
-  and ( $sub or (.subscription==null or .subscription=="basic" or .subscription=="") )
-  and ( (.installed_version==null or .installed_version=="")
-        or ( ((.latest_version|vnum) > (.installed_version|vnum))
-             and ( $autoup or ($n | IN($inuse[]) | not) ) ) );'
+            if [[ ! "${EXCLUDED_INTEGRATIONS[@]}" =~ "$package_name" ]]; then
+            {% if not SUB %}
+                if [[ "$subscription" != "basic" && "$subscription" != "null" && -n "$subscription" ]]; then
+                    # pass over integrations that require non-basic elastic license
+                    echo "$package_name integration requires an Elastic license of $subscription or greater... skipping"
+                    continue
+                else
+                    if [[ "$installed_version" == "null" || -z "$installed_version" ]]; then
+                        echo "$package_name is not installed... Adding to next update."
+                        jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
 
-        JQ_ARGS=(--argjson sub "$SUB" --argjson autoup "$AUTOUP" --argjson excluded "$EXCLUDED_JSON" --argjson inuse "$INUSE_JSON")
+                        PENDING_UPDATE=true
+                    else
+                        results=$(compare_versions "$latest_version" "$installed_version")
+                        if [ $results == "greater" ]; then
+                            {#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations  #}
+                            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                            if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then
+                            {%- endif %}
+                                echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
+                                jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
 
-        # (a) Per-package status messages (parity with the previous echo output).
-        jq -r "${JQ_ARGS[@]}" "$JQ_DECISION"'
-          .packages[]
-          | .name as $n
-          | if ($n|IN($excluded[])) then "Skipping \($n)..."
-            elif (($sub|not) and (.subscription!=null and .subscription!="basic" and .subscription!="")) then
-                 "\($n) integration requires an Elastic license of \(.subscription) or greater... skipping"
-            elif (.installed_version==null or .installed_version=="") then
-                 "\($n) is not installed... Adding to next update."
-            elif ((.latest_version|vnum) > (.installed_version|vnum)) then
-                 (if ($autoup or ($n|IN($inuse[])|not))
-                  then "\($n) is at version \(.installed_version) latest version is \(.latest_version)... Adding to next update."
-                  else "skipping available upgrade for in use integration - \($n)." end)
-            else empty end
-        ' "$INSTALLED_PACKAGE_LIST"
-
-        # (b) The bulk install list, built in a single pass.
-        jq "${JQ_ARGS[@]}" "$JQ_DECISION"'
-          {packages: [ .packages[] | select(needs($sub;$autoup;$excluded;$inuse)) | {name, version: .latest_version} ]}
-        ' "$INSTALLED_PACKAGE_LIST" > "$BULK_INSTALL_PACKAGE_LIST"
-
-        if jq -e '.packages | length > 0' "$BULK_INSTALL_PACKAGE_LIST" >/dev/null; then
-            PENDING_UPDATE=true
-        fi
+                                PENDING_UPDATE=true
+                            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                            else
+                                echo "skipping available upgrade for in use integration - $package_name."
+                            fi
+                            {%- endif %}
+                        fi
+                    fi
+                fi
+            {% else %}
+                if [[ "$installed_version" == "null" || -z "$installed_version" ]]; then
+                    echo "$package_name is not installed... Adding to next update."
+                    jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
+                    PENDING_UPDATE=true
+                else
+                    results=$(compare_versions "$latest_version" "$installed_version")
+                    if [ $results == "greater" ]; then
+                        {#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations  #}
+                        {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                        if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then
+                        {%- endif %}
+                            echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
+                            jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
+                            PENDING_UPDATE=true
+                        {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                        else
+                            echo "skipping available upgrade for in use integration - $package_name."
+                        fi
+                        {%- endif %}
+                    fi
+                fi
+            {% endif %}
+            else
+                echo "Skipping $package_name..."
+            fi
+        done <<< "$(jq -c '.packages[]' "$INSTALLED_PACKAGE_LIST")"
 
         if [ "$PENDING_UPDATE" = true ]; then
             # Run chunked install of packages

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade

@@ -8,35 +8,18 @@
 
 . /usr/sbin/so-elastic-fleet-common
 
-PKG_LOAD_FAILURES=0
-PKG_LOAD_FAILURES_NAMES=()
-
 {%- for PACKAGE in SUPPORTED_PACKAGES %}
-if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && LATEST_VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
-
-    if [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then
-        echo "{{ PACKAGE }} integration version $INSTALLED_VERSION is already at the reported latest version $LATEST_VERSION, skipping upgrade."
-    else
-        echo "Upgrading {{ PACKAGE }} package from $INSTALLED_VERSION to version $LATEST_VERSION..."
-        if ! elastic_fleet_package_install "{{ PACKAGE }}" "$LATEST_VERSION"; then
-            PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
-            PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
-        fi
+echo "Upgrading {{ PACKAGE }} package..."
+if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
+    if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
+        # exit 1 on failure to upgrade a default package, allow salt to handle retries
+        echo -e "\nERROR: Failed to upgrade $PACKAGE to version: $VERSION"
+        exit 1
     fi
 else
-    echo "ERROR: Failed to get version information for integration {{ PACKAGE }}"
-    PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
-    PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
+    echo -e "\nERROR: Failed to get version information for integration $PACKAGE"
 fi
+echo
 {%- endfor %}
-
-if [ $PKG_LOAD_FAILURES -gt 0 ]; then
-    echo "ERROR: Failed to upgrade $PKG_LOAD_FAILURES package(s):"
-    for PKG in "${PKG_LOAD_FAILURES_NAMES[@]}"; do
-        echo " - $PKG"
-    done
-    # exit 1 on failure to upgrade a default package, allow salt to handle retries
-    exit 1
-else
-    echo "Successfully upgraded all packages."
-fi
+echo
+/usr/sbin/so-elasticsearch-templates-load

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -181,9 +181,6 @@ if ! elastic_fleet_policy_create "so-grid-nodes_heavy" "SO Grid Nodes - Heavy No
     exit 1
 fi
 
-# Check for package upgrades
-so-elastic-fleet-package-upgrade
-
 # Load Integrations for default policies
 so-elastic-fleet-integration-policy-load
 
@@ -244,37 +241,11 @@ printf '%s\n'\
     "" >> "$global_pillar_file"
 
 # Call Elastic-Fleet Salt State
-printf "\nApplying elasticfleet state\n"
-for state_attempt in {1..3}; do
-    if salt-call state.apply elasticfleet queue=True; then
-        break
-    elif [[ $state_attempt -lt 3 ]]; then
-        printf "\nElasticfleet state did not complete successfully... Attempt (%s/3). Retrying...\n" "$state_attempt"
-        sleep 10
-    else
-        printf "\nFailure(s) in elasticfleet state... Exiting...\n"
-        exit 1
-    fi
-done
+printf "\nApplying elasticfleet state"
+salt-call state.apply elasticfleet queue=True
 
-printf "\nRunning so-elastic-agent-gen-installers\n"
 # Generate installers & install Elastic Agent on the node
-for agent_gen_attempt in {1..3}; do
-    if so-elastic-agent-gen-installers; then
-        break
-    elif [[ $agent_gen_attempt -lt 3 ]]; then
-        printf "\nUnable to generate Elastic Agent installers... Attempt (%s/3). Retrying...\n" "$agent_gen_attempt"
-        sleep 10
-    else
-        printf "\nFailed to generate Elastic Agent installers after 3 attempts. Exiting...\n"
-        exit 1
-    fi
-done
-
-printf "\nApplying elasticfleet.install_agent_grid state\n"
-if ! salt-call state.apply elasticfleet.install_agent_grid queue=True; then
-    printf "\nFailure(s) in elasticfleet.install_agent_grid state... Exiting...\n"
-    exit 1
-fi
-
-printf "\nElastic Fleet setup completed successfully\n"
+so-elastic-agent-gen-installers
+printf "\nApplying elasticfleet.install_agent_grid state"
+salt-call state.apply elasticfleet.install_agent_grid queue=True
+exit 0

salt/elasticsearch/cluster.sls

@@ -9,12 +9,9 @@
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %}
 {%   if GLOBALS.role != 'so-heavynode' %}
-{%     from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS, ADDON_INDICES %}
+{%     from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
 {%   endif %}
 
-include:
-  - elasticsearch.enabled
-
 escomponenttemplates:
   file.recurse:
     - name: /opt/so/conf/elasticsearch/templates/component
@@ -38,20 +35,6 @@ so_index_template_dir:
       {%- endfor %}
     {%- endif %}
 
-{%  if GLOBALS.role != "so-heavynode" %}
-# Clean up legacy and non-SO managed templates from the elasticsearch/templates/addon-index/ directory
-addon_index_template_dir:
-  file.directory:
-    - name: /opt/so/conf/elasticsearch/templates/addon-index
-    - clean: True
-    {%- if ADDON_INDICES %}
-    - require:
-      {%- for index in ADDON_INDICES %}
-      - file: addon_index_template_{{index}}
-      {%- endfor %}
-    {%- endif %}
-{%  endif %}
-
 # Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml)
 #   These index templates are for the core SO datasets and are always required
 {%  for index, settings in ES_INDEX_SETTINGS.items() %}
@@ -133,18 +116,6 @@ so-elasticsearch-templates:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
 
-so-elasticsearch-dlm-apply:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-dlm-apply
-    - cwd: /opt/so
-    - require:
-      - docker_container: so-elasticsearch
-      - file: elasticsearch_sbin_jinja
-      - cmd: so-elasticsearch-templates
-    - retry:
-        attempts: 3
-        interval: 10
-
 so-elasticsearch-pipelines:
   cmd.run:
     - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }}
@@ -165,8 +136,7 @@ so-elasticsearch-roles-load:
 {%    set ap = "absent" %}
 {%  endif %}
 {%  if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %}
-{#    Remove so-elasticsearch-indices-delete script when using DLM #}
-{%    if ELASTICSEARCHMERGED.index_clean and ELASTICSEARCHMERGED.data_retention_method == "ILM" %}
+{%    if ELASTICSEARCHMERGED.index_clean %}
 {%      set ap = "present" %}
 {%    else %}
 {%      set ap = "absent" %}

salt/elasticsearch/enabled.sls

@@ -24,7 +24,6 @@ include:
 so-elasticsearch:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:{{ ELASTICSEARCHMERGED.version }}
-    - restart_policy: unless-stopped
     - hostname: elasticsearch
     - name: so-elasticsearch
     - user: elasticsearch

salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2

@@ -118,70 +118,70 @@
     {
       "pipeline": {
         "tag": "pipeline_e16851a7",
-        "name": "logs-pfsense.log-1.25.4-firewall",
+        "name": "logs-pfsense.log-1.25.2-firewall",
         "if": "ctx.event.provider == 'filterlog'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_828590b5",
-        "name": "logs-pfsense.log-1.25.4-openvpn",
+        "name": "logs-pfsense.log-1.25.2-openvpn",
         "if": "ctx.event.provider == 'openvpn'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_9d37039c",
-        "name": "logs-pfsense.log-1.25.4-ipsec",
+        "name": "logs-pfsense.log-1.25.2-ipsec",
         "if": "ctx.event.provider == 'charon'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_ad56bbca",
-        "name": "logs-pfsense.log-1.25.4-dhcp",
+        "name": "logs-pfsense.log-1.25.2-dhcp",
         "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\", \"dnsmasq-dhcp\"].contains(ctx.event.provider)"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_dd85553d",
-        "name": "logs-pfsense.log-1.25.4-unbound",
+        "name": "logs-pfsense.log-1.25.2-unbound",
         "if": "ctx.event.provider == 'unbound'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_720ed255",
-        "name": "logs-pfsense.log-1.25.4-haproxy",
+        "name": "logs-pfsense.log-1.25.2-haproxy",
         "if": "ctx.event.provider == 'haproxy'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_456beba5",
-        "name": "logs-pfsense.log-1.25.4-php-fpm",
+        "name": "logs-pfsense.log-1.25.2-php-fpm",
         "if": "ctx.event.provider == 'php-fpm'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_a0d89375",
-        "name": "logs-pfsense.log-1.25.4-squid",
+        "name": "logs-pfsense.log-1.25.2-squid",
         "if": "ctx.event.provider == 'squid'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_c2f1ed55",
-        "name": "logs-pfsense.log-1.25.4-snort",
+        "name": "logs-pfsense.log-1.25.2-snort",
         "if": "ctx.event.provider == 'snort'"
       }
     },
     {
       "pipeline": {
         "tag":"pipeline_33db1c9e",
-        "name": "logs-pfsense.log-1.25.4-suricata",
+        "name": "logs-pfsense.log-1.25.2-suricata",
         "if": "ctx.event.provider == 'suricata'"
       }
     },

salt/elasticsearch/files/ingest/zeek.ssl

@@ -5,7 +5,6 @@
     { "remove":   { "field": ["host"],     "ignore_failure": true } },
     { "json":             { "field": "message",                 "target_field": "message2",             "ignore_failure": true  } },
     { "rename":         { "field": "message2.version",          "target_field": "ssl.version",          "ignore_missing": true  } },
-    { "set":      { "description": "Set transport for the community_id processor", "if": "ctx.ssl?.version == null || !ctx.ssl.version.startsWith('DTLS')", "field": "network.transport", "value": "tcp", "ignore_failure": true } },
     { "rename":         { "field": "message2.cipher",           "target_field": "ssl.cipher",           "ignore_missing": true  } },
     { "rename":         { "field": "message2.curve",            "target_field": "ssl.curve",            "ignore_missing": true  } },
     { "rename":         { "field": "message2.server_name",      "target_field": "ssl.server_name",              "ignore_missing": true  } },

salt/elasticsearch/files/log4j2.properties

@@ -20,8 +20,7 @@ appender.rolling.strategy.type = DefaultRolloverStrategy
 appender.rolling.strategy.action.type = Delete
 appender.rolling.strategy.action.basepath = /var/log/elasticsearch
 appender.rolling.strategy.action.condition.type = IfFileName
-# age delete regular securityonion.log.gz and gc.log.NN files
-appender.rolling.strategy.action.condition.regex = (?:.*[.]log[.]gz|gc[.]log[.][0-9]+)
+appender.rolling.strategy.action.condition.glob = *.log.gz
 appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified
 appender.rolling.strategy.action.condition.nested_condition.age = 7D
 

salt/elasticsearch/soc_elasticsearch.yaml

@@ -4,13 +4,6 @@ elasticsearch:
     forcedType: bool
     advanced: True
     helpLink: elasticsearch
-  data_retention_method:
-    description: Method for data retention. Options are ILM or DLM. For single node deployments and most distributed grid users, DLM will be the recommended option for simplified management. Those with more complex use cases may prefer ILM. The latter allows for more granular control, but requires more management overhead.
-    options:
-    - ILM
-    - DLM
-    forcedType: string
-    global: True
   version:
     description: "This specifies the version of the following containers: so-elastic-fleet-package-registry, so-elastic-agent, so-elastic-fleet, so-kibana, so-logstash and so-elasticsearch. Modifying this value in the Elasticsearch defaults.yaml will result in catastrophic grid failure."
     readonly: True
@@ -20,7 +13,7 @@ elasticsearch:
     description: Specify the memory heap size in (m)egabytes for Elasticsearch.
     helpLink: elasticsearch
   index_clean:
-    description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, data is retained by the configured lifecycle settings. This setting only applies to EVAL, STANDALONE, and HEAVY NODE installations. Other installations use lifecycle settings only.
+    description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, indices will only be deleted by the age defined in the ILM settings. This setting only applies to EVAL, STANDALONE, and HEAVY NODE installations. Other installations can only use ILM settings.
     forcedType: bool
     helpLink: elasticsearch
   vm:
@@ -28,14 +21,14 @@ elasticsearch:
       description: The maximum number of memory map areas a process may use. Elasticsearch uses a mmapfs directory by default to store its indices. The default operating system limits on mmap counts could be too low, which may result in out of memory exceptions.
       forcedType: int
       helpLink: elasticsearch
-  retention:
+  retention: 
     retention_pct:
       decription: Total percentage of space used by Elasticsearch for multi node clusters
       helpLink: elasticsearch
       global: True
   config:
     cluster:
-      name:
+      name: 
         description: The name of the Security Onion Elasticsearch cluster, for identification purposes.
         readonly: True
         global: True
@@ -55,52 +48,15 @@ elasticsearch:
               forcedType: bool
               helpLink: elasticsearch
             watermark:
-              low:
+              low: 
                 description: The lower percentage of used disk space representing a healthy node.
                 helpLink: elasticsearch
-              high:
+              high: 
                 description: The higher percentage of used disk space representing an unhealthy node.
                 helpLink: elasticsearch
-              flood_stage:
+              flood_stage: 
                 description: The max percentage of used disk space that will cause the node to take protective actions, such as blocking incoming events.
                 helpLink: elasticsearch
-      lifecycle:
-        default:
-          rollover:
-            description: This property accepts a key value pair formatted string and configures the conditions that would trigger a data stream to rollover when it has lifecycle configured.
-            forcedType: string
-            regex: ^max_age=(auto|[1-9][0-9]*[hd]),max_primary_shard_size=[1-9][0-9]*gb,min_docs=(0|[1-9][0-9]*),max_primary_shard_docs=[1-9][0-9]*$
-            regexFailureMessage: Must be in the format of "max_age=auto|<number><h|d>,max_primary_shard_size=<number>gb,min_docs=<number>,max_primary_shard_docs=<number>".
-            advanced: True
-            global: True
-    data_streams:
-      lifecycle:
-        poll_interval:
-          description: How often Elasticsearch checks what the next action is for all data streams with a built-in lifecycle.
-          forcedType: string
-          regex: "^[1-9][0-9]*[mhd]$"
-          regexFailureMessage: Must be a number followed by m, h, or d.
-          advanced: True
-          global: true
-          helpLink: elasticsearch
-        target:
-          merge:
-            policy:
-              merge_factor:
-                description: Data stream lifecycle implements tail merging by updating the Lucene merge policy factor for the target backing index. The merge factor is both the number of segments that should be merged together, and the maximum number of segments that we expect to find.
-                forcedType: int
-                regex: "^[1-9][0-9]*$"
-                advanced: True
-                global: true
-                helpLink: elasticsearch
-              floor_segment:
-                description: Data stream lifecycle implements tail merging by updating the Lucene merge policy floor segment for the target backing index. This floor segment size is a way to prevent indices from having a long tail of very small segments.
-                forcedType: string
-                regex: "^[1-9][0-9]*[MG]B$"
-                regexFailureMessage: Must be a number followed by MB or GB, such as 100MB.
-                advanced: True
-                global: true
-                helpLink: elasticsearch
     action:
       destructive_requires_name:
         description: Requires explicit index names when deleting indices. Prevents accidental deletion of indices via wildcard patterns.
@@ -183,23 +139,6 @@ elasticsearch:
     custom010: *pipelines
   index_settings:
     global_overrides:
-      data_stream_lifecycle:
-        data_retention:
-          description: |
-            The retention period for all data streams. Retention does not define the period that the data will be removed, but the minimum time period they will be kept.
-
-            Use a number followed by a time unit, such as 7d. Leave blank for indefinite retention where supported.
-
-            Configured retention period also affects the frequency of rolling over data streams.
-              - If retention is less than or equal to 1 day, max_age will be 1 hour
-              - If retention is less than or equal to 14 days, max_age will be 1 day
-              - If retention is less than or equal to 90 days, max_age will be 7 days
-              - If retention is greater than 90 days, max_age will be 30 days
-          forcedType: string
-          allowedNodeTypes:
-            - heavynode
-          regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$
-          regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d.
       index_template:
         template:
           settings:
@@ -209,11 +148,11 @@ elasticsearch:
                 forcedType: int
                 global: True
                 helpLink: elasticsearch
-              refresh_interval:
+              refresh_interval: 
                 description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation.
                 global: True
                 helpLink: elasticsearch
-              number_of_shards:
+              number_of_shards: 
                 description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs.
                 global: True
                 helpLink: elasticsearch
@@ -306,7 +245,7 @@ elasticsearch:
                   global: True
                   advanced: True
           warm:
-            min_age:
+            min_age: 
               description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
               regex: ^[0-9]{1,5}d$
               forcedType: string
@@ -372,30 +311,13 @@ elasticsearch:
               forcedType: string
               global: True
               helpLink: elasticsearch
-    so-logs: &dataStreamSettings
+    so-logs: &indexSettings
       index_sorting:
         description: Sorts the index by event time, at the cost of additional processing resource consumption.
         forcedType: bool
         global: True
         advanced: True
         helpLink: elasticsearch
-      data_stream_lifecycle:
-        data_retention:
-          description: |
-            The retention period for this data stream. Retention does not define the period that the data will be removed, but the minimum time period it will be kept.
-
-            Use a number followed by a time unit, such as 7d. Leave blank for indefinite retention where supported.
-
-            Configured retention period also affects the frequency of rolling over this data stream.
-              - If retention is less than or equal to 1 day, max_age will be 1 hour
-              - If retention is less than or equal to 14 days, max_age will be 1 day
-              - If retention is less than or equal to 90 days, max_age will be 7 days
-              - If retention is greater than 90 days, max_age will be 30 days
-          forcedType: string
-          allowedNodeTypes:
-            - heavynode
-          regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$
-          regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d.
       index_template:
         index_patterns:
           description: Patterns for matching multiple indices or tables.
@@ -407,20 +329,12 @@ elasticsearch:
         template:
           settings:
             index:
-              number_of_replicas:
+              number_of_replicas: 
                 description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs.
                 forcedType: int
                 global: True
                 advanced: True
                 helpLink: elasticsearch
-              auto_expand_replicas:
-                description: Automatically expand the number of replicas based on the number of data nodes in the cluster. This can help ensure high availability as the cluster scales up or down.
-                forcedType: string
-                regex: "^(0-[1-9]|1-[2-9]|2-[3-9]|3-[4-9]|4-[5-9]|5-[6-9]|6-[7-9]|7-[89]|8-9|[0-9]-all|false)$"
-                regexFailureMessage: Must be in the format of "x-y" where x is minimum number of replicas and y is maximum number of replicas, or "0-all" to specify a minimum of 0 and no maximum, or "false" to disable automatic replica expansion.
-                global: True
-                advanced: True
-                helpLink: elasticsearch
               mapping:
                 total_fields:
                   limit:
@@ -428,12 +342,12 @@ elasticsearch:
                     global: True
                     advanced: True
                     helpLink: elasticsearch
-              refresh_interval:
+              refresh_interval: 
                 description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation.
                 global: True
                 advanced: True
                 helpLink: elasticsearch
-              number_of_shards:
+              number_of_shards: 
                 description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs.
                 global: True
                 advanced: True
@@ -682,357 +596,65 @@ elasticsearch:
             global: True
             advanced: True
             helpLink: elasticsearch
-    so-assistant-chat: *dataStreamSettings
-    so-assistant-session: *dataStreamSettings
-    so-elastic-agent-monitor: *dataStreamSettings
-    so-logs-soc: *dataStreamSettings
-    so-logs-system_x_auth: *dataStreamSettings
-    so-logs-system_x_syslog: *dataStreamSettings
-    so-logs-system_x_system: *dataStreamSettings
-    so-logs-system_x_application: *dataStreamSettings
-    so-logs-system_x_security: *dataStreamSettings
-    so-logs-windows_x_forwarded: *dataStreamSettings
-    so-logs-windows_x_powershell: *dataStreamSettings
-    so-logs-windows_x_powershell_operational: *dataStreamSettings
-    so-logs-windows_x_sysmon_operational: *dataStreamSettings
-    so-logs-winlog_x_winlog: *dataStreamSettings
-    so-logs-detections_x_alerts: *dataStreamSettings
-    so-logs-http_endpoint_x_generic: *dataStreamSettings
-    so-logs-httpjson_x_generic: *dataStreamSettings
-    so-logs-osquery-manager-actions: *dataStreamSettings
-    so-logs-osquery-manager-action_x_responses: *dataStreamSettings
-    so-logs-osquery-manager_x_action_x_responses: *dataStreamSettings
-    so-logs-osquery-manager_x_result: *dataStreamSettings
-    so-logs-elastic_agent_x_apm_server: *dataStreamSettings
-    so-logs-elastic_agent_x_auditbeat: *dataStreamSettings
-    so-logs-elastic_agent_x_cloudbeat: *dataStreamSettings
-    so-logs-elastic_agent_x_endpoint_security: *dataStreamSettings
-    so-logs-endpoint_x_actions: *dataStreamSettings
-    so-logs-endpoint_x_action_x_responses: *dataStreamSettings
-    so-logs-endpoint_x_alerts: *dataStreamSettings
-    so-logs-endpoint_x_diagnostic_x_collection: *dataStreamSettings
-    so-logs-endpoint_x_events_x_api: *dataStreamSettings
-    so-logs-endpoint_x_events_x_file: *dataStreamSettings
-    so-logs-endpoint_x_events_x_library: *dataStreamSettings
-    so-logs-endpoint_x_events_x_network: *dataStreamSettings
-    so-logs-endpoint_x_events_x_process: *dataStreamSettings
-    so-logs-endpoint_x_events_x_registry: *dataStreamSettings
-    so-logs-endpoint_x_events_x_security: *dataStreamSettings
-    so-logs-endpoint_x_heartbeat: *dataStreamSettings
-    so-logs-elastic_agent_x_filebeat: *dataStreamSettings
-    so-logs-elastic_agent_x_fleet_server: *dataStreamSettings
-    so-logs-elastic_agent_x_heartbeat: *dataStreamSettings
-    so-logs-elastic_agent: *dataStreamSettings
-    so-logs-elastic_agent_x_metricbeat: *dataStreamSettings
-    so-logs-elastic_agent_x_osquerybeat: *dataStreamSettings
-    so-logs-elastic_agent_x_packetbeat: *dataStreamSettings
-    so-logs-elasticsearch_x_server: *dataStreamSettings
-    so-metrics-endpoint_x_metadata: *dataStreamSettings
-    so-metrics-endpoint_x_metrics: *dataStreamSettings
-    so-metrics-endpoint_x_policy: *dataStreamSettings
-    so-metrics-nginx_x_stubstatus: *dataStreamSettings
-    so-metrics-vsphere_x_datastore: *dataStreamSettings
-    so-metrics-vsphere_x_host: *dataStreamSettings
-    so-metrics-vsphere_x_virtualmachine: *dataStreamSettings
-    so-metrics-system_x_core: *dataStreamSettings
-    so-metrics-system_x_cpu: *dataStreamSettings
-    so-metrics-system_x_diskio: *dataStreamSettings
-    so-metrics-system_x_filesystem: *dataStreamSettings
-    so-metrics-system_x_fsstat: *dataStreamSettings
-    so-metrics-system_x_load: *dataStreamSettings
-    so-metrics-system_x_memory: *dataStreamSettings
-    so-metrics-system_x_network: *dataStreamSettings
-    so-metrics-system_x_ntp: *dataStreamSettings
-    so-metrics-system_x_process: *dataStreamSettings
-    so-metrics-system_x_process_x_summary: *dataStreamSettings
-    so-metrics-system_x_socket_summary: *dataStreamSettings
-    so-metrics-system_x_uptime: *dataStreamSettings
-    so-common: *dataStreamSettings
-    so-endgame: *dataStreamSettings
-    so-idh: *dataStreamSettings
-    so-suricata: *dataStreamSettings
-    so-suricata_x_alerts: *dataStreamSettings
-    so-import: *dataStreamSettings
-    so-kratos: *dataStreamSettings
-    so-hydra: *dataStreamSettings
-    so-kismet: *dataStreamSettings
-    so-logstash: *dataStreamSettings
-    so-redis: *dataStreamSettings
-    so-strelka: *dataStreamSettings
-    so-syslog: *dataStreamSettings
-    so-zeek: *dataStreamSettings
-    # Managed SOC integration annotations are inserted below this line. Referencing '*dataStreamSettings'
-    so-case: &indexSettings
-      index_sorting:
-        description: Sorts the index by event time, at the cost of additional processing resource consumption.
-        forcedType: bool
-        global: True
-        advanced: True
-        helpLink: elasticsearch
-      index_template:
-        index_patterns:
-          description: Patterns for matching multiple indices or tables.
-          forcedType: "[]string"
-          multiline: True
-          global: True
-          advanced: True
-          helpLink: elasticsearch
-        template:
-          settings:
-            index:
-              number_of_replicas:
-                description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs.
-                forcedType: int
-                global: True
-                advanced: True
-                helpLink: elasticsearch
-              auto_expand_replicas:
-                description: Automatically expand the number of replicas based on the number of data nodes in the cluster. This can help ensure high availability as the cluster scales up or down.
-                forcedType: string
-                regex: "^(0-[1-9]|1-[2-9]|2-[3-9]|3-[4-9]|4-[5-9]|5-[6-9]|6-[7-9]|7-[89]|8-9|[0-9]-all|false)$"
-                regexFailureMessage: Must be in the format of "x-y" where x is minimum number of replicas and y is maximum number of replicas, or "0-all" to specify a minimum of 0 and no maximum, or "false" to disable automatic replica expansion.
-                global: True
-                advanced: True
-                helpLink: elasticsearch
-              mapping:
-                total_fields:
-                  limit:
-                    description: Max number of fields that can exist on a single index. Larger values will consume more resources.
-                    global: True
-                    advanced: True
-                    helpLink: elasticsearch
-              refresh_interval:
-                description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation.
-                global: True
-                advanced: True
-                helpLink: elasticsearch
-              number_of_shards:
-                description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs.
-                global: True
-                advanced: True
-                helpLink: elasticsearch
-              sort:
-                field:
-                  description: The field to sort by. Must set index_sorting to True.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-                order:
-                  description: The order to sort by. Must set index_sorting to True.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-          mappings:
-            _meta:
-              package:
-                name:
-                  description: Meta settings for the mapping.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-              managed_by:
-                  description: Meta settings for the mapping.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-              managed:
-                  description: Meta settings for the mapping.
-                  forcedType: bool
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-        composed_of:
-          description: The index template is composed of these component templates.
-          forcedType: "[]string"
-          global: True
-          advanced: True
-          helpLink: elasticsearch
-        priority:
-          description: The priority of the index template.
-          forcedType: int
-          global: True
-          advanced: True
-          helpLink: elasticsearch
-      policy:
-        phases:
-          hot:
-            min_age:
-              description: Minimum age of index. This determines when the index should be moved to the hot tier.
-              global: True
-              advanced: True
-              helpLink: elasticsearch
-            actions:
-              set_priority:
-                priority:
-                  description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
-                  forcedType: int
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-              rollover:
-                max_age:
-                  description: Maximum age of index.  Once an index reaches this limit, it will be rolled over into a new index.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-                max_primary_shard_size:
-                  description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-              shrink:
-                method:
-                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
-                  options:
-                  - COUNT
-                  - SIZE
-                  global: True
-                  advanced: True
-                  forcedType: string
-                number_of_shards:
-                  title: shard count
-                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
-                  global: True
-                  forcedType: int
-                  advanced: True
-                max_primary_shard_size:
-                  title: max shard size
-                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
-                  regex: ^[0-9]+(?:gb|tb|pb)$
-                  global: True
-                  forcedType: string
-                  advanced: True
-                allow_write_after_shrink:
-                  description: Allow writes after shrink.
-                  global: True
-                  forcedType: bool
-                  default: False
-                  advanced: True
-              forcemerge:
-                max_num_segments:
-                  description: Reduce the number of segments in each index shard and clean up deleted documents.
-                  global: True
-                  forcedType: int
-                  advanced: True
-                index_codec:
-                  title: compression
-                  description: Use higher compression for stored fields at the cost of slower performance.
-                  forcedType: bool
-                  global: True
-                  default: False
-                  advanced: True
-          warm:
-            min_age:
-              description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
-              regex: ^[0-9]{1,5}d$
-              forcedType: string
-              global: True
-              advanced: True
-              helpLink: elasticsearch
-            actions:
-              set_priority:
-                priority:
-                  description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
-                  forcedType: int
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-              shrink:
-                method:
-                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
-                  options:
-                  - COUNT
-                  - SIZE
-                  global: True
-                  advanced: True
-                number_of_shards:
-                  title: shard count
-                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
-                  global: True
-                  forcedType: int
-                  advanced: True
-                max_primary_shard_size:
-                  title: max shard size
-                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
-                  regex: ^[0-9]+(?:gb|tb|pb)$
-                  global: True
-                  forcedType: string
-                  advanced: True
-                allow_write_after_shrink:
-                  description: Allow writes after shrink.
-                  global: True
-                  forcedType: bool
-                  default: False
-                  advanced: True
-              forcemerge:
-                max_num_segments:
-                  description: Reduce the number of segments in each index shard and clean up deleted documents.
-                  global: True
-                  forcedType: int
-                  advanced: True
-                index_codec:
-                  title: compression
-                  description: Use higher compression for stored fields at the cost of slower performance.
-                  forcedType: bool
-                  global: True
-                  default: False
-                  advanced: True
-              allocate:
-                number_of_replicas:
-                  description: Set the number of replicas. Remains the same as the previous phase by default.
-                  forcedType: int
-                  global: True
-                  advanced: True
-          cold:
-            min_age:
-              description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
-              regex: ^[0-9]{1,5}d$
-              forcedType: string
-              global: True
-              advanced: True
-              helpLink: elasticsearch
-            actions:
-              set_priority:
-                priority:
-                  description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
-                  forcedType: int
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-              allocate:
-                number_of_replicas:
-                  description: Set the number of replicas. Remains the same as the previous phase by default.
-                  forcedType: int
-                  global: True
-                  advanced: True
-          delete:
-            min_age:
-              description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.
-              regex: ^[0-9]{1,5}d$
-              forcedType: string
-              global: True
-              advanced: True
-              helpLink: elasticsearch
-        _meta:
-          package:
-            name:
-              description: Meta settings for the mapping.
-              global: True
-              advanced: True
-              helpLink: elasticsearch
-          managed_by:
-            description: Meta settings for the mapping.
-            global: True
-            advanced: True
-            helpLink: elasticsearch
-          managed:
-            description: Meta settings for the mapping.
-            forcedType: bool
-            global: True
-            advanced: True
-            helpLink: elasticsearch
-    sos-backup: *indexSettings
-    so-detection: *indexSettings
+    so-logs-system_x_auth: *indexSettings
+    so-logs-system_x_syslog: *indexSettings
+    so-logs-system_x_system: *indexSettings
+    so-logs-system_x_application: *indexSettings
+    so-logs-system_x_security: *indexSettings
+    so-logs-windows_x_forwarded: *indexSettings
+    so-logs-windows_x_powershell: *indexSettings
+    so-logs-windows_x_powershell_operational: *indexSettings
+    so-logs-windows_x_sysmon_operational: *indexSettings
+    so-logs-winlog_x_winlog: *indexSettings
+    so-logs-detections_x_alerts: *indexSettings
+    so-logs-http_endpoint_x_generic: *indexSettings
+    so-logs-httpjson_x_generic: *indexSettings
+    so-logs-osquery-manager-actions: *indexSettings
+    so-logs-osquery-manager-action_x_responses: *indexSettings
+    so-logs-osquery-manager_x_action_x_responses: *indexSettings
+    so-logs-osquery-manager_x_result: *indexSettings
+    so-logs-elastic_agent_x_apm_server: *indexSettings
+    so-logs-elastic_agent_x_auditbeat: *indexSettings
+    so-logs-elastic_agent_x_cloudbeat: *indexSettings
+    so-logs-elastic_agent_x_endpoint_security: *indexSettings
+    so-logs-endpoint_x_alerts: *indexSettings
+    so-logs-endpoint_x_events_x_api: *indexSettings
+    so-logs-endpoint_x_events_x_file: *indexSettings
+    so-logs-endpoint_x_events_x_library: *indexSettings
+    so-logs-endpoint_x_events_x_network: *indexSettings
+    so-logs-endpoint_x_events_x_process: *indexSettings
+    so-logs-endpoint_x_events_x_registry: *indexSettings
+    so-logs-endpoint_x_events_x_security: *indexSettings
+    so-logs-elastic_agent_x_filebeat: *indexSettings
+    so-logs-elastic_agent_x_fleet_server: *indexSettings
+    so-logs-elastic_agent_x_heartbeat: *indexSettings
+    so-logs-elastic_agent: *indexSettings
+    so-logs-elastic_agent_x_metricbeat: *indexSettings
+    so-logs-elastic_agent_x_osquerybeat: *indexSettings
+    so-logs-elastic_agent_x_packetbeat: *indexSettings
+    so-logs-elasticsearch_x_server: *indexSettings
+    so-metrics-endpoint_x_metadata: *indexSettings
+    so-metrics-endpoint_x_metrics: *indexSettings
+    so-metrics-endpoint_x_policy: *indexSettings
+    so-metrics-nginx_x_stubstatus: *indexSettings
+    so-metrics-vsphere_x_datastore: *indexSettings
+    so-metrics-vsphere_x_host: *indexSettings
+    so-metrics-vsphere_x_virtualmachine: *indexSettings
+    so-case: *indexSettings
+    so-common: *indexSettings
+    so-endgame: *indexSettings
+    so-idh: *indexSettings
+    so-suricata: *indexSettings
+    so-suricata_x_alerts: *indexSettings
+    so-import: *indexSettings
+    so-kratos: *indexSettings
+    so-hydra: *indexSettings
+    so-kismet: *indexSettings
+    so-logstash: *indexSettings
+    so-redis: *indexSettings
+    so-strelka: *indexSettings
+    so-syslog: *indexSettings
+    so-zeek: *indexSettings
     so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings
       index_sorting:
         description: Sorts the index by event time, at the cost of additional processing resource consumption.

salt/elasticsearch/template.map.jinja

@@ -4,11 +4,7 @@
    Elastic License 2.0. #}
 
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
-{# ELASTICSEARCHMERGED only used here to collect data_retention_method. This file intentionally works with ELASTICSEARCHDEFAULTS #}
-{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
-
 {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %}
-{% set DATA_RETENTION_METHOD = ELASTICSEARCHMERGED.data_retention_method %}
 
 {% set PILLAR_GLOBAL_OVERRIDES = {} %}
 {% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings', {}) %}
@@ -65,25 +61,15 @@
 {% if ALL_ADDON_SETTINGS_ORIG.keys() | length > 0 %}
 {%   for index in ALL_ADDON_SETTINGS_ORIG.keys() %}
 {%     do ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ALL_ADDON_SETTINGS_ORIG[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}
-{#     Explicitly excluding addon indices from ES_INDEX_SETTINGS_ORIG
-         When manager.soc_managed_annotations runs, new entries are added to the salt/elasticsearch/defaults.yaml file to support 'revert to default' functionality.
-         Subsequent map renders will then incorrectly include 'integration X' in 'ES_INDEX_SETTINGS_ORIG' due to being in the defaults.yaml file. #}
-{%     if index in ES_INDEX_SETTINGS_ORIG.keys() %}
-{%       do ES_INDEX_SETTINGS_ORIG.pop(index) %}
-{%     endif %}
 {%   endfor %}
 {% endif %}
 
 {% set ES_INDEX_SETTINGS = {} %}
-{% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS, EXCLUDE_INDICES=[]) %}
+{% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS) %}
 
 {% do GLOBAL_OVERRIDES.update(salt['defaults.merge'](GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
 {% for index, settings in GLOBAL_OVERRIDES.items() %}
 
-{%   if index in EXCLUDE_INDICES %}
-{%     continue %}
-{%   endif %}
-
 {#   prevent this action from being performed on custom defined indices. #}
 {#   the custom defined index is not present in either of the dictionaries and fails to reder. #}
 {%   if index in DEFINED_SETTINGS and index in GLOBAL_OVERRIDES %}
@@ -109,23 +95,6 @@
 {%     if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %}
 {%       do settings.index_template.template.settings.index.pop('sort') %}
 {%     endif %}
-{%     if DATA_RETENTION_METHOD == 'DLM' and settings.index_template.data_stream is defined %}
-{#       Addon defaults are generated without data_stream_lifecycle, so fall back to global defaults. #}
-{%       if settings.data_stream_lifecycle is defined %}
-{%         set DATA_STREAM_LIFECYCLE = settings.data_stream_lifecycle %}
-{%       else %}
-{%         set DATA_STREAM_LIFECYCLE = DEFAULT_GLOBAL_OVERRIDES.data_stream_lifecycle %}
-{%       endif %}
-{%       if DATA_STREAM_LIFECYCLE.data_retention is defined and DATA_STREAM_LIFECYCLE.data_retention %}
-{%         do settings.index_template.template.update({'lifecycle': {'data_retention': DATA_STREAM_LIFECYCLE.data_retention}}) %}
-{%       else %}
-{%         do settings.index_template.template.update({'lifecycle': {}}) %}
-{%       endif %}
-{%       if settings.index_template.template.settings.index.lifecycle is not defined %}
-{%         do settings.index_template.template.settings.index.update({'lifecycle': {}}) %}
-{%       endif %}
-{%       do settings.index_template.template.settings.index.lifecycle.update({'prefer_ilm': false}) %}
-{%     endif %}
 {%   endif %}
 
 {# advanced ilm actions #}
@@ -181,19 +150,10 @@
 {% endfor %}
 {% endmacro %}
 
-{# Exclude addon integrations from final ES_INDEX_SETTINGS #}
-{{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS, ALL_ADDON_SETTINGS_ORIG.keys() | list ) }}
-
-{# Exclude SO managed indices, otherwise ALL_ADDON_SETTINGS will include pillar values
-  of core integrations without merging defaults, resulting in an overlapping, but bad index template being generated. #}
-{{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS, ES_INDEX_SETTINGS_ORIG.keys() | list ) }}
+{{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS) }}
+{{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS) }}
 
 {% set SO_MANAGED_INDICES = [] %}
 {% for index, settings in ES_INDEX_SETTINGS.items() %}
 {%   do SO_MANAGED_INDICES.append(index) %}
-{% endfor %}
-
-{% set ADDON_INDICES = [] %}
-{% for index, settings in ALL_ADDON_SETTINGS.items() %}
-{%   do ADDON_INDICES.append(index) %}
-{% endfor %}
+{% endfor %}

salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load

@@ -11,8 +11,10 @@ ADDON_STATEFILE_SUCCESS=/opt/so/state/addon_estemplates.txt
 ELASTICSEARCH_TEMPLATES_DIR="/opt/so/conf/elasticsearch/templates"
 SO_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/index"
 ADDON_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/addon-index"
-FAILED_NAMES=()
-FAILED_COUNT=0
+SO_LOAD_FAILURES=0
+ADDON_LOAD_FAILURES=0
+SO_LOAD_FAILURES_NAMES=()
+ADDON_LOAD_FAILURES_NAMES=()
 IS_HEAVYNODE="false"
 FORCE="false"
 VERBOSE="false"
@@ -44,86 +46,20 @@ while [[ $# -gt 0 ]]; do
     shift
 done
 
-# Max number of concurrent template PUT jobs. Override via env if needed.
-MAX_TEMPLATE_JOBS=${MAX_TEMPLATE_JOBS:-10}
-
-# Block until fewer than MAX_TEMPLATE_JOBS background jobs are running.
-template_throttle() {
-    while (( $(jobs -rp | wc -l) >= MAX_TEMPLATE_JOBS )); do
-        wait -n
-    done
-}
-
-# Per-job failure markers and an output lock for serializing parallel job output.
-# Each failed load drops one file (named after the template) into FAIL_DIR; the
-# output of each job is flushed as a single block under flock so concurrent jobs
-# never interleave their (chatty) retry output.
-FAIL_DIR=$(mktemp -d)
-OUTPUT_LOCK="${FAIL_DIR}/.output.lock"
-: > "$OUTPUT_LOCK"
-trap 'rm -rf "$FAIL_DIR"' EXIT
-
-# Record a failure: $1 = the template name/path to report later. Slashes are
-# encoded so the path becomes a safe single filename.
-record_failure() {
-    local marker="${1//\//__}"
-    : > "${FAIL_DIR}/fail.${marker}"
-}
-
-# Populate FAILED_NAMES and FAILED_COUNT from the current phase's markers.
-# Must run in the current shell (not a command substitution) so the array sticks.
-collect_failures() {
-    FAILED_NAMES=()
-    FAILED_COUNT=0
-    local f name
-    shopt -s nullglob
-    for f in "${FAIL_DIR}"/fail.*; do
-        name="${f##*/fail.}"
-        name="${name//__//}"
-        FAILED_NAMES+=("$name")
-        FAILED_COUNT=$((FAILED_COUNT + 1))
-    done
-    shopt -u nullglob
-}
-
-# Clear markers and names between phases so SO and addon counts stay independent.
-reset_failures() {
-    shopt -s nullglob
-    rm -f "${FAIL_DIR}"/fail.*
-    shopt -u nullglob
-    FAILED_NAMES=()
-    FAILED_COUNT=0
-}
-
-# Print a block of text atomically (under the shared output lock) so the output
-# of concurrent background jobs is not interleaved.
-locked_echo() {
-    { flock 9; printf '%s\n' "$1"; } 9>>"$OUTPUT_LOCK"
-}
-
-# Loads one template file via PUT. Intended to be dispatched as a background job.
-#   $1 uri          - e.g. _component_template/foo or _index_template/foo
-#   $2 file         - path to the template JSON
-#   $3 report_name  - name/path to record if this load fails
 load_template() {
     local uri="$1"
     local file="$2"
-    local report_name="$3"
-    local out rc=0 block
 
-    # Capture everything (including retry's diagnostic chatter) into one block so
-    # concurrent jobs never interleave; the whole block is flushed under one flock.
-    block="Loading template file $file"$'\n'
-    if ! out=$(retry 3 3 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}" 2>&1); then
-        block+="$out"$'\n'
-        rc=1
+    echo "Loading template file $file"
+    if ! output=$(retry 3 3 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"); then
+        echo "$output"
+
+        return 1
+
     elif [[ "$VERBOSE" == "true" ]]; then
-        block+="$out"$'\n'
+        echo "$output"
     fi
 
-    { flock 9; printf '%s' "$block"; } 9>>"$OUTPUT_LOCK"
-
-    (( rc != 0 )) && record_failure "$report_name"
 }
 
 check_required_component_template_exists() {
@@ -174,9 +110,6 @@ load_component_templates() {
         return
     fi
 
-    # Dispatch loads as throttled background jobs. The barrier (wait) happens in
-    # the caller after all component groups have been dispatched, since index
-    # templates must not load until every component template is in place.
     for component in "$pattern"/*.json; do
         tmpl_name=$(basename "${component%.json}")
 
@@ -185,11 +118,21 @@ load_component_templates() {
             tmpl_name="${tmpl_name%-mappings}-mappings"
         fi
 
-        template_throttle
-        load_template "_component_template/${tmpl_name}" "$component" "$component" &
+        if ! load_template "_component_template/${tmpl_name}" "$component"; then
+            SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
+            SO_LOAD_FAILURES_NAMES+=("$component")
+        fi
     done
 }
 
+check_elasticsearch_responsive() {
+    # Cannot load templates if Elasticsearch is not responding.
+    #  NOTE: Slightly faster exit w/ failure than previous "retry 240 1" if there is a problem with Elasticsearch the
+    #    script should exit sooner rather than hang at the 'so-elasticsearch-templates' salt state.
+    retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" ||
+        fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot."
+}
+
 index_templates_exist() {
     local templates_dir="$1"
 
@@ -237,9 +180,6 @@ if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]] && index_templates_e
     load_component_templates "Elastic Agent" "elastic-agent"
     load_component_templates "Security Onion" "so"
 
-    # Barrier: every component template PUT must complete before we snapshot the
-    # component template list and start loading index templates that depend on them.
-    wait
     component_templates=$(so-elasticsearch-component-templates-list)
     echo -e "Loading Security Onion index templates...\n"
     for so_idx_tmpl in "${SO_TEMPLATES_DIR}"/*.json; do
@@ -249,7 +189,7 @@ if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]] && index_templates_e
             # TODO: Better way to load only heavynode specific templates
             if ! check_heavynode_compatiable_index_template "$tmpl_name"; then
                 if [[ "$VERBOSE" == "true" ]]; then
-                    locked_echo "Skipping over $so_idx_tmpl, template is not a heavynode specific index template."
+                    echo "Skipping over $so_idx_tmpl, template is not a heavynode specific index template."
                 fi
 
                 continue
@@ -257,34 +197,32 @@ if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]] && index_templates_e
         fi
 
         if check_required_component_template_exists "$so_idx_tmpl"; then
-            template_throttle
-            load_template "_index_template/$tmpl_name" "$so_idx_tmpl" "$so_idx_tmpl" &
+            if ! load_template "_index_template/$tmpl_name" "$so_idx_tmpl"; then
+                SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
+                SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl")
+            fi
         else
-            locked_echo "Skipping over $so_idx_tmpl due to missing required component template(s)."
-            record_failure "$so_idx_tmpl"
+            echo "Skipping over $so_idx_tmpl due to missing required component template(s)."
+            SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
+            SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl")
 
             continue
         fi
     done
 
-    # Barrier: all SO index template PUTs must finish before tallying failures.
-    wait
-
-    collect_failures
-    if [[ $FAILED_COUNT -eq 0 ]]; then
+    if [[ $SO_LOAD_FAILURES -eq 0 ]]; then
         echo "All Security Onion core templates loaded successfully."
 
         touch "$SO_STATEFILE_SUCCESS"
     else
-        echo "Encountered $FAILED_COUNT failure(s) loading templates:"
-        for failed_template in "${FAILED_NAMES[@]}"; do
+        echo "Encountered $SO_LOAD_FAILURES failure(s) loading templates:"
+        for failed_template in "${SO_LOAD_FAILURES_NAMES[@]}"; do
             echo "  - $failed_template"
         done
         if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then
             fail "Failed to load all Security Onion core templates successfully."
         fi
     fi
-    reset_failures
 elif ! index_templates_exist "$SO_TEMPLATES_DIR"; then
     echo "No Security Onion core index templates found in ${SO_TEMPLATES_DIR}, skipping."
 elif [[ -f "$SO_STATEFILE_SUCCESS" ]]; then
@@ -303,27 +241,26 @@ if should_load_addon_templates; then
         tmpl_name=$(basename "${addon_idx_tmpl%-template.json}")
 
         if check_required_component_template_exists "$addon_idx_tmpl"; then
-            template_throttle
-            load_template "_index_template/${tmpl_name}" "$addon_idx_tmpl" "$addon_idx_tmpl" &
+            if ! load_template "_index_template/${tmpl_name}" "$addon_idx_tmpl"; then
+                ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1))
+                ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl")
+            fi
         else
-            locked_echo "Skipping over $addon_idx_tmpl due to missing required component template(s)."
-            record_failure "$addon_idx_tmpl"
+            echo "Skipping over $addon_idx_tmpl due to missing required component template(s)."
+            ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1))
+            ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl")
 
             continue
         fi
     done
 
-    # Barrier: all addon index template PUTs must finish before tallying failures.
-    wait
-
-    collect_failures
-    if [[ $FAILED_COUNT -eq 0 ]]; then
+    if [[ $ADDON_LOAD_FAILURES -eq 0 ]]; then
         echo "All addon integration templates loaded successfully."
 
         touch "$ADDON_STATEFILE_SUCCESS"
     else
-        echo "Encountered $FAILED_COUNT failure(s) loading addon integration templates:"
-        for failed_template in "${FAILED_NAMES[@]}"; do
+        echo "Encountered $ADDON_LOAD_FAILURES failure(s) loading addon integration templates:"
+        for failed_template in "${ADDON_LOAD_FAILURES_NAMES[@]}"; do
             echo "  - $failed_template"
         done
         if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply

@@ -1,175 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-
-{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
-{%- set DATA_RETENTION_METHOD = ELASTICSEARCHMERGED.data_retention_method %}
-
-ELASTICSEARCH_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR:-/opt/so/conf/elasticsearch/templates}"
-TEMPLATE_DIRS=(
-    "${ELASTICSEARCH_TEMPLATES_DIR}/index"
-    "${ELASTICSEARCH_TEMPLATES_DIR}/addon-index"
-)
-DATA_RETENTION_METHOD=$(cat <<'EOF'
-{{ DATA_RETENTION_METHOD }}
-EOF
-)
-DLM_FAILURES=0
-DLM_FAILURE_NAMES=()
-
-if [[ "$DATA_RETENTION_METHOD" != "DLM" && "$DATA_RETENTION_METHOD" != "ILM" ]]; then
-    echo "Unsupported data retention method $DATA_RETENTION_METHOD. Expected DLM or ILM."
-    exit 1
-fi
-
-validate_template_file() {
-    local template_file="$1"
-
-    if ! jq -e 'type == "object" and (.data_stream == null or (.data_stream | type == "object")) and (.template.lifecycle == null or (.template.lifecycle | type == "object")) and (.template.lifecycle.data_retention == null or (.template.lifecycle.data_retention | type == "string"))' >/dev/null 2>&1 "$template_file"; then
-        echo "Invalid index template JSON: $template_file"
-        return 1
-    fi
-}
-
-is_data_stream_template() {
-    jq -e '.data_stream | type == "object"' >/dev/null 2>&1 "$1"
-}
-
-has_data_stream_lifecycle() {
-    jq -e '.template.lifecycle | type == "object"' >/dev/null 2>&1 "$1"
-}
-
-get_data_retention() {
-    jq -r '.template.lifecycle.data_retention // ""' "$1"
-}
-
-find_template_file() {
-    local template="$1"
-    local template_dir
-    local template_file
-
-    for template_dir in "${TEMPLATE_DIRS[@]}"; do
-        template_file="${template_dir}/${template}-template.json"
-
-        if [[ -f "$template_file" ]]; then
-            echo "$template_file"
-            return 0
-        fi
-    done
-
-    return 1
-}
-
-set_data_stream_lifecycle() {
-    local data_stream="$1"
-    local data_retention="$2"
-    local body
-    local output
-
-    if [[ -n "$data_retention" ]]; then
-        if jq -e --arg data_stream "$data_stream" --arg data_retention "$data_retention" '.data_streams[]? | select(.name == $data_stream and .lifecycle.enabled == true and .lifecycle.data_retention == $data_retention)' >/dev/null 2>&1 <<< "$data_streams"; then
-            echo "DLM lifecycle already set for $data_stream with data_retention $data_retention, skipping."
-            return 0
-        fi
-    elif jq -e --arg data_stream "$data_stream" '.data_streams[]? | select(.name == $data_stream and .lifecycle.enabled == true and (.lifecycle.data_retention == null))' >/dev/null 2>&1 <<< "$data_streams"; then
-        echo "DLM lifecycle already set for $data_stream with indefinite retention, skipping."
-        return 0
-    fi
-
-    if [[ -n "$data_retention" ]]; then
-        body=$(jq -cn --arg data_retention "$data_retention" '{data_retention: $data_retention}')
-    else
-        # Setting indefinite retention
-        body='{}'
-    fi
-
-    if ! output=$(so-elasticsearch-query "_data_stream/${data_stream}/_lifecycle" -XPUT -d "$body" --retry 3 --retry-delay 5 --fail); then
-        echo "Failed to set data stream lifecycle for $data_stream."
-        return 1
-    fi
-
-    if [[ -n "$data_retention" ]]; then
-        echo "Set DLM lifecycle for $data_stream with data_retention $data_retention."
-    else
-        echo "Set DLM lifecycle for $data_stream with indefinite retention."
-    fi
-}
-
-disable_data_stream_lifecycle() {
-    local data_stream="$1"
-    local body='{"enabled":false}'
-    local output
-
-    if ! jq -e --arg data_stream "$data_stream" '.data_streams[]? | select(.name == $data_stream and .lifecycle != null and .lifecycle.enabled != false)' >/dev/null 2>&1 <<< "$data_streams"; then
-        # No action needed
-        return 0
-    fi
-
-    if ! output=$(so-elasticsearch-query "_data_stream/${data_stream}/_lifecycle" -XPUT -d "$body" --retry 3 --retry-delay 5 --fail); then
-        echo "Failed to disable data stream lifecycle for $data_stream."
-        return 1
-    fi
-
-    echo "Disabled DLM lifecycle for $data_stream."
-}
-
-process_data_stream() {
-    local data_stream="$1"
-    local data_retention="$2"
-
-    if [[ "$DATA_RETENTION_METHOD" == "DLM" ]]; then
-        set_data_stream_lifecycle "$data_stream" "$data_retention"
-    else
-        disable_data_stream_lifecycle "$data_stream"
-    fi
-}
-
-check_elasticsearch_responsive
-
-if ! data_streams=$(so-elasticsearch-query "_data_stream?format=json" --retry 3 --retry-delay 5 --fail); then
-    echo "Failed to retrieve data streams."
-    exit 1
-fi
-
-while read -r data_stream_config; do
-    data_stream=$(jq -r '.name' <<< "$data_stream_config")
-    template=$(jq -r '.template' <<< "$data_stream_config")
-
-    if ! template_file=$(find_template_file "$template"); then
-        echo "Skipping $data_stream: index template file not found for $template."
-        continue
-    fi
-
-    validate_template_file "$template_file" || exit 1
-
-    if ! is_data_stream_template "$template_file"; then
-        echo "Skipping $data_stream: $template_file is not a data stream template."
-        continue
-    fi
-
-    if [[ "$DATA_RETENTION_METHOD" == "DLM" ]] && ! has_data_stream_lifecycle "$template_file"; then
-        echo "Skipping $data_stream: $template_file does not define data stream lifecycle."
-        continue
-    fi
-
-    data_retention=$(get_data_retention "$template_file")
-
-    if ! process_data_stream "$data_stream" "$data_retention"; then
-        DLM_FAILURES=$((DLM_FAILURES + 1))
-        DLM_FAILURE_NAMES+=("$data_stream")
-    fi
-done < <(jq -c '.data_streams[]' <<< "$data_streams")
-
-if [[ $DLM_FAILURES -eq 0 ]]; then
-    echo "Data stream lifecycle updates completed successfully."
-else
-    echo "Encountered $DLM_FAILURES failure(s) updating data stream lifecycle:"
-    for failed_data_stream in "${DLM_FAILURE_NAMES[@]}"; do
-        echo "  - $failed_data_stream"
-    done
-    exit 1
-fi

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load

@@ -6,48 +6,6 @@
 
 . /usr/sbin/so-common
 
-MAX_JOBS=${MAX_ILM_JOBS:-10}
-
-# Lock used to serialize block writes so concurrent jobs never interleave their output.
-ILM_OUTPUT_LOCK=$(mktemp)
-ILM_FAIL_FILE=$(mktemp)
-trap 'rm -f "$ILM_OUTPUT_LOCK" "$ILM_FAIL_FILE"' EXIT
-
-# Policies are loaded concurrently (up to MAX_JOBS at a time) for speed. Each policy's block is
-# printed the moment its curl returns, so output appears in COMPLETION ORDER, not the order
-# policies are defined in configuration.
-echo "Loading ILM policies concurrently; output below appears in completion order, not configuration order."
-echo
-
-put_policy() {
-  local desc="$1" policyname="$2" data="$3" result rc=0
-  if ! result=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L --fail \
-    -X PUT "https://localhost:9200/_ilm/policy/${policyname}" \
-    -H 'Content-Type: application/json' -d"${data}" 2>&1); then
-    rc=1
-  elif ! jq -e '.acknowledged == true' <<<"$result" >/dev/null 2>&1; then
-    rc=1
-  fi
-
-  # curl above ran in parallel; serialize just this block write so concurrent jobs never interleave.
-  {
-    flock 200
-    printf 'Setting up %s policy...\n%s\n\n' "${desc}" "${result}"
-    if (( rc != 0 )); then
-      printf '%s\n' "${policyname}" >>"$ILM_FAIL_FILE"
-    fi
-  } 200>>"${ILM_OUTPUT_LOCK}"
-
-  return "$rc"
-}
-
-# Block until fewer than MAX_JOBS background curls are running.
-throttle() {
-  while (( $(jobs -rp | wc -l) >= MAX_JOBS )); do
-    wait -n || true
-  done
-}
-
 {%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
 {%- if GLOBALS.role != "so-heavynode" %}
 {%-   from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
@@ -56,36 +14,35 @@ throttle() {
 {%- for index, settings in ES_INDEX_SETTINGS.items() %}
 {%-   if settings.policy is defined %}
 {%-     if index == 'so-logs-detections.alerts' %}
-  throttle
-  put_policy "so-logs-detections.alerts-so" "{{ index }}-so" '{ "policy": {{ settings.policy | tojson(true) }} }' &
+  echo
+  echo "Setting up so-logs-detections.alerts-so policy..."
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-so" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+  echo
 {%-     elif index == 'so-logs-soc' %}
-  throttle
-  put_policy "so-soc-logs" "so-soc-logs" '{ "policy": {{ settings.policy | tojson(true) }} }' &
-  throttle
-  put_policy "{{ index }}-logs" "{{ index }}-logs" '{ "policy": {{ settings.policy | tojson(true) }} }' &
+  echo
+  echo "Setting up so-soc-logs policy..."
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/so-soc-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+  echo
+  echo
+  echo "Setting up {{ index }}-logs policy..."
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+  echo
 {%-     else %}
-  throttle
-  put_policy "{{ index }}-logs" "{{ index }}-logs" '{ "policy": {{ settings.policy | tojson(true) }} }' &
+  echo
+  echo "Setting up {{ index }}-logs policy..."
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+  echo
 {%-     endif %}
 {%-   endif %}
 {%- endfor %}
+echo
 {%- if GLOBALS.role != "so-heavynode" %}
 {%-   for index, settings in ALL_ADDON_SETTINGS.items() %}
 {%-     if settings.policy is defined %}
-  throttle
-  put_policy "{{ index }}-logs" "{{ index }}-logs" '{ "policy": {{ settings.policy | tojson(true) }} }' &
+  echo
+  echo "Setting up {{ index }}-logs policy..."
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+  echo
 {%-     endif %}
 {%-   endfor %}
 {%- endif %}
-
-wait || true
-
-if [[ -s "$ILM_FAIL_FILE" ]]; then
-  echo "ERROR: Failed to load ILM policy(s):"
-  while read -r POLICY; do
-    echo " - $POLICY"
-  done < "$ILM_FAIL_FILE"
-  exit 1
-else
-  echo "Successfully loaded all ILM policies."
-fi

salt/global/defaults.yaml

@@ -1,3 +1,3 @@
 global:
   pcapengine: SURICATA
-  pipeline: REDIS
+  pipeline: REDIS

salt/hydra/enabled.sls

@@ -22,7 +22,6 @@ include:
 so-hydra:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-hydra:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: hydra
     - name: so-hydra
     - networks:
@@ -59,6 +58,7 @@ so-hydra:
       - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
     {%   endfor %}
     {% endif %}
+    - restart_policy: unless-stopped
     - watch:
       - file: hydraconfig
     - require:

salt/idh/enabled.sls

@@ -15,7 +15,6 @@ include:
 so-idh:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-idh:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - name: so-idh
     - detach: True
     - network_mode: host

salt/influxdb/enabled.sls

@@ -18,7 +18,6 @@ include:
 so-influxdb:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-influxdb:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: influxdb
     - networks:
       - sobridge:

salt/kafka/enabled.sls

@@ -27,13 +27,12 @@ include:
 so-kafka:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kafka:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: so-kafka
     - name: so-kafka
     - networks:
       - sobridge:
         - ipv4_address: {{ DOCKERMERGED.containers['so-kafka'].ip }}
-    - user: "960"
+    - user: kafka
     - environment:
         KAFKA_HEAP_OPTS: -Xmx2G -Xms1G
         KAFKA_OPTS: "-javaagent:/opt/jolokia/agents/jolokia-agent-jvm-javaagent.jar=port=8778,host={{ DOCKERMERGED.containers['so-kafka'].ip }},policyLocation=file:/opt/jolokia/jolokia.xml {%- if KAFKA_EXTERNAL_ACCESS %} -Djava.security.auth.login.config=/opt/kafka/config/kafka_server_jaas.conf {% endif -%}"

salt/kibana/defaults.yaml

@@ -22,7 +22,7 @@ kibana:
           - default
           - file
     migrations:
-      discardCorruptObjects: "9.3.7"
+      discardCorruptObjects: "9.3.3"
     telemetry:
       enabled: False
     xpack:

salt/kibana/enabled.sls

@@ -6,7 +6,6 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
-{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 
 include:
@@ -17,9 +16,8 @@ include:
 so-kibana:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kibana:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: kibana
-    - user: "932:0"
+    - user: kibana
     - networks:
       - sobridge:
         - ipv4_address: {{ DOCKERMERGED.containers['so-kibana'].ip }}
@@ -62,19 +60,6 @@ so-kibana:
     - watch:
       - file: kibanaconfig
 
-wait_for_so-kibana:
-  http.wait_for_successful_query:
-    - name: "http://localhost:5601/api/status"
-    - username: 'so_elastic'
-    - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}'
-    - ssl: True
-    - verify_ssl: False
-    - status: 200
-    - wait_for: 600
-    - request_interval: 15
-    - require:
-      - docker_container: so-kibana
-
 delete_so-kibana_so-status.disabled:
   file.uncomment:
     - name: /opt/so/conf/so-status/so-status.conf

salt/kratos/enabled.sls

@@ -15,7 +15,6 @@ include:
 so-kratos:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kratos:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: kratos
     - name: so-kratos
     - networks:
@@ -52,6 +51,7 @@ so-kratos:
       - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
     {%   endfor %}
     {% endif %}
+    - restart_policy: unless-stopped
     - watch:
       - file: kratosschema
       - file: kratosconfig

salt/kratos/soc_kratos.yaml

@@ -103,7 +103,7 @@ kratos:
   config:
     session:
       lifespan: 
-        description: Defines the length of a login session before it will timeout, and require a new login.
+        description: Defines the length of a login session.
         global: True
         helpLink: kratos
       whoami:

salt/libvirt/dev/saltDev.sls

@@ -1,67 +0,0 @@
-# This state is designed to run on a development manager running in a libvirt VM. It will map the default pillar and salt directories
-# from /opt/so/saltstack/default to your local development machine as the source path.
-# The VM requires a filesystem to be added. Only the source path should be changed to your development codebase
-#  Driver: virtio-9p
-#  Source path: ~/project/securityonion
-#  Target path: saltDev
-
-# If you want a directory to be RW, then kvm must have group privileges.
-# ll /home/user/projects/securityonion/salt/hypervisor
-# total 48
-# drwxrwxr-x  3 user kvm 4096 Feb 13 11:18 ./
-# drwxrwxr-x 64 user user 4096 Feb 13 10:32 ../
-# -rw-rw-r--  1 user kvm 2238 Feb 12 15:06 defaults.yaml
-# -rw-rw-r--  1 user kvm 1467 Feb 12 15:06 init.sls
-# -rw-rw-r--  1 user kvm   70 Feb 13 09:37 soc_hypervisor.yaml
-# drwxrwxr-x  3 user kvm 4096 Feb 12 15:06 tools/
-
-# Ensure required kernel modules are configured for loading
-/etc/modules-load.d/virtio-9p.conf:
-  file.managed:
-    - contents: |
-        9pnet_virtio
-        9pnet
-        9p
-    - mode: 644
-    - user: root
-    - group: root
-
-# Load the kernel modules immediately (in the correct order)
-load_9p_modules:
-  cmd.run:
-    - names:
-      - modprobe 9pnet_virtio
-      - modprobe 9pnet
-      - modprobe 9p
-    - unless: lsmod | grep -E '9pnet_virtio|9pnet|9p'
-
-# Ensure mount point exists
-/opt/so/saltstack/default:
-  file.directory:
-    - user: root
-    - group: root
-    - mode: 755
-    - makedirs: True
-
-# Configure fstab entry using mount.fstab_present
-# Configure fstab entry using mount.fstab_present
-saltdev_fstab:
-  mount.fstab_present:
-    - name: saltDev
-    - fs_file: /opt/so/saltstack/default
-    - fs_vfstype: 9p
-    - fs_mntops: _netdev,trans=virtio,version=9p2000.L
-    - fs_freq: 0
-    - fs_passno: 0
-
-# Mount the filesystem if not already mounted
-mount_saltdev:
-  mount.mounted:
-    - name: /opt/so/saltstack/default
-    - device: saltDev
-    - fstype: 9p
-    - opts: _netdev,trans=virtio,version=9p2000.L
-    - require:
-      - file: /opt/so/saltstack/default
-      - mount: saltdev_fstab
-      - cmd: load_9p_modules

salt/logrotate/defaults.yaml

@@ -150,16 +150,6 @@ logrotate:
       - extension .log
       - dateext
       - dateyesterday
-    /opt/so/log/postgres/*_x_log:
-      - daily
-      - rotate 14
-      - missingok
-      - copytruncate
-      - compress
-      - create
-      - extension .log
-      - dateext
-      - dateyesterday
     /opt/so/log/telegraf/*_x_log:
       - daily
       - rotate 14
@@ -220,7 +210,17 @@ logrotate:
       - extension .log
       - dateext
       - dateyesterday
-    /opt/so/log/salt/so-soup-grid-highstate:
+    /opt/so/log/salt/virtual_node_manager:
+      - daily
+      - rotate 14
+      - missingok
+      - copytruncate
+      - compress
+      - create
+      - extension .log
+      - dateext
+      - dateyesterday
+    /opt/so/log/salt/so-salt-cloud:
       - daily
       - rotate 14
       - missingok

salt/logrotate/soc_logrotate.yaml

@@ -91,13 +91,6 @@ logrotate:
       multiline: True
       global: True
       forcedType: "[]string"
-    "/opt/so/log/postgres/*_x_log":
-      description: List of logrotate options for this file.
-      title: /opt/so/log/postgres/*.log
-      advanced: True
-      multiline: True
-      global: True
-      forcedType: "[]string"
     "/opt/so/log/telegraf/*_x_log":
       description: List of logrotate options for this file.
       title: /opt/so/log/telegraf/*.log
@@ -140,9 +133,16 @@ logrotate:
       multiline: True
       global: True
       forcedType: "[]string"
-    "/opt/so/log/salt/so-soup-grid-highstate":
+    "/opt/so/log/salt/virtual_node_manager":
       description: List of logrotate options for this file.
-      title: /opt/so/log/salt/so-soup-grid-highstate
+      title: /opt/so/log/salt/virtual_node_manager
+      advanced: True
+      multiline: True
+      global: True
+      forcedType: "[]string"
+    "/opt/so/log/salt/so-salt-cloud":
+      description: List of logrotate options for this file.
+      title: /opt/so/log/salt/so-salt-cloud
       advanced: True
       multiline: True
       global: True

salt/logstash/config.sls

@@ -81,10 +81,6 @@ ls_custom_pipeline_conf_{{assigned_pipeline}}_{{pipeline}}:
 
 
 {% for assigned_pipeline in ASSIGNED_PIPELINES %}
-{# a blank per-pipeline setting falls back to the global logstash.yml value #}
-{% set PIPELINE_OVERRIDES = LOGSTASH_MERGED.get('pipeline_settings', {}).get(assigned_pipeline, {}) %}
-{% set THREADS = PIPELINE_OVERRIDES.get('pipeline_x_workers') or LOGSTASH_MERGED.config.pipeline_x_workers %}
-{% set BATCH = PIPELINE_OVERRIDES.get('pipeline_x_batch_x_size') or LOGSTASH_MERGED.config.pipeline_x_batch_x_size %}
     {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %}
 ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}:
   file.managed:
@@ -96,8 +92,8 @@ ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_")
         GLOBALS: {{ GLOBALS }}
         ES_USER: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') }}"
         ES_PASS: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') }}"
-        THREADS: {{ THREADS }}
-        BATCH: {{ BATCH }}
+        THREADS: {{ LOGSTASH_MERGED.config.pipeline_x_workers }}
+        BATCH: {{ LOGSTASH_MERGED.config.pipeline_x_batch_x_size }}
       {% else %}
     - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1]}}
       {% endif %}

salt/logstash/defaults.yaml

@@ -60,259 +60,6 @@ logstash:
     custom008: PLACEHOLDER
     custom009: PLACEHOLDER
     custom010: PLACEHOLDER
-  pipeline_settings:
-    fleet:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    manager:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    receiver:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    search:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom0:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom1:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom2:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom3:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom4:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
   settings:
     lsheap: 500m
   config:

salt/logstash/enabled.sls

@@ -28,13 +28,12 @@ include:
 so-logstash:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-logstash:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: so-logstash
     - name: so-logstash
     - networks:
       - sobridge:
         - ipv4_address: {{ DOCKERMERGED.containers['so-logstash'].ip }}
-    - user: "931:0"
+    - user: logstash
     - extra_hosts:
     {% for node in LOGSTASH_NODES %}
     {%   for hostname, ip in node.items() %}
@@ -105,7 +104,6 @@ so-logstash:
     {% endif %}
     - watch:
       - file: lsetcsync
-      - file: lspipelinesyml
       - file: trusttheca
       {% if GLOBALS.is_manager %}
       - file: elasticsearch_cacerts

salt/logstash/etc/pipelines.yml.jinja

@@ -1,17 +1,4 @@
-{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
-{%- set PIPELINE_SETTINGS = LOGSTASH_MERGED.get('pipeline_settings', {}) %}
 {%- for assigned_pipeline in ASSIGNED_PIPELINES %}
 - pipeline.id: {{ assigned_pipeline }}
   path.config: "/usr/share/logstash/pipelines/{{ assigned_pipeline }}/"
-{%-   set extra = PIPELINE_SETTINGS.get(assigned_pipeline, {}) %}
-{%-   if extra is mapping %}
-{#-     values are emitted unquoted so yaml re-infers the type logstash expects:
-        4 as an integer, false as a boolean, 1024mb and auto as strings #}
-{%-     for key, value in extra | dictsort %}
-{%-       set rendered = key | replace('_x_', '.') %}
-{%-       if value not in ['', None] and rendered not in ['pipeline.id', 'path.config'] %}
-  {{ rendered }}: {{ value }}
-{%-       endif %}
-{%-     endfor %}
-{%-   endif %}
 {% endfor -%}

salt/logstash/soc_logstash.yaml

@@ -16,7 +16,6 @@ logstash:
       heavynode: *assigned_pipelines
       searchnode: *assigned_pipelines
       manager: *assigned_pipelines
-      managerhype: *assigned_pipelines
       managersearch: *assigned_pipelines
       fleet: *assigned_pipelines
   defined_pipelines:
@@ -52,322 +51,6 @@ logstash:
     custom008: *pipeline_config
     custom009: *pipeline_config
     custom010: *pipeline_config
-  pipeline_settings:
-    manager: &pipeline_settings
-      pipeline_x_workers:
-        description: >-
-          Number of worker threads that run filters and outputs for this pipeline. May be set higher
-          than the CPU core count when outputs spend time waiting on I/O. Leave blank to use the value
-          from logstash.yml.
-        title: pipeline.workers
-        regex: '^$|^[1-9][0-9]*$'
-        regexFailureMessage: Must be blank, or a positive whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_batch_x_size:
-        description: >-
-          Maximum number of events an individual worker thread collects before running filters and
-          outputs. Larger batches are more efficient but increase heap use; total in-flight events is
-          workers multiplied by batch size. Leave blank to use the value from logstash.yml.
-        title: pipeline.batch.size
-        regex: '^$|^[1-9][0-9]*$'
-        regexFailureMessage: Must be blank, or a positive whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_batch_x_delay:
-        description: >-
-          Milliseconds a worker waits for the next event before running a batch that is not yet full.
-          Leave blank to use the value from logstash.yml.
-        title: pipeline.batch.delay
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_batch_x_metrics_x_sampling_mode:
-        description: >-
-          How much batch size metering this pipeline records. Fuller sampling helps size batches but
-          consumes additional heap. Leave blank to use the value from logstash.yml.
-        title: pipeline.batch.metrics.sampling_mode
-        options:
-        - ''
-        - 'disabled'
-        - 'minimal'
-        - 'full'
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_ordered:
-        description: >-
-          Whether event order is preserved through this pipeline. auto enables ordering only when the
-          pipeline runs a single worker. Leave blank to use the value from logstash.yml.
-        title: pipeline.ordered
-        options:
-        - ''
-        - 'auto'
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_ecs_compatibility:
-        description: >-
-          Elastic Common Schema compatibility mode for plugins in this pipeline. Security Onion sets
-          this globally and it should rarely be changed per pipeline. Leave blank to use the value
-          from logstash.yml.
-        title: pipeline.ecs_compatibility
-        options:
-        - ''
-        - 'disabled'
-        - 'v1'
-        - 'v8'
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_reloadable:
-        description: >-
-          Whether this pipeline may be reloaded when its configuration changes. Leave blank to use the
-          value from logstash.yml.
-        title: pipeline.reloadable
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_type:
-        description: >-
-          Queue backing this pipeline. persisted buffers events to disk under /nsm/logstash so they
-          survive a restart, at some throughput cost; memory does not. Leave blank to use the value
-          from logstash.yml.
-        title: queue.type
-        options:
-        - ''
-        - 'memory'
-        - 'persisted'
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_max_bytes:
-        description: >-
-          Total size of the persistent queue for this pipeline. Only applies when queue.type is
-          persisted, and must fit the disk backing /nsm/logstash. Leave blank to use the value from
-          logstash.yml.
-        title: queue.max_bytes
-        regex: '^$|^[0-9]+(b|kb|mb|gb|tb|pb)$'
-        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_page_capacity:
-        description: >-
-          Size of each page in the persistent queue for this pipeline. Leave blank to use the value
-          from logstash.yml.
-        title: queue.page_capacity
-        regex: '^$|^[0-9]+(b|kb|mb|gb|tb|pb)$'
-        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_max_events:
-        description: >-
-          Maximum number of events in the persistent queue for this pipeline. 0 means unlimited. Leave
-          blank to use the value from logstash.yml.
-        title: queue.max_events
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_checkpoint_x_acks:
-        description: >-
-          Number of acknowledged events before a persistent queue checkpoint is forced. 0 means
-          unlimited. Leave blank to use the value from logstash.yml.
-        title: queue.checkpoint.acks
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_checkpoint_x_writes:
-        description: >-
-          Number of written events before a persistent queue checkpoint is forced. 0 means unlimited.
-          Leave blank to use the value from logstash.yml.
-        title: queue.checkpoint.writes
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_checkpoint_x_interval:
-        description: >-
-          Milliseconds between persistent queue head page checkpoints. 0 disables periodic
-          checkpointing. Leave blank to use the value from logstash.yml.
-        title: queue.checkpoint.interval
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_checkpoint_x_retry:
-        description: >-
-          Whether Logstash retries a failed persistent queue checkpoint write. Leave blank to use the
-          value from logstash.yml.
-        title: queue.checkpoint.retry
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_compression:
-        description: >-
-          Compression applied to persistent queue pages for this pipeline, trading CPU for disk. Leave
-          blank to use the value from logstash.yml.
-        title: queue.compression
-        options:
-        - ''
-        - 'none'
-        - 'speed'
-        - 'balanced'
-        - 'size'
-        - 'disabled'
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_drain:
-        description: >-
-          Whether Logstash drains the persistent queue before shutting down this pipeline. Draining a
-          large queue makes shutdown take considerably longer. Leave blank to use the value from
-          logstash.yml.
-        title: queue.drain
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_enable:
-        description: >-
-          Whether events this pipeline cannot process are written to a dead letter queue instead of
-          being dropped. Leave blank to use the value from logstash.yml.
-        title: dead_letter_queue.enable
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_max_bytes:
-        description: >-
-          Total size of the dead letter queue for this pipeline. Leave blank to use the value from
-          logstash.yml.
-        title: dead_letter_queue.max_bytes
-        regex: '^$|^[0-9]+(b|kb|mb|gb|tb|pb)$'
-        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb.
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_flush_interval:
-        description: >-
-          Milliseconds before a partial dead letter queue segment is flushed. Leave blank to use the
-          value from logstash.yml.
-        title: dead_letter_queue.flush_interval
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_flush_check_interval:
-        description: >-
-          Milliseconds between checks for a dead letter queue segment that needs flushing. Leave blank
-          to use the value from logstash.yml.
-        title: dead_letter_queue.flush_check_interval
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_storage_policy:
-        description: >-
-          What happens when the dead letter queue is full: drop_newer discards incoming events,
-          drop_older discards the oldest stored events. Leave blank to use the value from
-          logstash.yml.
-        title: dead_letter_queue.storage_policy
-        options:
-        - ''
-        - 'drop_newer'
-        - 'drop_older'
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_retain_x_age:
-        description: >-
-          How long an event is kept in the dead letter queue before removal, such as 5d. Leave blank
-          to use the value from logstash.yml.
-        title: dead_letter_queue.retain.age
-        regex: '^$|^[0-9]+[dhms]$'
-        regexFailureMessage: Must be blank, or a number followed by d, h, m, or s, such as 5d.
-        advanced: True
-        global: False
-        helpLink: logstash
-      path_x_queue:
-        description: >-
-          Directory inside the Logstash container holding the persistent queue for this pipeline. The
-          default lives under the /nsm/logstash bind mount; a path outside it will not survive a
-          container restart. Leave blank to use the value from logstash.yml.
-        title: path.queue
-        advanced: True
-        global: False
-        helpLink: logstash
-      path_x_dead_letter_queue:
-        description: >-
-          Directory inside the Logstash container holding the dead letter queue for this pipeline. The
-          default lives under the /nsm/logstash bind mount; a path outside it will not survive a
-          container restart. Leave blank to use the value from logstash.yml.
-        title: path.dead_letter_queue
-        advanced: True
-        global: False
-        helpLink: logstash
-      config_x_debug:
-        description: >-
-          Whether the fully compiled configuration for this pipeline is written to the log. The output
-          may contain sensitive values from the pipeline configuration. Leave blank to use the value
-          from logstash.yml.
-        title: config.debug
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      config_x_support_escapes:
-        description: >-
-          Whether escape sequences such as \n and \t in this pipeline's quoted strings are
-          interpreted. Leave blank to use the value from logstash.yml.
-        title: config.support_escapes
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-    fleet: *pipeline_settings
-    receiver: *pipeline_settings
-    search: *pipeline_settings
-    custom0: *pipeline_settings
-    custom1: *pipeline_settings
-    custom2: *pipeline_settings
-    custom3: *pipeline_settings
-    custom4: *pipeline_settings
   settings:
     lsheap: 
       description: Heap size to use for logstash

salt/manager/beacons.sls

@@ -1,21 +0,0 @@
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-{% from 'salt/auto_apply.map.jinja' import AUTOAPPLY %}
-
-include:
-  - salt.minion
-
-{% if GLOBALS.is_manager and AUTOAPPLY.enabled %}
-salt_beacons_pushstate:
-  file.managed:
-    - name: /etc/salt/minion.d/beacons_pushstate.conf
-    - source: salt://manager/files/beacons_pushstate.conf.jinja
-    - template: jinja
-    - watch_in:
-      - service: salt_minion_service
-{% else %}
-salt_beacons_pushstate:
-  file.absent:
-    - name: /etc/salt/minion.d/beacons_pushstate.conf
-    - watch_in:
-      - service: salt_minion_service
-{% endif %}

salt/manager/files/beacons_pushstate.conf.jinja

@@ -1,11 +0,0 @@
-{% from 'salt/auto_apply.map.jinja' import AUTOAPPLY %}
-beacons:
-  postgres_pillar_beacon:
-    - interval: {{ AUTOAPPLY.drain_interval }}
-    - disable_during_state_run: False
-  rules_beacon:
-    - interval: {{ AUTOAPPLY.drain_interval }}
-    - disable_during_state_run: False
-    - paths:
-        /opt/so/saltstack/local/salt/suricata/rules: suricata
-        /opt/so/saltstack/local/salt/strelka/rules/compiled: strelka

salt/manager/files/mirror-kernel.txt

@@ -1,2 +0,0 @@
-https://repo.securityonion.net/file/so-repo/prod/3/oracle/9-uek8
-https://repo-alt.securityonion.net/prod/3/oracle/9-uek8

salt/manager/files/repodownload.conf

@@ -10,9 +10,4 @@ keepcache=0
 name=Security Onion Repo repo
 mirrorlist=file:///opt/so/conf/reposync/mirror.txt
 enabled=1
-gpgcheck=1
-[securityonionkernelsync]
-name=Security Onion Kernel Repo repo
-mirrorlist=file:///opt/so/conf/reposync/mirror-kernel.txt
-enabled=1
-gpgcheck=1
+gpgcheck=1

salt/manager/init.sls

@@ -15,7 +15,6 @@ include:
   - manager.elasticsearch
   - manager.kibana
   - manager.managed_soc_annotations
-  - manager.beacons
 
 repo_log_dir:
   file.directory:
@@ -87,28 +86,6 @@ repo_dir:
       - group
     - show_changes: False
 
-kernelrepo_dir:
-  file.directory:
-    - name: /nsm/kernelrepo
-    - user: socore
-    - group: socore
-    - recurse:
-      - user
-      - group
-    - show_changes: False
-
-# Ensure /nsm/kernelrepo is always a valid (if empty) repo before it is ever assigned to
-# a client. Without repodata/repomd.xml an enabled file:///nsm/kernelrepo repo makes every
-# dnf operation fail; so-repo-sync only populates it after the highstate, so seed an empty
-# repo here. Only runs when repodata is missing, so it won't clobber a synced repo.
-kernelrepo_init_empty:
-  cmd.run:
-    - name: createrepo /nsm/kernelrepo
-    - unless: 'test -e /nsm/kernelrepo/repodata/repomd.xml'
-    - require:
-      - file: kernelrepo_dir
-      - pkg: install_createrepo
-
 manager_sbin:
   file.recurse:
     - name: /usr/sbin
@@ -145,13 +122,6 @@ so-repo-mirrorlist:
     - user: socore
     - group: socore
 
-so-repo-kernel-mirrorlist:
-  file.managed:
-    - name: /opt/so/conf/reposync/mirror-kernel.txt
-    - source: salt://manager/files/mirror-kernel.txt
-    - user: socore
-    - group: socore
-
 so-repo-sync:
   {%     if MANAGERMERGED.reposync.enabled %}
   cron.present:
@@ -261,7 +231,6 @@ surifiltersrules:
     - user: 939
     - group: 939
 
-
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/manager/managed_soc_annotations.sls

@@ -16,35 +16,40 @@
 {%       endif %}
 {%     endfor %}
 {%   endfor %}
-{%   set soc_annotation_lines = [] %}
-{%   set defaults_lines = [] %}
-{%   for k in matched_integration_names %}
-{%     do soc_annotation_lines.append('    ' ~ k ~ ': *dataStreamSettings') %}
-{%     do defaults_lines.append('    ' ~ k ~ ':') %}
-{%     set defaults_yaml = salt['slsutil.serialize']('yaml', ADDON_INTEGRATION_DEFAULTS[k], default_flow_style=False).strip() %}
-{%     for line in defaults_yaml.splitlines() %}
-{%       do defaults_lines.append('      ' ~ line) %}
-{%     endfor %}
-{%   endfor %}
 {%   set es_soc_annotations = '/opt/so/saltstack/default/salt/elasticsearch/soc_elasticsearch.yaml' %}
-manage_soc_annotations:
-  file.blockreplace:
-    - name: {{ es_soc_annotations }}
-    - marker_start: '    # START managed SOC integration annotations'
-    - marker_end: '    # END managed SOC integration annotations'
-    - content: {{ soc_annotation_lines | join('\n') | tojson }}
-    - insert_after_match: '^    # Managed SOC integration annotations are inserted below this line\.'
-    - append_if_not_found: False
-    - show_changes: True
+{{   es_soc_annotations }}:
+     file.serialize:
+       - dataset:
+           {% set data = salt['file.read'](es_soc_annotations) | load_yaml %}
+           {% set es = data.get('elasticsearch', {}) %}
+           {% set index_settings = es.get('index_settings', {}) %}
+           {% set input = index_settings.get('so-logs', {}) %}
+           {% for k in matched_integration_names %}
+           {%   do index_settings.update({k: input}) %}
+           {% endfor %}
+           {% for k in addon_integration_keys %}
+           {%   if k not in matched_integration_names and k in index_settings %}
+           {%     do index_settings.pop(k) %}
+           {%   endif %}
+           {% endfor %}
+           {{ data }}
 
 {#   Managed elasticsearch/defaults.yaml file for enabling 'Revert to default' via SOC UI for newly added config items #}
 {%   set es_defaults = '/opt/so/saltstack/default/salt/elasticsearch/defaults.yaml' %}
 {{   es_defaults }}:
-  file.blockreplace:
-    - marker_start: '    # START managed SOC integration defaults'
-    - marker_end: '    # END managed SOC integration defaults'
-    - content: {{ defaults_lines | join('\n') | tojson }}
-    - insert_after_match: '^  index_settings:$'
-    - append_if_not_found: False
-    - show_changes: True
-{% endif %}
+     file.serialize:
+       - dataset:
+           {% set data = salt['file.read'](es_defaults) | load_yaml %}
+           {% set es = data.get('elasticsearch', {}) %}
+           {% set index_settings = es.get('index_settings', {}) %}
+           {% for k in matched_integration_names %}
+           {%   set input = ADDON_INTEGRATION_DEFAULTS[k] %}
+           {%     do index_settings.update({k: input})%}
+           {% endfor %}
+           {% for k in addon_integration_keys %}
+           {%   if k not in matched_integration_names and k in index_settings %}
+           {%     do index_settings.pop(k) %}
+           {%   endif %}
+           {% endfor %}
+           {{ data }}
+{% endif %}

salt/manager/sync_es_users.sls

@@ -31,13 +31,11 @@ sync_es_users:
       - http: wait_for_kratos
       - file: so-user.lock # require so-user.lock file to be missing
 
-# we dont want this added too early in setup, so the onlyif gates on the
-# /opt/so/state/setup-complete marker. The marker is written by
-# mark_setup_complete in setup/so-functions just before the final setup
-# highstate (and by an upgrade-path state for systems set up under the old gate).
+# we dont want this added too early in setup, so we add the onlyif to verify 'startup_states: highstate'
+# is in the minion config. That line is added before the final highstate during setup
 so-user_sync:
   cron.present:
     - user: root
     - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log'
     - identifier: so-user_sync
-    - onlyif: "test -e /opt/so/state/setup-complete"
+    - onlyif: "grep -x 'startup_states: highstate' /etc/salt/minion"

salt/manager/tools/sbin/so-boot-mine-update

@@ -1,117 +0,0 @@
-#!/bin/bash
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Runs once per boot on managers (via so-boot-mine-update.service), before
-# so-boot-highstate.service. Waits for the responsive minion set to settle, pushes
-# mine.update, waits until every up minion has actually reported to the mine, then
-# warms the master's per-minion pillar cache so the mine-backed node pillars (node
-# IPs, ES/Redis/Logstash/hypervisor discovery -- some glob- and some pillar/grain-
-# targeted) are complete before the boot highstate renders them. Otherwise a node
-# that is up but not yet fully reported gets dropped from those pillars and torn
-# out of the configs they build (e.g. so-elasticsearch ExtraHosts -> container recreate).
-
-MAX_WAIT=${MINE_UPDATE_MAX_WAIT:-180}   # hard backstop only
-INTERVAL=10
-STABLE_CHECKS=3                          # up-count must hold steady this many polls
-elapsed=0
-prev=-1
-stable=0
-up=0
-
-# Wait for the *reachable* minion set to settle rather than for every accepted
-# key to report up: an operator may accept a minion's key and then intentionally
-# power off that host, so requiring up >= accepted would never be satisfied and
-# we'd always burn the full MAX_WAIT. Once the responsive count stops growing we
-# stop waiting and run mine.update against whoever is up.
-while [ "$elapsed" -lt "$MAX_WAIT" ]; do
-  up=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null \
-    | python3 -c 'import sys,json; print(len(json.load(sys.stdin)))' 2>/dev/null)
-  up=${up:-0}
-  if [ "$up" -gt 0 ] && [ "$up" -eq "$prev" ]; then
-    stable=$((stable + 1))
-    [ "$stable" -ge "$STABLE_CHECKS" ] && break
-  else
-    stable=0
-  fi
-  prev=$up
-  sleep "$INTERVAL"
-  elapsed=$((elapsed + INTERVAL))
-done
-
-echo "so-boot-mine-update: ${up} minions up (settled after ${elapsed}s); running mine.update"
-/usr/bin/salt '*' mine.update --out=txt
-
-# A node that is up but has not yet re-reported network.ip_addrs to the mine is
-# silently dropped from mine-backed pillars (elasticsearch:nodes, node_data, ...)
-# when highstate recompiles them -- which e.g. removes it from so-elasticsearch
-# ExtraHosts and forces a container recreate. After the broad mine.update above,
-# wait until every up minion actually has network.ip_addrs in the mine, re-pushing
-# mine.update to stragglers, before releasing the boot highstate. Bounded by the
-# same MAX_WAIT backstop so a slow/down node never blocks boot indefinitely.
-missing=""
-while [ "$elapsed" -lt "$MAX_WAIT" ]; do
-  up_json=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null)
-  mine_json=$(/usr/bin/salt-run mine.get '*' network.ip_addrs tgt_type=glob --out=json 2>/dev/null)
-  missing=$(printf '%s' "$up_json" | python3 -c '
-import sys, json
-up = set(json.load(sys.stdin) or [])
-mine = {k for k, v in (json.loads(sys.argv[1]) or {}).items() if v}
-print("\n".join(sorted(up - mine)))
-' "$mine_json" 2>/dev/null)
-  if [ -z "$missing" ]; then
-    echo "so-boot-mine-update: mine complete for all up minions after ${elapsed}s"
-    break
-  fi
-  echo "so-boot-mine-update: mine missing up minion(s): $(echo $missing); re-running mine.update"
-  for m in $missing; do /usr/bin/salt "$m" mine.update --out=txt; done
-  sleep "$INTERVAL"
-  elapsed=$((elapsed + INTERVAL))
-done
-[ -n "$missing" ] && echo "so-boot-mine-update: WARNING ${MAX_WAIT}s backstop hit; up minion(s) still absent from mine: $(echo $missing); highstate may drop them from configs"
-
-# The pillar/compound-targeted node pillars (elasticsearch:nodes, redis:nodes,
-# logstash:nodes, hypervisor:nodes) resolve their target against the master's
-# per-minion data cache (grains+pillar in .../minions/<id>/data.p), populated only
-# when a minion's pillar is (re)compiled -- separately from the mine. A freshly
-# booted node can be in the mine (glob/node_data sees it) yet absent from that
-# cache, so it is dropped from those pillars and from the configs they build (e.g.
-# so-elasticsearch ExtraHosts). Force a synchronous pillar refresh so the master
-# caches every up node's pillar; refresh_pillar wait=True returns only once the
-# pillar is recompiled (and thus cached for matching). Retry stragglers <= MAX_WAIT.
-echo "so-boot-mine-update: warming master pillar cache for pillar/grain-targeted node pillars"
-/usr/bin/salt '*' saltutil.refresh_pillar wait=True --out=txt
-missing=""
-while [ "$elapsed" -lt "$MAX_WAIT" ]; do
-  up_json=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null)
-  cached_json=$(/usr/bin/salt-run cache.pillar tgt='*' --out=json 2>/dev/null)
-  missing=$(printf '%s' "$up_json" | python3 -c '
-import sys, json
-up = set(json.load(sys.stdin) or [])
-cached = {k for k, v in (json.loads(sys.argv[1]) or {}).items() if v}
-print("\n".join(sorted(up - cached)))
-' "$cached_json" 2>/dev/null)
-  if [ -z "$missing" ]; then
-    echo "so-boot-mine-update: pillar cache warm for all up minions after ${elapsed}s"
-    break
-  fi
-  echo "so-boot-mine-update: pillar not yet cached for: $(echo $missing); refreshing"
-  for m in $missing; do /usr/bin/salt "$m" saltutil.refresh_pillar wait=True --out=txt; done
-  sleep "$INTERVAL"
-  elapsed=$((elapsed + INTERVAL))
-done
-[ -n "$missing" ] && echo "so-boot-mine-update: WARNING ${MAX_WAIT}s backstop hit; pillar not cached for: $(echo $missing); pillar-targeted pillars may drop them"
-
-# Log what the mine-backed pillars render so the boot-time state is inspectable.
-/usr/bin/salt-call saltutil.refresh_pillar >/dev/null 2>&1
-sleep 2
-for key in node_data elasticsearch:nodes; do
-  rendered=$(/usr/bin/salt-call --out=json pillar.get "$key" 2>/dev/null \
-    | python3 -c 'import sys,json; print(json.dumps(json.load(sys.stdin).get("local"), indent=2, sort_keys=True))' 2>/dev/null)
-  echo "so-boot-mine-update: ${key} rendered as:"
-  echo "${rendered:-null}"
-done
-exit 0

salt/manager/tools/sbin/so-minion

@@ -138,8 +138,6 @@ function getinstallinfo() {
 		log "ERROR" "Failed to source install variables"
 		return 1
 	fi
-
-	log "INFO" "Fetched install info for $MINION_ID (node type: ${NODETYPE:-unset})"
 }
 
 function pcapspace() {
@@ -485,7 +483,6 @@ function add_sensoroni_with_analyze_to_minion() {
 
 # Sensor settings for the minion pillar
 function add_sensor_to_minion() {
-	log "INFO" "Writing sensor configuration for $MINION_ID (interface: ${INTERFACE:-unset})"
     {
         echo "sensor:"
         echo "  interface: '$INTERFACE'"
@@ -512,8 +509,6 @@ function add_sensor_to_minion() {
         log "ERROR" "Failed to add sensor configuration to $PILLARFILE"
         return 1
     fi
-
-	log "INFO" "Wrote sensor configuration for $MINION_ID"
 }
 
 function add_elastalert_to_minion() {
@@ -586,14 +581,11 @@ function add_telegraf_to_minion() {
     # generates a password on first add and is a no-op on re-add so the cred
     # is stable across repeated so-minion runs. postgres.telegraf_users on the
     # manager creates/updates the DB role from the same pillar.
-	log "INFO" "Provisioning postgres telegraf credential for $MINION_ID"
-	so-telegraf-cred add "$MINION_ID"
-	local result=$?
-	if [ $result -ne 0 ]; then
-		log "ERROR" "Failed to provision postgres telegraf cred for $MINION_ID (exit code: $result)"
-		return 1
-	fi
-	log "INFO" "Provisioned postgres telegraf credential for $MINION_ID"
+    so-telegraf-cred add "$MINION_ID"
+    if [ $? -ne 0 ]; then
+        log "ERROR" "Failed to provision postgres telegraf cred for $MINION_ID"
+        return 1
+    fi
 }
 
 function add_influxdb_to_minion() {
@@ -1051,7 +1043,7 @@ function updateMineAndApplyStates() {
 }
 
 function setupMinionFiles() {
-	log "INFO" "Setting up minion files for $MINION_ID (pillar: $PILLARFILE)"
+	log "INFO" "Setting up minion files for $MINION_ID"
 	
 	# Check to see if nodetype is set
 	if [ -z $NODETYPE ]; then
@@ -1077,10 +1069,7 @@ function setupMinionFiles() {
 	fi
 
 	# Create node-specific configuration
-	create$NODETYPE || {
-		log "ERROR" "Failed to create $NODETYPE configuration for $MINION_ID"
-		return 1
-	}
+	create$NODETYPE || return 1
 	
 	# Ensure proper ownership after all content is written
 	ensure_socore_ownership || return 1

salt/manager/tools/sbin/so-push-drainer

@@ -1,231 +0,0 @@
-#!/opt/saltstack/salt/bin/python3
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-"""
-so-push-drainer
-===============
-
-Scheduled drainer for the active-push feature. Runs on the manager every
-drain_interval seconds (default 15) via a salt schedule in salt/salt/push_drain_schedule.sls.
-
-For each intent file under /opt/so/state/push_pending/*.json whose last_touch
-is older than debounce_seconds, this script:
-  * concatenates the actions lists from every ready intent
-  * dedupes by (state or __highstate__, tgt, tgt_type)
-  * dispatches a single `salt-run state.orchestrate orch.push_batch --async`
-    with the deduped actions list passed as pillar kwargs
-  * deletes the contributed intent files on successful dispatch
-
-Reactor sls files (push_suricata, push_strelka, push_pillar) write intents
-but never dispatch directly
-"""
-
-import fcntl
-import glob
-import json
-import logging
-import logging.handlers
-import os
-import subprocess
-import sys
-import time
-
-import salt.client
-
-PENDING_DIR = '/opt/so/state/push_pending'
-LOCK_FILE = os.path.join(PENDING_DIR, '.lock')
-LOG_FILE = '/opt/so/log/salt/so-push-drainer.log'
-
-HIGHSTATE_SENTINEL = '__highstate__'
-
-
-def _make_logger():
-    logger = logging.getLogger('so-push-drainer')
-    logger.setLevel(logging.INFO)
-    if not logger.handlers:
-        os.makedirs(os.path.dirname(LOG_FILE), exist_ok=True)
-        handler = logging.handlers.RotatingFileHandler(
-            LOG_FILE, maxBytes=5 * 1024 * 1024, backupCount=3,
-        )
-        handler.setFormatter(logging.Formatter(
-            '%(asctime)s | %(levelname)s | %(message)s',
-        ))
-        logger.addHandler(handler)
-    return logger
-
-
-def _load_push_cfg():
-    """Read the salt:auto_apply pillar subtree via salt-call. Returns a dict."""
-    caller = salt.client.Caller()
-    cfg = caller.cmd('pillar.get', 'salt:auto_apply', {})
-    return cfg if isinstance(cfg, dict) else {}
-
-
-def _read_intent(path, log):
-    try:
-        with open(path, 'r') as f:
-            return json.load(f)
-    except (IOError, ValueError) as exc:
-        log.warning('cannot read intent %s: %s', path, exc)
-        return None
-    except Exception:
-        log.exception('unexpected error reading %s', path)
-        return None
-
-
-def _dedupe_actions(actions):
-    seen = set()
-    deduped = []
-    for action in actions:
-        if not isinstance(action, dict):
-            continue
-        state_key = HIGHSTATE_SENTINEL if action.get('highstate') else action.get('state')
-        tgt = action.get('tgt')
-        tgt_type = action.get('tgt_type', 'compound')
-        if not state_key or not tgt:
-            continue
-        key = (state_key, tgt, tgt_type)
-        if key in seen:
-            continue
-        seen.add(key)
-        deduped.append(action)
-    return deduped
-
-
-def _dispatch(actions, log):
-    pillar_arg = json.dumps({'actions': actions})
-    cmd = [
-        'salt-run',
-        'state.orchestrate',
-        'orch.push_batch',
-        'pillar={}'.format(pillar_arg),
-        '--async',
-    ]
-    log.info('dispatching: %s', ' '.join(cmd[:3]) + ' pillar=<{} actions>'.format(len(actions)))
-    try:
-        result = subprocess.run(
-            cmd, check=True, capture_output=True, text=True, timeout=60,
-        )
-    except subprocess.CalledProcessError as exc:
-        log.error('dispatch failed (rc=%s): stdout=%s stderr=%s',
-                  exc.returncode, exc.stdout, exc.stderr)
-        return False
-    except subprocess.TimeoutExpired:
-        log.error('dispatch timed out after 60s')
-        return False
-    except Exception:
-        log.exception('dispatch raised')
-        return False
-    log.info('dispatch accepted: %s', (result.stdout or '').strip())
-    return True
-
-
-def main():
-    log = _make_logger()
-
-    if not os.path.isdir(PENDING_DIR):
-        # Nothing to do; reactors create the dir on first use.
-        return 0
-
-    try:
-        push = _load_push_cfg()
-    except Exception:
-        log.exception('failed to read salt:auto_apply pillar; aborting drain pass')
-        return 1
-
-    if not push.get('enabled', True):
-        log.debug('push disabled; exiting')
-        return 0
-
-    debounce_seconds = int(push.get('debounce_seconds', 30))
-
-    os.makedirs(PENDING_DIR, exist_ok=True)
-    lock_fd = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR, 0o644)
-    try:
-        fcntl.flock(lock_fd, fcntl.LOCK_EX)
-
-        intent_files = [
-            p for p in sorted(glob.glob(os.path.join(PENDING_DIR, '*.json')))
-            if os.path.basename(p) != '.lock'
-        ]
-        if not intent_files:
-            return 0
-
-        now = time.time()
-        ready = []
-        skipped = 0
-        broken = []
-        for path in intent_files:
-            intent = _read_intent(path, log)
-            if not isinstance(intent, dict):
-                broken.append(path)
-                continue
-            last_touch = intent.get('last_touch', 0)
-            if now - last_touch < debounce_seconds:
-                skipped += 1
-                continue
-            ready.append((path, intent))
-
-        for path in broken:
-            try:
-                os.unlink(path)
-            except OSError:
-                pass
-
-        if not ready:
-            if skipped:
-                log.debug('no ready intents (%d still in debounce window)', skipped)
-            return 0
-
-        combined_actions = []
-        oldest_first_touch = now
-        all_paths = []
-        for path, intent in ready:
-            combined_actions.extend(intent.get('actions', []) or [])
-            first = intent.get('first_touch', now)
-            if first < oldest_first_touch:
-                oldest_first_touch = first
-            all_paths.extend(intent.get('paths', []) or [])
-
-        deduped = _dedupe_actions(combined_actions)
-        if not deduped:
-            log.warning('%d intent(s) had no usable actions; clearing', len(ready))
-            for path, _ in ready:
-                try:
-                    os.unlink(path)
-                except OSError:
-                    pass
-            return 0
-
-        debounce_duration = now - oldest_first_touch
-        log.info(
-            'draining %d intent(s): %d action(s) after dedupe (raw=%d), '
-            'debounce_duration=%.1fs, paths=%s',
-            len(ready), len(deduped), len(combined_actions),
-            debounce_duration, all_paths[:20],
-        )
-
-        if not _dispatch(deduped, log):
-            log.warning('dispatch failed; leaving intent files in place for retry')
-            return 1
-
-        for path, _ in ready:
-            try:
-                os.unlink(path)
-            except OSError:
-                log.exception('failed to remove drained intent %s', path)
-
-        return 0
-    finally:
-        try:
-            fcntl.flock(lock_fd, fcntl.LOCK_UN)
-        finally:
-            os.close(lock_fd)
-
-
-if __name__ == '__main__':
-    sys.exit(main())

salt/manager/tools/sbin/so-repo-sync

@@ -10,16 +10,5 @@ NOROOT=1
 set -e
 
 curl --retry 5 --retry-delay 60 -A "reposync/$(sync_options)" https://sigs.securityonion.net/checkup --output /tmp/checkup
-
 dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/
 createrepo /nsm/repo
-
-# The kernel repo section is deployed to repodownload.conf by the manager highstate, which
-# runs AFTER this script during soup. On the first upgrade to a kernel-aware version the
-# on-disk config still predates the section, so guard on its presence to avoid dnf's
-# "Unknown repo: 'securityonionkernelsync'" aborting the sync (set -e). The next sync after the
-# highstate deploys the section will pick it up.
-if grep -q '^\[securityonionkernelsync\]' /opt/so/conf/reposync/repodownload.conf; then
-  dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernelsync --download-metadata -p /nsm/kernelrepo/
-  createrepo /nsm/kernelrepo
-fi

salt/manager/tools/sbin/so-soup-grid-highstate

@@ -1,219 +0,0 @@
-#!/bin/bash
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# so-soup-grid-highstate
-# ======================
-# Drives a batched, role-tiered highstate across every non-manager minion in the
-# grid. soup fires this (detached) after it finishes upgrading the manager so the
-# rest of the grid converges immediately instead of waiting for its own scheduled
-# highstate -- which, since the schedule moved from 15 minutes to 120 minutes
-# (salt:schedule:highstate_interval_minutes), could otherwise leave nodes on the
-# old version for up to ~2.5 hours (interval + splay) while the manager runs new code.
-#
-# Work is done by the existing orch.push_batch orchestration (salt/orch/push_batch.sls),
-# the same runner the active-push drainer uses, so batching/queueing behavior matches.
-# Tiers are dispatched in declaration order: searchnodes/heavynodes (Elasticsearch data
-# nodes) first, then receivers, then everything else -- so the data tier converges before
-# the ingest tier before sensors/fleet/idh/etc.
-#
-# When soup also upgraded Salt itself, remote minions must first highstate onto the new
-# salt-minion package (top.sls gates every real state on G@saltversion, so a stale-version
-# minion only gets salt.minion until it upgrades and reconnects). --salt-upgraded runs that
-# preliminary pass and waits for the fleet to settle before the tiered pass.
-#
-# This is best-effort: soup has already completed by the time this runs, and the 120-minute
-# scheduled highstate remains the backstop for any node that is offline or missed a batch.
-
-LOG_FILE=/opt/so/log/salt/so-soup-grid-highstate
-LOCK_FILE=/opt/so/state/so-soup-grid-highstate.lock
-SETTLE_MAX_WAIT=${GRID_HIGHSTATE_SETTLE_WAIT:-900}   # backstop for the post-salt-upgrade settle loop
-SETTLE_INTERVAL=15
-SETTLE_STABLE_CHECKS=3
-# salt-minion on an upgraded node restarts ~30s after the upgrade state runs
-# (salt/salt/minion/init.sls start_minion_post_upgrade); wait past that before sampling
-# so the settle loop sees the drop-off instead of settling on the pre-restart set.
-SETTLE_INITIAL_WAIT=${GRID_HIGHSTATE_SETTLE_INITIAL_WAIT:-45}
-
-BATCH=""
-BATCH_WAIT=""
-SALT_UPGRADED=false
-REASON="manual"
-
-log() {
-  echo "$(date '+%Y-%m-%d %H:%M:%S') | $*" | tee -a "$LOG_FILE"
-}
-
-usage() {
-  echo "Usage: so-soup-grid-highstate [--batch <spec>] [--batch-wait <sec>] [--salt-upgraded] [--reason <text>]"
-  exit 1
-}
-
-while [ $# -gt 0 ]; do
-  case "$1" in
-    --batch)        BATCH="$2"; shift 2 ;;
-    --batch-wait)   BATCH_WAIT="$2"; shift 2 ;;
-    --salt-upgraded) SALT_UPGRADED=true; shift ;;
-    --reason)       REASON="$2"; shift 2 ;;
-    -h|--help)      usage ;;
-    *)              echo "Unknown option: $1"; usage ;;
-  esac
-done
-
-mkdir -p "$(dirname "$LOG_FILE")" "$(dirname "$LOCK_FILE")"
-
-# Serialize: a second invocation (e.g. two soups, or a manual run overlapping soup's)
-# should not dispatch a competing set of batches.
-exec 9>"$LOCK_FILE"
-if ! flock -n 9; then
-  log "another so-soup-grid-highstate is already running (lock $LOCK_FILE held); exiting"
-  exit 0
-fi
-
-# Resolve batch settings from the salt:auto_apply pillar when not overridden on the
-# command line, falling back to the same defaults orch.push_batch/salt.defaults use.
-if [ -z "$BATCH" ]; then
-  BATCH=$(salt-call --out=newline_values_only pillar.get salt:auto_apply:batch 2>/dev/null)
-  [ -z "$BATCH" ] && BATCH='10%'
-fi
-if [ -z "$BATCH_WAIT" ]; then
-  BATCH_WAIT=$(salt-call --out=newline_values_only pillar.get salt:auto_apply:batch_wait 2>/dev/null)
-  [ -z "$BATCH_WAIT" ] && BATCH_WAIT=15
-fi
-
-MINIONID=$(salt-call --local --out=newline_values_only grains.get id 2>/dev/null)
-[ -z "$MINIONID" ] && MINIONID=$(cat /etc/salt/minion_id 2>/dev/null)
-if [ -z "$MINIONID" ]; then
-  log "could not determine this minion's id; aborting"
-  exit 1
-fi
-
-# Single-node grids (eval/standalone/import with no other accepted keys) have nothing
-# remote to push -- the manager already highstated during soup.
-NUM_ACCEPTED=$(salt-key --out=json --list=accepted 2>/dev/null | jq -r '.minions | length' 2>/dev/null)
-NUM_ACCEPTED=${NUM_ACCEPTED:-0}
-if [ "$NUM_ACCEPTED" -le 1 ]; then
-  log "single node grid ($NUM_ACCEPTED accepted minion(s)); nothing to push (reason=$REASON)"
-  exit 0
-fi
-
-log "starting grid highstate: reason=$REASON minion=$MINIONID accepted=$NUM_ACCEPTED batch=$BATCH batch_wait=$BATCH_WAIT salt_upgraded=$SALT_UPGRADED"
-
-# Count minions currently responsive on the bus (includes this manager).
-count_up() {
-  salt-run manage.up --out=json 2>/dev/null \
-    | python3 -c 'import sys,json; print(len(json.load(sys.stdin)))' 2>/dev/null
-}
-
-# Dispatch a single synchronous orch.push_batch run for the given actions JSON.
-# Synchronous is fine: soup launched us detached, so blocking here does not hold soup up.
-# expect_restart=true marks a dispatch (the salt-upgrade pass) where a non-zero rc is normal
-# because targets restart salt-minion mid-run -- so we don't log a misleading failure warning.
-dispatch() {
-  local desc="$1"
-  local actions="$2"
-  local expect_restart="${3:-false}"
-  local rc
-  log "dispatching $desc"
-  salt-run state.orchestrate orch.push_batch pillar="{\"actions\": $actions}" >>"$LOG_FILE" 2>&1
-  rc=$?
-  if [ "$rc" -eq 0 ]; then
-    log "$desc dispatch completed (rc=0)"
-  elif [ "$expect_restart" = "true" ]; then
-    log "$desc returned rc=$rc; this is expected during a salt upgrade (targets restart salt-minion mid-run). Waiting for them to reconnect before the tiered pass."
-  else
-    log "WARNING: $desc dispatch returned rc=$rc; nodes it missed will converge on the scheduled highstate"
-  fi
-}
-
-# Wait for the reachable minion set to recover to its pre-upgrade size and hold steady.
-# Used after the salt-upgrade pass, where targets restart salt-minion (~30s delayed, see
-# salt/salt/minion/init.sls) and drop off the bus before reconnecting on the new version.
-# target = how many minions were reachable just before the pass; requiring up >= target keeps
-# us from releasing the tiered pass while nodes are still down for their restart (settling on
-# the not-yet-restarted subset). We deliberately compare against the pre-upgrade reachable
-# count, not accepted keys, so a node an operator intentionally powered off never stalls us.
-# Bounded by SETTLE_MAX_WAIT.
-wait_for_settle() {
-  local target="$1"
-  local elapsed=0 prev=-1 stable=0 up=0
-  # Let the delayed salt-minion restart begin before we start counting stability, otherwise
-  # we could see the pre-restart set as "stable" and settle before the drop-off even happens.
-  sleep "$SETTLE_INITIAL_WAIT"
-  elapsed=$SETTLE_INITIAL_WAIT
-  while [ "$elapsed" -lt "$SETTLE_MAX_WAIT" ]; do
-    up=$(count_up); up=${up:-0}
-    if [ "$up" -ge "$target" ] && [ "$up" -eq "$prev" ]; then
-      stable=$((stable + 1))
-      [ "$stable" -ge "$SETTLE_STABLE_CHECKS" ] && break
-    else
-      stable=0
-    fi
-    prev=$up
-    sleep "$SETTLE_INTERVAL"
-    elapsed=$((elapsed + SETTLE_INTERVAL))
-  done
-  if [ "$up" -ge "$target" ]; then
-    log "fleet recovered to ${up} minions up (>= pre-upgrade ${target}) after ${elapsed}s"
-  else
-    log "WARNING: ${SETTLE_MAX_WAIT}s settle backstop hit; only ${up}/${target} pre-upgrade minions back up; proceeding (stragglers converge on the scheduled highstate)"
-  fi
-}
-
-# Pass 0: when Salt itself was upgraded, remote minions still on the old version only match
-# top.sls's 'not G@saltversion' block (salt.minion, which performs the package upgrade). Push
-# an untiered highstate so they upgrade+reconnect, then wait for them to come back before the
-# real tiered pass applies the new version's states.
-if [ "$SALT_UPGRADED" = "true" ]; then
-  PRE_UP=$(count_up); PRE_UP=${PRE_UP:-1}
-  log "pre-upgrade reachable minions (incl. this manager): $PRE_UP"
-  dispatch "salt-upgrade pass (all remote minions)" \
-    "[{\"highstate\": true, \"tgt\": \"not $MINIONID\", \"tgt_type\": \"compound\", \"batch\": \"$BATCH\", \"batch_wait\": $BATCH_WAIT}]" \
-    true
-  log "waiting for minions to reconnect on the new salt version"
-  wait_for_settle "$PRE_UP"
-fi
-
-# Tiered pass: Elasticsearch data nodes first, then receivers, then the remainder. The last
-# tier is defined as the complement of the earlier tiers (and of this manager) so coverage is
-# exhaustive -- sensors, fleet, idh, desktop, hypervisor, and any future role are all included.
-TIER_TGTS=(
-  "( *_searchnode or *_heavynode ) and not $MINIONID"
-  "*_receiver and not $MINIONID"
-  "not $MINIONID and not *_searchnode and not *_heavynode and not *_receiver"
-)
-
-# Count minions a compound target matches, using the master's key/cache data (no execution).
-tier_count() {
-  salt --out=json -C "$1" --preview-target 2>/dev/null | jq 'length' 2>/dev/null
-}
-
-# Build the actions JSON, including only tiers that actually match minions. An empty target
-# would make orch.push_batch's salt.state step return "No minions returned" -- a failure --
-# even though nothing needed to run, and grids commonly lack a tier (no receiver, etc.).
-# Keep the JSON on a single line: salt parses `pillar=<value>` kwargs with a non-DOTALL
-# regex, so an embedded newline makes it treat the whole token as a positional saltenv
-# instead ("No matching salt environment for environment 'pillar=...'").
-actions=""
-for tgt in "${TIER_TGTS[@]}"; do
-  n=$(tier_count "$tgt"); n=${n:-0}
-  if [ "$n" -ge 1 ]; then
-    [ -n "$actions" ] && actions="$actions, "
-    actions="$actions{\"highstate\": true, \"tgt\": \"$tgt\", \"tgt_type\": \"compound\", \"batch\": \"$BATCH\", \"batch_wait\": $BATCH_WAIT}"
-    log "tier matched $n minion(s): $tgt"
-  else
-    log "tier matched 0 minions, skipping: $tgt"
-  fi
-done
-
-if [ -z "$actions" ]; then
-  log "no remote minions matched any tier; nothing to push (reason=$REASON)"
-  exit 0
-fi
-dispatch "tiered pass (searchnodes/heavynodes -> receivers -> remainder)" "[$actions]"
-
-log "grid highstate complete (reason=$REASON)"
-exit 0

salt/nginx/enabled.sls

@@ -34,7 +34,6 @@ make-rule-dir-nginx:
 so-nginx:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-nginx:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: so-nginx
     - networks:
       - sobridge:
@@ -60,7 +59,6 @@ so-nginx:
       - /opt/so/conf/navigator/layers/:/opt/socore/html/navigator/assets/so:ro
       - /opt/so/conf/navigator/config.json:/opt/socore/html/navigator/assets/config.json:ro
       - /nsm/repo:/opt/socore/html/repo:ro
-      - /nsm/kernelrepo:/opt/socore/html/kernelrepo:ro
       - /nsm/rules:/nsm/rules:ro
       {%   if NGINXMERGED.external_suricata %}
       - /opt/so/rules/nids/suri:/surirules:ro

salt/nginx/etc/nginx.conf

@@ -96,14 +96,14 @@ http {
 		add_header 	X-XSS-Protection            "1; mode=block";
 		add_header 	X-Content-Type-Options      nosniff;
 		add_header 	Strict-Transport-Security   "max-age=31536000; includeSubDomains";
-		add_header 	Referrer-Policy             no-referrer;
+		add_header 	referrer-Policy             no-referrer;
 
 		ssl_certificate "/etc/pki/nginx/server.crt";
 		ssl_certificate_key "/etc/pki/nginx/server.key";
 		ssl_session_cache shared:SSL:1m;
 		ssl_session_timeout  10m;
 		ssl_ciphers TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256:TLS_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384:TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256:TLS_RSA_WITH_AES_256_GCM_SHA384:TLS_RSA_WITH_AES_256_CCM:TLS_RSA_WITH_ARIA_256_GCM_SHA384:TLS_RSA_WITH_AES_128_GCM_SHA256:TLS_RSA_WITH_AES_128_CCM:TLS_RSA_WITH_ARIA_128_GCM_SHA256;
-		ssl_ecdh_curve X25519:secp521r1:secp384r1;
+		ssl_ecdh_curve secp521r1:secp384r1;
 		ssl_prefer_server_ciphers on;
 		ssl_protocols TLSv1.2 TLSv1.3;
 	}
@@ -138,14 +138,14 @@ http {
 		add_header 	X-XSS-Protection            "1; mode=block";
 		add_header 	X-Content-Type-Options      nosniff;
 		add_header 	Strict-Transport-Security   "max-age=31536000; includeSubDomains";
-		add_header 	Referrer-Policy             no-referrer;
+		add_header 	referrer-Policy             no-referrer;
 
 		ssl_certificate "/etc/pki/nginx/server.crt";
 		ssl_certificate_key "/etc/pki/nginx/server.key";
 		ssl_session_cache shared:SSL:1m;
 		ssl_session_timeout  10m;
 		ssl_ciphers TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256:TLS_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384:TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256:TLS_RSA_WITH_AES_256_GCM_SHA384:TLS_RSA_WITH_AES_256_CCM:TLS_RSA_WITH_ARIA_256_GCM_SHA384:TLS_RSA_WITH_AES_128_GCM_SHA256:TLS_RSA_WITH_AES_128_CCM:TLS_RSA_WITH_ARIA_128_GCM_SHA256;
-		ssl_ecdh_curve X25519:secp521r1:secp384r1;
+		ssl_ecdh_curve secp521r1:secp384r1;
 		ssl_prefer_server_ciphers on;
 		ssl_protocols TLSv1.2 TLSv1.3;
         location / {
@@ -172,14 +172,14 @@ http {
 		add_header 	X-XSS-Protection            "1; mode=block";
 		add_header 	X-Content-Type-Options      nosniff;
 		add_header 	Strict-Transport-Security   "max-age=31536000; includeSubDomains";
-		add_header 	Referrer-Policy             no-referrer;
+		add_header 	referrer-Policy             no-referrer;
 
 		ssl_certificate "/etc/pki/nginx/server.crt";
 		ssl_certificate_key "/etc/pki/nginx/server.key";
 		ssl_session_cache shared:SSL:1m;
 		ssl_session_timeout  10m;
 		ssl_ciphers TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256:TLS_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384:TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256:TLS_RSA_WITH_AES_256_GCM_SHA384:TLS_RSA_WITH_AES_256_CCM:TLS_RSA_WITH_ARIA_256_GCM_SHA384:TLS_RSA_WITH_AES_128_GCM_SHA256:TLS_RSA_WITH_AES_128_CCM:TLS_RSA_WITH_ARIA_128_GCM_SHA256;
-		ssl_ecdh_curve X25519:secp521r1:secp384r1;
+		ssl_ecdh_curve secp521r1:secp384r1;
 		ssl_prefer_server_ciphers on;
 		ssl_protocols TLSv1.2 TLSv1.3;
 
@@ -198,10 +198,6 @@ http {
 		}
 
 		location / {
-			if ($http_authorization ~* "^Bearer .*$") {
-				return 401;
-			}
-
 			auth_request            /auth/sessions/whoami;
 			auth_request_set        $userid $upstream_http_x_kratos_authenticated_identity_id;
 			proxy_set_header        x-user-id $userid;
@@ -222,13 +218,6 @@ http {
 			add_header              Cache-Control "no-cache, no-store, must-revalidate";
 			add_header              Pragma "no-cache";
 			add_header              Expires "0";
-
-			add_header 	Content-Security-Policy     "default-src 'self' 'unsafe-inline' 'unsafe-eval' https: data: blob: wss:; frame-ancestors 'self'";
-			add_header 	X-Frame-Options             SAMEORIGIN;
-			add_header 	X-XSS-Protection            "1; mode=block";
-			add_header 	X-Content-Type-Options      nosniff;
-			add_header 	Strict-Transport-Security   "max-age=31536000; includeSubDomains";
-			add_header 	Referrer-Policy             no-referrer;
 		}
 
 		location ~ ^/auth/.*?(login|oidc/callback) {
@@ -334,16 +323,6 @@ http {
 			autoindex_localtime on;
 		}
 
-		location /kernelrepo/ {
-			allow all;
-			sendfile on;
-			sendfile_max_chunk 1m;
-			autoindex on;
-			autoindex_exact_size off;
-			autoindex_format html;
-			autoindex_localtime on;
-		}
-
 		location /influxdb/ {
 			auth_request          /auth/sessions/whoami;
 			rewrite               /influxdb/api/(.*) /api/$1 break;
@@ -394,11 +373,6 @@ http {
 			if ($http_authorization = "") {
 				return 403;
 			}
-
-			if ($http_authorization ~* "^Bearer .*$") {
-				return 401;
-			}
-
 			proxy_pass            http://{{ GLOBALS.manager }}:9822/;
 			proxy_read_timeout    90;
 			proxy_connect_timeout 90;
@@ -415,7 +389,7 @@ http {
 		error_page 429 = @error429;
 
 		location @error401 {
-			if ($request_uri ~* (^.*/api/.*|^.*/login.*|^.*/logout.*|^/connect/.*|^/oauth2/.*|^/.*\.map$)) {
+			if ($request_uri ~* (^/api/.*|^/connect/.*|^/oauth2/.*|^/.*\.map$)) {
 				return    401;
 			}
 

salt/orch/push_batch.sls

@@ -1,37 +0,0 @@
-{% from 'salt/auto_apply.map.jinja' import AUTOAPPLY %}
-{% set actions = salt['pillar.get']('actions', []) %}
-{% set BATCH = AUTOAPPLY.batch %}
-{% set BATCH_WAIT = AUTOAPPLY.batch_wait %}
-
-{% for action in actions %}
-{%   if action.get('highstate') %}
-apply_highstate_{{ loop.index }}:
-  salt.state:
-    - tgt: '{{ action.tgt }}'
-    - tgt_type: {{ action.get('tgt_type', 'compound') }}
-    - highstate: True
-    - batch: {{ action.get('batch', BATCH) }}
-    - batch_wait: {{ action.get('batch_wait', BATCH_WAIT) }}
-    - kwarg:
-        queue: 2
-{%   else %}
-refresh_pillar_{{ loop.index }}:
-  salt.function:
-    - name: saltutil.refresh_pillar
-    - tgt: '{{ action.tgt }}'
-    - tgt_type: {{ action.get('tgt_type', 'compound') }}
-
-apply_{{ action.state | replace('.', '_') }}_{{ loop.index }}:
-  salt.state:
-    - tgt: '{{ action.tgt }}'
-    - tgt_type: {{ action.get('tgt_type', 'compound') }}
-    - sls:
-      - {{ action.state }}
-    - batch: {{ action.get('batch', BATCH) }}
-    - batch_wait: {{ action.get('batch_wait', BATCH_WAIT) }}
-    - kwarg:
-        queue: 2
-    - require:
-      - salt: refresh_pillar_{{ loop.index }}
-{%   endif %}
-{% endfor %}

salt/postgres/defaults.yaml

@@ -16,4 +16,4 @@ postgres:
     logging_collector: 'off'
     log_min_messages: 'warning'
     shared_preload_libraries: pg_cron
-    cron.database_name: postgres
+    cron.database_name: so_telegraf

salt/postgres/enabled.sls

@@ -19,7 +19,6 @@ include:
 so-postgres:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-postgres:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: so-postgres
     - networks:
       - sobridge:
@@ -85,23 +84,6 @@ so-postgres:
       - x509: postgres_crt
       - x509: postgres_key
 
-postgres_wait_ready:
-  cmd.run:
-    - name: /usr/sbin/so-postgres-wait
-    - require:
-      - docker_container: so-postgres
-      - file: postgres_sbin
-
-# Reconcile the SOC database (role, grants, so_telegraf) every highstate so a
-# partially-initialized cluster self-heals. POSTGRES_USER is injected because
-# the container env omits it.
-postgres_bootstrap_soc_db:
-  cmd.run:
-    - name: docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh
-    - require:
-      - cmd: postgres_wait_ready
-      - file: postgresinitdb
-
 delete_so-postgres_so-status.disabled:
   file.uncomment:
     - name: /opt/so/conf/so-status/so-status.conf

salt/postgres/files/init-db.sh

@@ -8,20 +8,15 @@ if [ -z "${SO_POSTGRES_PASS:-}" ] && [ -n "${SO_POSTGRES_PASS_FILE:-}" ] && [ -r
     SO_POSTGRES_PASS="$(< "$SO_POSTGRES_PASS_FILE")"
 fi
 psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
-    -- Keep the password out of postgres.log if this DDL errors.
-    SET log_min_error_statement = panic;
-    -- Idempotent, race-safe upsert: CREATE, falling back to ALTER if the role
-    -- already exists or is created concurrently.
     DO \$\$
     BEGIN
-        BEGIN
+        IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${SO_POSTGRES_USER}') THEN
             EXECUTE format('CREATE ROLE %I WITH LOGIN PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}');
-        EXCEPTION WHEN duplicate_object OR unique_violation THEN
-            EXECUTE format('ALTER ROLE %I WITH LOGIN PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}');
-        END;
+        ELSE
+            EXECUTE format('ALTER ROLE %I WITH PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}');
+        END IF;
     END
     \$\$;
-    GRANT ALL ON SCHEMA public TO "$SO_POSTGRES_USER";
     GRANT ALL PRIVILEGES ON DATABASE "$POSTGRES_DB" TO "$SO_POSTGRES_USER";
     -- Lock the SOC database down at the connect layer; PUBLIC gets CONNECT
     -- by default, which would let per-minion telegraf roles open sessions
@@ -36,4 +31,4 @@ EOSQL
 # only ensures the shared database exists on first initialization.
 if ! psql -U "$POSTGRES_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1; then
     psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE DATABASE so_telegraf"
-fi
+fi

salt/postgres/soc_postgres.yaml

@@ -83,7 +83,7 @@ postgres:
       advanced: True
       helpLink: postgres
     cron.database_name:
-      description: Database pg_cron keeps its job metadata in. Must already exist when PostgreSQL starts, because pg_cron's launcher connects to it at startup and never retries if it is missing. The maintenance job itself targets so_telegraf.
+      description: Database pg_cron schedules jobs in. Must be so_telegraf so partman maintenance runs in the right database context.
       global: True
       advanced: True
       helpLink: postgres

salt/postgres/telegraf_users.sls

@@ -8,14 +8,37 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'telegraf/map.jinja' import TELEGRAFMERGED %}
 
-{# postgres.enabled declares the so-postgres container and postgres_wait_ready
-   that the requires below reference. Salt de-duplicates the circular include. #}
+{# postgres_wait_ready below requires `docker_container: so-postgres`, which is
+   declared in postgres.enabled. Include it here so state.apply postgres.telegraf_users
+   on its own (e.g. from orch.deploy_newnode) still has that ID in scope. Salt
+   de-duplicates the circular include. #}
 include:
   - postgres.enabled
 
 {% set TG_OUT = TELEGRAFMERGED.output | upper %}
 {% if TG_OUT in ['POSTGRES', 'BOTH'] %}
 
+# docker_container.running returns as soon as the container starts, but on
+# first-init docker-entrypoint.sh starts a temporary postgres with
+# `listen_addresses=''` to run /docker-entrypoint-initdb.d scripts, then
+# shuts it down before exec'ing the real CMD. A default pg_isready check
+# (Unix socket) passes during that ephemeral phase and races the shutdown
+# with "the database system is shutting down". Checking TCP readiness on
+# 127.0.0.1 only succeeds after the final postgres binds the port.
+postgres_wait_ready:
+  cmd.run:
+    - name: |
+        for i in $(seq 1 60); do
+          if docker exec so-postgres pg_isready -h 127.0.0.1 -U postgres -q 2>/dev/null; then
+            exit 0
+          fi
+          sleep 2
+        done
+        echo "so-postgres did not accept TCP connections within 120s" >&2
+        exit 1
+    - require:
+      - docker_container: so-postgres
+
 # Ensure the shared Telegraf database exists. init-db.sh only runs on a
 # fresh data dir, so hosts upgraded onto an existing /nsm/postgres volume
 # would otherwise never get so_telegraf.

salt/postgres/tools/sbin/so-postgres-backup

@@ -47,12 +47,7 @@ trap 'rm -f "$TMPFILE"' EXIT
 
 # Dump all databases and roles, compress. Write to a temp file so the final
 # filename only ever appears for a complete, verified backup.
-#
-# so_telegraf is excluded: it is transient metrics on a short retention window,
-# it dominates the dump size, and it is rebuilt automatically after a restore --
-# init-db.sh recreates the database and Telegraf recreates its tables on first
-# write. Roles are globals, so the per-minion telegraf logins are still dumped.
-if ! docker exec so-postgres pg_dumpall -U postgres --exclude-database=so_telegraf | gzip > "$TMPFILE"; then
+if ! docker exec so-postgres pg_dumpall -U postgres | gzip > "$TMPFILE"; then
   log "ERROR: pg_dumpall/gzip failed; backup aborted"
   exit 1
 fi

salt/postgres/tools/sbin/so-postgres-wait

@@ -1,32 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Wait for the so-postgres container to accept TCP connections.
-#
-# docker_container.running returns as soon as the container starts, but on
-# first-init docker-entrypoint.sh starts a temporary postgres with
-# `listen_addresses=''` to run /docker-entrypoint-initdb.d scripts, then
-# shuts it down before exec'ing the real CMD. A default pg_isready check
-# (Unix socket) passes during that ephemeral phase and races the shutdown
-# with "the database system is shutting down". Checking TCP readiness on
-# 127.0.0.1 only succeeds after the final postgres binds the port.
-#
-# Usage: so-postgres-wait [iterations] [sleep_seconds]
-# Default: 60 iterations, 2s sleep (~120s total).
-
-ITERATIONS=${1:-60}
-SLEEP_SECONDS=${2:-2}
-
-for i in $(seq 1 "$ITERATIONS"); do
-  if docker exec so-postgres pg_isready -h 127.0.0.1 -U postgres -q 2>/dev/null; then
-    exit 0
-  fi
-  sleep "$SLEEP_SECONDS"
-done
-
-echo "so-postgres did not accept TCP connections within $((ITERATIONS * SLEEP_SECONDS))s" >&2
-exit 1