Update baseline agents

ignore file already closed

pillar/data/addtotab.sh

@@ -1,59 +0,0 @@
-#!/usr/bin/env bash
-
-# This script adds sensors/nodes/etc to the nodes tab
-default_salt_dir=/opt/so/saltstack/default
-local_salt_dir=/opt/so/saltstack/local
-TYPE=$1
-NAME=$2
-IPADDRESS=$3
-CPUS=$4
-GUID=$5
-MANINT=$6
-ROOTFS=$7
-NSM=$8
-MONINT=$9
-#NODETYPE=$10
-#HOTNAME=$11
-
-echo "Seeing if this host is already in here. If so delete it"
-if grep -q $NAME "$local_salt_dir/pillar/data/$TYPE.sls"; then
-  echo "Node Already Present - Let's re-add it"
-  awk -v blah="  $NAME:" 'BEGIN{ print_flag=1 }
-{
-    if( $0 ~ blah )
-    {
-       print_flag=0;
-       next
-    }
-    if( $0 ~ /^  [a-zA-Z0-9]+:$/ )
-    {
-        print_flag=1;
-    }
-    if ( print_flag == 1 )
-        print $0
-
-} ' $local_salt_dir/pillar/data/$TYPE.sls > $local_salt_dir/pillar/data/tmp.$TYPE.sls
-mv $local_salt_dir/pillar/data/tmp.$TYPE.sls $local_salt_dir/pillar/data/$TYPE.sls
-echo "Deleted $NAME from the tab. Now adding it in again with updated info"
-fi
-echo "  $NAME:" >> $local_salt_dir/pillar/data/$TYPE.sls
-echo "    ip: $IPADDRESS" >> $local_salt_dir/pillar/data/$TYPE.sls
-echo "    manint: $MANINT" >> $local_salt_dir/pillar/data/$TYPE.sls
-echo "    totalcpus: $CPUS" >> $local_salt_dir/pillar/data/$TYPE.sls
-echo "    guid: $GUID" >> $local_salt_dir/pillar/data/$TYPE.sls
-echo "    rootfs: $ROOTFS" >> $local_salt_dir/pillar/data/$TYPE.sls
-echo "    nsmfs: $NSM" >> $local_salt_dir/pillar/data/$TYPE.sls
-if [ $TYPE == 'sensorstab' ]; then
-  echo "    monint: bond0" >> $local_salt_dir/pillar/data/$TYPE.sls
-fi
-if [ $TYPE == 'evaltab' ] || [ $TYPE == 'standalonetab' ]; then
-  echo "    monint: bond0" >> $local_salt_dir/pillar/data/$TYPE.sls
-  if [ ! $10 ]; then
-    salt-call state.apply utility queue=True
-  fi
-fi
-if [ $TYPE == 'nodestab' ]; then
-  salt-call state.apply elasticsearch queue=True
-#  echo "    nodetype: $NODETYPE" >> $local_salt_dir/pillar/data/$TYPE.sls
-#  echo "    hotname: $HOTNAME" >> $local_salt_dir/pillar/data/$TYPE.sls
-fi

salt/allowed_states.map.jinja

@@ -37,8 +37,7 @@
     'elasticfleet',
     'elasticfleet.manager',
     'elasticsearch.cluster',
-    'elastic-fleet-package-registry',
+    'elastic-fleet-package-registry'
-    'utility'
 ] %}
 
 {% set sensor_states = [

salt/common/init.sls

@@ -130,6 +130,17 @@ common_sbin:
       - so-pcap-import
 {% endif %}
 
+# Pin physical NIC names by MAC (run-once) so a kernel upgrade can't renumber the
+# interfaces SO binds by name. The marker keeps it a one-time setup; an admin can
+# pre-create the marker to opt out.
+pin_nic_names:
+  cmd.run:
+    - name: /usr/sbin/so-nic-pin
+    - unless: 'test -e /opt/so/state/nic_names_pinned'
+    - require:
+      - file: common_sbin
+      - file: statedir
+
 common_sbin_jinja:
   file.recurse:
     - name: /usr/sbin

salt/common/tools/sbin/so-common

@@ -291,6 +291,20 @@ download_and_verify() {
 	fi
 }
 
+# check if container with name is running and optionally stop it
+docker_check_running() {
+    # show running containers, only names
+    if docker ps --format '{{.Names}}' | grep -q "^so-${1}$"; then
+        if [[ "$2" == "--stop" ]]; then
+            docker stop "so-${1}"
+        fi
+
+        return 0
+    else
+        return 1
+    fi
+}
+
 elastic_license() {
 
 read -r -d '' message <<- EOM

salt/common/tools/sbin/so-kernel-upgrade

@@ -0,0 +1,243 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+#
+# so-kernel-upgrade — install the UEK8 (6.x) kernel and make it the boot default.
+#
+# Security Onion is moving off the EL9 stock kernel (RHCK, 5.14) and UEK7 (5.15) onto UEK8
+# (6.x). Three things have to happen, and the tool has to drive each one:
+#
+#   1. Populate. The manager mirrors the UEK8 packages into /nsm/kernelrepo via so-repo-sync,
+#      and serves them to the grid over https://<manager>/kernelrepo. Until that sync runs the
+#      repo is valid but EMPTY -- dnf resolves it happily and installs nothing, with no error.
+#   2. Install. A node on RHCK has no kernel-uek* package at all, so there is nothing for
+#      'dnf update' to upgrade. A node on UEK7 does have kernel-uek installed, so
+#      'dnf install kernel-uek' reports "Nothing to do" and exits 0 without installing 6.x.
+#      Both cases need an explicit install of the UEK8 NEVRA.
+#   3. Boot it. Whether a newly installed UEK8 kernel becomes the boot default depends on the
+#      RUNNING kernel's flavor. kernel-install/grubby (with UPDATEDEFAULT=yes) only auto-promote
+#      within the running kernel's flavor lineage:
+#        - From UEK7 (5.x, kernel-uek) the install stays in the kernel-uek lineage and IS
+#          auto-promoted, so no grubby change is needed -- just make sure the repo is populated
+#          and install UEK8.
+#        - From the stock EL9 kernel (RHCK, 5.14, no UEK) it is a flavor CROSS that is NOT
+#          auto-promoted, so the box keeps booting RHCK until grubby is told otherwise.
+#      This tool inspects the running kernel and only runs 'grubby --set-default' for RHCK.
+#
+# Every one of those failure modes is silent by default. This tool handles each case and fails
+# loudly when it cannot, rather than reporting success while changing nothing.
+#
+# Manager vs minion: only the manager owns /nsm/kernelrepo, so only the manager can populate
+# it. If the repo is empty here, a manager runs so-repo-sync itself; a minion has no way to
+# fix it and exits non-zero telling the admin to sync the manager first.
+#
+# Idempotent: an already-installed, already-default UEK8 kernel is left alone. It only sets
+# the boot default; it does NOT reboot -- the admin reboots the node on their own schedule.
+
+. /usr/sbin/so-common
+
+# Client-side repo id (what dnf enables on this node, from repo/client/oracle.sls) vs the
+# reposync-side section in repodownload.conf that the manager mirrors from (mirrors the
+# securityonion/securityonionsync split for the main repo).
+KERNEL_REPO="securityonionkernel"
+KERNEL_REPO_SYNC="securityonionkernelsync"
+KERNEL_PKG="kernel-uek"
+KERNEL_REPO_DIR="/nsm/kernelrepo"
+REPOSYNC_CONF="/opt/so/conf/reposync/repodownload.conf"
+GLOBAL_PILLAR="/opt/so/saltstack/local/pillar/global/soc_global.sls"
+
+log() { echo "[so-kernel-upgrade] $*"; }
+die() { echo "[so-kernel-upgrade] ERROR: $*" >&2; exit 1; }
+
+command -v grubby >/dev/null 2>&1 || die "grubby not found"
+command -v dnf >/dev/null 2>&1 || die "dnf not found"
+
+ARCH="$(rpm -E '%{_arch}')"
+
+is_airgap() {
+    [ -f "$GLOBAL_PILLAR" ] && grep -q 'airgap: *[Tt]rue' "$GLOBAL_PILLAR"
+}
+
+# Newest installed UEK8 (6.x) kernel known to the bootloader. UEK8 vmlinuz paths look like
+# /boot/vmlinuz-6.12.0-204.92.4.2.el9uek.x86_64; UEK7 (5.15) and RHCK (5.14) won't match.
+find_uek8() {
+    grubby --info=ALL 2>/dev/null \
+        | sed -n 's/^kernel="\(.*\)"$/\1/p' \
+        | grep -E '/vmlinuz-6\.[0-9]+.*uek' \
+        | sort -V | tail -1
+}
+
+# Classify the RUNNING kernel (uname -r) -- this, not what's installed, is what decides whether
+# a UEK8 install auto-promotes to the boot default:
+#   uek8  6.x UEK    already on the target line; nothing to do
+#   uek7  5.x UEK    a UEK8 install stays in the kernel-uek lineage and auto-promotes (no grubby)
+#   rhck  5.14 EL9   crossing into the UEK flavor does NOT auto-promote (needs grubby --set-default)
+running_flavor() {
+    case "$(uname -r)" in
+        6.*uek*) echo uek8 ;;
+        *uek*)   echo uek7 ;;
+        *)       echo rhck ;;
+    esac
+}
+
+# Newest UEK8 kernel-uek NEVRA offered by the kernel repo, empty if the repo has none.
+# Restricted to the kernel repo so a UEK7 kernel-uek in the main repo can't be picked up,
+# and filtered to 6.x so we never "succeed" by reinstalling the 5.15 we already have.
+uek8_available() {
+    dnf -q repoquery --disablerepo='*' --enablerepo="$KERNEL_REPO" \
+        --arch="$ARCH" --latest-limit=1 \
+        --qf '%{name}-%{evr}.%{arch}\n' "$KERNEL_PKG" 2>/dev/null \
+        | grep -E "^${KERNEL_PKG}-6\." | tail -1
+}
+
+kernelrepo_rpm_count() {
+    find "$KERNEL_REPO_DIR" -maxdepth 1 -name '*.rpm' 2>/dev/null | wc -l
+}
+
+# The kernel repo starts life as valid-but-empty (kernelrepo_init_empty in
+# salt/manager/init.sls) and is filled by so-repo-sync. During a soup, so-repo-sync runs
+# BEFORE the highstate deploys the [securityonionkernelsync] section into repodownload.conf, so
+# the first kernel-aware soup leaves the repo empty until the next nightly sync.
+sync_kernel_repo() {
+    if is_airgap; then
+        log "airgap install: $KERNEL_REPO_DIR is populated from the airgap ISO, not by so-repo-sync."
+        return 1
+    fi
+    if ! grep -q "^\[${KERNEL_REPO_SYNC}\]" "$REPOSYNC_CONF" 2>/dev/null; then
+        log "$REPOSYNC_CONF has no [${KERNEL_REPO_SYNC}] section -- run a highstate to deploy it."
+        return 1
+    fi
+
+    log "populating $KERNEL_REPO_DIR with so-repo-sync (mirrors upstream; can take several minutes)"
+    su socore -c '/usr/sbin/so-repo-sync' || { log "so-repo-sync failed"; return 1; }
+
+    dnf -q clean expire-cache >/dev/null 2>&1
+    return 0
+}
+
+# Make the kernel repo actually able to serve a UEK8 package, or fail trying.
+ensure_kernel_repo() {
+    # The repo is assigned by the repo.client highstate, and only once NICs are pinned by MAC
+    # (/opt/so/state/nic_names_pinned) so the kernel swap can't renumber interfaces SO binds
+    # by name. skip_if_unavailable=1 means a broken repo is silently ignored, so check first.
+    if ! dnf -q repolist --enabled 2>/dev/null | awk '{print $1}' | grep -qx "$KERNEL_REPO"; then
+        log "repo '$KERNEL_REPO' is not enabled on this node."
+        log "Run a highstate first; the repo is skipped until /opt/so/state/nic_names_pinned"
+        log "exists (run so-nic-pin) and this node's salt matches the version this release ships."
+        die "kernel repo unavailable"
+    fi
+
+    [ -n "$(uek8_available)" ] && return 0
+
+    log "repo '$KERNEL_REPO' is enabled but offers no UEK8 $KERNEL_PKG package"
+
+    if ! is_manager_node; then
+        log "This is a minion; it consumes the kernel repo from the manager and cannot populate it."
+        log "On the manager, run:  su socore -c /usr/sbin/so-repo-sync"
+        log "then re-run this script here."
+        die "manager's kernel repo is empty"
+    fi
+
+    log "this is a manager and $KERNEL_REPO_DIR holds $(kernelrepo_rpm_count) rpm(s)"
+    sync_kernel_repo || die "could not populate $KERNEL_REPO_DIR"
+
+    [ -n "$(uek8_available)" ] \
+        || die "so-repo-sync completed but $KERNEL_REPO still offers no UEK8 $KERNEL_PKG"
+}
+
+reboot_notice() {
+    [ "$(uname -r)" = "$(basename "$1" | sed 's/^vmlinuz-//')" ] \
+        || log "REBOOT REQUIRED to start using the UEK8 kernel (currently running $(uname -r))."
+}
+
+# Keep future kernel updates on the UEK line rather than falling back to RHCK. Oracle ships
+# /etc/sysconfig/kernel; only rewrite it when it's actually pointing somewhere else.
+set_default_kernel_conf() {
+    if [ -f /etc/sysconfig/kernel ] && ! grep -q '^DEFAULTKERNEL=kernel-uek-core$' /etc/sysconfig/kernel; then
+        log "setting DEFAULTKERNEL=kernel-uek-core in /etc/sysconfig/kernel"
+        sed -i 's/^DEFAULTKERNEL=.*/DEFAULTKERNEL=kernel-uek-core/' /etc/sysconfig/kernel
+    fi
+}
+
+# Make sure a UEK8 kernel is installed, leaving its boot entry in INSTALLED_UEK8. If one is
+# already present we leave the repo alone -- it may be disabled or empty and we don't need it
+# just to flip the boot default. Otherwise install the explicit NEVRA, not the bare package
+# name: on a UEK7 node 'dnf install kernel-uek' sees 5.15 already present, prints "Nothing to
+# do" and exits 0 without installing 6.x.
+ensure_uek8_installed() {
+    INSTALLED_UEK8="$(find_uek8)"
+    if [ -n "$INSTALLED_UEK8" ]; then
+        log "UEK8 kernel already installed: $INSTALLED_UEK8"
+        return 0
+    fi
+
+    ensure_kernel_repo
+    local nevra; nevra="$(uek8_available)"
+    log "installing $nevra from $KERNEL_REPO"
+    dnf -y install "$nevra" || die "failed to install $nevra"
+
+    INSTALLED_UEK8="$(find_uek8)"
+    [ -n "$INSTALLED_UEK8" ] || die "$nevra installed but no 6.x UEK boot entry appeared -- check 'grubby --info=ALL'"
+    log "installed UEK8 kernel: $INSTALLED_UEK8"
+}
+
+case "$(running_flavor)" in
+uek8)
+    # Already on the 6.x UEK line. A plain 'dnf update' keeps this node current within the
+    # lineage and auto-promotes newer builds, so there is nothing for this tool to do.
+    log "already running a UEK8 kernel ($(uname -r)); nothing to do."
+    exit 0
+    ;;
+
+uek7)
+    # On a 5.x UEK kernel. Installing UEK8 stays inside the kernel-uek lineage, so dnf/grubby
+    # (UPDATEDEFAULT=yes) auto-promote it and we do NOT touch grubby. A node still on UEK7
+    # usually means the kernel repo was empty when it last updated, so populate it and install.
+    log "running UEK7 kernel ($(uname -r)); the kernel repo was likely not yet populated when"
+    log "this node last updated. Populating it and installing UEK8 -- the update stays on the"
+    log "kernel-uek line, so it becomes the boot default automatically (no grubby change needed)."
+    set_default_kernel_conf
+    ensure_uek8_installed
+
+    now="$(grubby --default-kernel 2>/dev/null)"
+    if [ "$now" = "$INSTALLED_UEK8" ]; then
+        log "boot default auto-promoted to UEK8 kernel: $INSTALLED_UEK8"
+    else
+        log "WARNING: expected the UEK8 kernel to auto-promote but the default is still"
+        log "'${now:-unknown}'. Run 'grubby --set-default=$INSTALLED_UEK8' to force it."
+    fi
+    reboot_notice "$INSTALLED_UEK8"
+    ;;
+
+rhck)
+    # On the stock EL9 kernel (5.14, no UEK installed). Crossing from RHCK into the UEK flavor
+    # does NOT auto-promote -- kernel-install/grubby only auto-promote within the running
+    # kernel's flavor lineage -- so after installing we must set the boot default explicitly.
+    log "running stock EL9 (RHCK) kernel ($(uname -r)); installing UEK8 and setting it as the"
+    log "boot default explicitly (a RHCK->UEK flavor change does not auto-promote)."
+    set_default_kernel_conf
+    ensure_uek8_installed
+    target="$INSTALLED_UEK8"
+
+    current="$(grubby --default-kernel 2>/dev/null)"
+    if [ "$current" = "$target" ]; then
+        log "UEK8 kernel is already the boot default: $target"
+        reboot_notice "$target"
+        exit 0
+    fi
+
+    log "current default kernel: ${current:-unknown}"
+    log "switching boot default to UEK8 kernel: $target"
+    grubby --set-default="$target" || die "grubby --set-default failed for $target"
+
+    # Verify the change actually took before claiming success.
+    now="$(grubby --default-kernel 2>/dev/null)"
+    [ "$now" = "$target" ] || die "default kernel is still '${now:-unknown}' after set-default"
+
+    log "boot default is now $target"
+    reboot_notice "$target"
+    ;;
+esac

salt/common/tools/sbin/so-log-check

@@ -132,6 +132,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|HTTP 404: Not Found"          # Salt loops until Kratos returns 200, during startup Kratos may not be ready
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Cancelling deferred write event maybeFenceReplicas because the event queue is now closed" # Kafka controller log during shutdown/restart
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Redis may have been restarted" # Redis likely restarted by salt
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|file already closed"          # Go logging race condition during container restart
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then

salt/common/tools/sbin/so-nic-pin

@@ -0,0 +1,76 @@
+#!/bin/bash
+#
+# so-nic-pin — pin physical NIC names by permanent MAC via classic by-MAC udev
+#              rules, so a kernel upgrade can't renumber them.
+#
+# Security Onion binds its management and monitor interfaces BY NAME in pillar
+# (host:mainint, sensor:mainint, and bond0 is built on a specific physical NIC).
+# A kernel upgrade can change the kernel/systemd-udevd predictable-naming output
+# and renumber those NICs (e.g. enp1s0 -> enp2s0), which breaks the grid: the
+# pillar references a name that no longer exists and bond/bridge bring-up fails.
+#
+# This writes /etc/udev/rules.d/70-persistent-net.rules pinning each PHYSICAL NIC
+# to its CURRENT name by its PERMANENT MAC, freezing the names across future kernel
+# changes. It only writes the rules file; it does NOT live-trigger a rename (the
+# rules apply on the next boot/kernel, and a live rename would be disruptive).
+#
+# Run-once: gated by the drop file /opt/so/state/nic_names_pinned. If the marker is
+# present the script does nothing, so an admin can pre-create it to opt out. Invoked
+# from the common state on every highstate; the marker keeps it a one-time setup.
+
+NET_RULES_FILE="/etc/udev/rules.d/70-persistent-net.rules"
+MARKER="/opt/so/state/nic_names_pinned"
+
+log() { echo -e "[so-nic-pin] $*"; }
+
+# Echo "<name> <permanent-mac>" for every PHYSICAL NIC. A physical NIC is backed by a
+# real device (has device/driver), which excludes bond0/sobridge/docker0/veth*/lo whose
+# MACs are dynamic and must never be pinned. The PERMANENT MAC is used (ethtool -P, with
+# fallbacks), not the current one: an enslaved bond member's current MAC is rewritten to
+# the bond's, so matching on it would be wrong/ambiguous.
+physical_nics() {
+    local path n mac
+    for path in /sys/class/net/*; do
+        n="${path##*/}"
+        [ "$n" = "lo" ] && continue
+        [ -e "${path}/device/driver" ] || continue          # real device only
+        mac="$(ethtool -P "$n" 2>/dev/null | awk '/Permanent address/{print $NF}')"
+        case "$mac" in ""|00:00:00:00:00:00) mac="$(cat "${path}/bonding_slave/perm_hwaddr" 2>/dev/null)" ;; esac
+        case "$mac" in ""|00:00:00:00:00:00) mac="$(cat "${path}/address" 2>/dev/null)" ;; esac
+        case "$mac" in ""|00:00:00:00:00:00) continue ;; esac
+        echo "$n $mac"
+    done
+}
+
+# Turn "<name> <mac>" lines on stdin into classic by-MAC persistent-net udev rules.
+render_net_rules() {
+    echo "# Generated by so-nic-pin: pin NIC names by MAC so kernel upgrades can't renumber them."
+    echo "# Security Onion binds its management/monitor interfaces by name; do not hand-edit."
+    local n mac
+    while read -r n mac; do
+        [ -n "$n" ] || continue
+        printf 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="%s", NAME="%s"\n' \
+            "$mac" "$n"
+    done
+}
+
+[ "$(id -u)" -eq 0 ] || exit 0                   # salt runs us as root; bail quietly otherwise
+[ -e "${MARKER}" ] && exit 0                      # run-once guard (mirrors the state's unless)
+
+nics="$(physical_nics)"
+if [ -z "${nics}" ]; then
+    log "no physical NICs detected — nothing to pin (will retry on next highstate)"
+    exit 0                                         # do NOT drop the marker; let it retry later
+fi
+
+log "pinning physical NICs by permanent MAC:"
+echo "${nics}" | sed 's/^/    /'
+
+[ -f "${NET_RULES_FILE}" ] && cp -f "${NET_RULES_FILE}" "${NET_RULES_FILE}.bak"
+echo "${nics}" | render_net_rules > "${NET_RULES_FILE}" || {
+    log "ERROR: failed to write ${NET_RULES_FILE}"
+    exit 1
+}
+
+mkdir -p "$(dirname "${MARKER}")" && touch "${MARKER}"
+log "wrote ${NET_RULES_FILE} ($(grep -c '^SUBSYSTEM' "${NET_RULES_FILE}") NIC(s) pinned); dropped ${MARKER}"

salt/common/tools/sbin/so-restart

@@ -5,27 +5,44 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-
-
-# Usage: so-restart  kibana | playbook 
-
 . /usr/sbin/so-common
 
-if [ $# -ge 1 ]; then
+usage() {
+    echo "Usage: $0 <component> [args]"
+    echo ""
+    echo "Supported args:"
+    echo "  --force | -f          Force stop all Salt jobs before starting component."
+    echo ""
+    echo "Examples:"
+    echo "  $0 kibana             Restart Kibana"
+    echo "  $0 kibana --force     Force stop all Salt jobs before restarting Kibana"
+    exit 1
+}
 
-        echo $banner
+if [[ $# -lt 1 ]]; then
-        printf "Restarting $1...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n"
+    usage
-        echo $banner
+fi
 
-        if [ "$2" = "--force" ]; then
+#shellcheck disable=SC2154
+echo "$banner"
+printf "Restarting %s...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n" "$1"
+echo "$banner"
+if [[ "$2" = "--force" ]] || [[ "$2" = "-f" ]]; then
     printf "\nForce-stopping all Salt jobs before proceeding\n\n"
     salt-call saltutil.kill_all_jobs
 fi
-
 case $1 in
-                "elastic-fleet") docker stop so-elastic-fleet && docker rm so-elastic-fleet && salt-call state.apply elasticfleet queue=True;;
+    "elastic-fleet"|"elasticfleet")
-                *)  docker stop so-$1 ; docker rm so-$1 ; salt-call state.apply $1 queue=True;;
+        docker_check_running "elastic-fleet" "--stop"
+        docker rm "so-elastic-fleet" 2> /dev/null
+        # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy
+        rm -rf /opt/so/conf/elastic-fleet/state
+
+        salt-call state.apply elasticfleet queue=True
+        ;;
+    *)
+        docker_check_running "$1" "--stop"
+        docker rm "so-${1}" 2> /dev/null
+        salt-call state.apply "$1" queue=True
+        ;;
 esac
-else
-        echo -e "\nPlease provide an argument by running like so-restart $component, or by using the component-specific script.\nEx. so-restart logstash, or so-logstash-restart\n"
-fi

salt/common/tools/sbin/so-start

@@ -5,27 +5,54 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-
+# shellcheck disable=SC1091
-
-# Usage: so-start  all | kibana | playbook 
-
 . /usr/sbin/so-common
 
-if [ $# -ge 1 ]; then
+usage() {
-	echo $banner
+    echo "Usage: $0 <component> [args]"
-	printf "Starting $1...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n"
+    echo ""
-	echo $banner
+    echo "Supported args:"
+    echo "  --force | -f          Force stop all Salt jobs before starting component."
+    echo ""
+    echo "Examples:"
+    echo "  $0 kibana             Start Kibana"
+    echo "  $0 kibana --force     Force stop all Salt jobs before starting Kibana"
+    exit 1
+}
 
-	if [ "$2" = "--force" ]; then
+if [[ $# -lt 1 ]]; then
+    usage
+fi
+
+#shellcheck disable=SC2154
+echo "$banner"
+printf "Starting %s...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n" "$1"
+echo "$banner"
+if [[ "$2" = "--force" ]] || [[ "$2" == "-f" ]]; then
 	printf "\nForce-stopping all Salt jobs before proceeding\n\n"
 	salt-call saltutil.kill_all_jobs
 fi
 
-	case $1 in
+case "$1" in
-   		"all") salt-call state.highstate queue=True;;
+    "all")
-   		"elastic-fleet") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply elasticfleet queue=True; fi ;; 
+        salt-call state.highstate queue=True
-   		*) if docker ps | grep -E -q '^so-$1$'; then printf "\n$1 is already running\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply $1 queue=True; fi ;; 
+        ;;
-	esac
+   	"elastic-fleet"|"elasticfleet")
+        if docker_check_running "elastic-fleet"; then
+            printf "\nso-%s is already running!\n\n" "elastic-fleet"
+            /usr/sbin/so-status
         else
-	echo -e "\nPlease provide an argument by running like so-start $component, or by using the component-specific script.\nEx. so-start logstash, or so-logstash-start\n"	
+            docker rm "so-elastic-fleet" 2> /dev/null
+            salt-call state.apply elasticfleet queue=True
         fi
+        ;;
+   	*)
+        if docker_check_running "$1"; then
+            printf "\nso-%s is already running\n\n" "$1"
+            /usr/sbin/so-status
+        else
+            docker rm "so-${1}" 2> /dev/null
+            salt-call state.apply "$1" queue=True
+        fi
+        ;;
+esac

salt/common/tools/sbin/so-stop

@@ -5,21 +5,35 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-
+# shellcheck disable=SC1091
-
-# Usage: so-stop kibana | playbook | thehive
-
 . /usr/sbin/so-common
 
-if [ $# -ge 1 ]; then
+usage() {
-	echo $banner
+    echo "Usage: $0 <component>"
-	printf "Stopping $1...\n"
+    echo ""
-	echo $banner
+    echo "Examples:"
+    echo "  $0 kibana             Stop Kibana"
+    exit 1
+}
 
-	case $1 in
+if [[ $# -lt 1 ]]; then
-    		*) docker stop so-$1 ; docker rm so-$1 ;;
+    usage
-	esac
-else
-	echo -e "\nPlease provide an argument by running like so-stop $component, or by using the component-specific script.\nEx. so-stop logstash, or so-logstash-stop\n"	
 fi
 
+
+#shellcheck disable=SC2154
+echo "$banner"
+printf "Stopping %s...\n" "$1"
+echo "$banner"
+case $1 in
+	"elasticfleet"|"elastic-fleet")
+        docker_check_running "elastic-fleet" "--stop"
+        docker rm "so-elastic-fleet" 2> /dev/null
+        # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy
+        rm -rf /opt/so/conf/elastic-fleet/state
+        ;;
+    *)
+        docker_check_running "$1" "--stop"
+        docker rm "so-${1}" 2> /dev/null
+    ;;
+esac

salt/common/tools/sbin_jinja/so-import-pcap

@@ -63,7 +63,8 @@ function status {
 function pcapinfo() {
   PCAP=$1
   ARGS=$2
-  docker run --rm -v "$PCAP:/input.pcap" --entrypoint capinfos {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} /input.pcap -ae $ARGS
+  docker run --rm -v "$PCAP:/input.pcap" --entrypoint capinfos {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} /input.pcap -ae $ARGS |\
+    sed 's/First packet/Earliest packet/g' | sed 's/Last packet/Latest packet/g'
 }
 
 function pcapfix() {

salt/elasticfleet/config.sls

@@ -173,7 +173,7 @@ eaoptionalintegrationsdir:
 
 {% for minion in node_data %}
 {% set role = node_data[minion]["role"] %}
-{% if role in [ "eval","fleet","heavynode","import","manager", "managerhype", "managersearch","standalone" ] %}
+{% if role in [ "eval","fleet","import","manager", "managerhype", "managersearch","standalone" ] %}
 {% set optional_integrations = ELASTICFLEETMERGED.optional_integrations %}
 {% set integration_keys = optional_integrations.keys() %}
 fleet_server_integrations_{{ minion }}:

salt/elasticfleet/defaults.yaml

@@ -1,6 +1,5 @@
 elasticfleet:
   enabled: False
-  patch_version: 9.3.3+build202604082258 # Elastic Agent specific patch release.
   enable_manager_output: True
   config:
     server:

salt/elasticfleet/enabled.sls

@@ -11,6 +11,10 @@
 
 {#   This value is generated during node install and stored in minion pillar #}
 {%   set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %}
+{#   Prevent Elastic Agent from re-enrolling with a new agent.id everytime the container starts up.
+       - if a fresh enrollment is needed use 'so-stop elasticfleet'
+       #}
+{%   set ENROLLED = salt['file.file_exists']('/opt/so/conf/elastic-fleet/state/fleet.enc') %}
 
 include:
   - ca
@@ -65,6 +69,7 @@ so-elastic-fleet:
       - /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro
       - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro
       - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro
+      - /opt/so/conf/elastic-fleet/state:/usr/share/elastic-agent/state
       - /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs 
      {% if DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
         {% for BIND in DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
@@ -72,6 +77,7 @@ so-elastic-fleet:
         {% endfor %}
       {% endif %}      
     - environment:
+      {% if not ENROLLED %}
       - FLEET_SERVER_ENABLE=true
       - FLEET_URL=https://{{ GLOBALS.hostname }}:8220
       - FLEET_SERVER_ELASTICSEARCH_HOST=https://{{ GLOBALS.manager }}:9200
@@ -81,6 +87,9 @@ so-elastic-fleet:
       - FLEET_SERVER_CERT_KEY=/etc/pki/elasticfleet-server.key
       - FLEET_CA=/etc/pki/tls/certs/intca.crt     
       - FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt
+      {% endif %}
+      - STATE_PATH=/usr/share/elastic-agent/state
+      - CONFIG_PATH=/usr/share/elastic-agent/state
       - LOGS_PATH=logs
       {% if DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
         {% for XTRAENV in DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
@@ -99,6 +108,7 @@ so-elastic-fleet:
       - x509: etc_elasticfleet_crt
     - require:
       - file: trusttheca
+      - file: eastatedir
       - x509: etc_elasticfleet_key
       - x509: etc_elasticfleet_crt
 

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -5,7 +5,7 @@
   "package": {
     "name": "endpoint",
     "title": "Elastic Defend",
-    "version": "9.3.0",
+    "version": "9.3.1",
     "requires_root": true
   },
   "enabled": true,

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -29,7 +29,7 @@
               "\\.gz$"
             ],
             "include_files": [],
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.15.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.15.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.15.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.20.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.3\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.20.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.20.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.3\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
             ],

salt/elasticfleet/install_agent_grid.sls

@@ -10,6 +10,15 @@
 {% set AGENT_STATUS = salt['service.available']('elastic-agent') %}
 {% set AGENT_EXISTS = salt['file.file_exists']('/opt/Elastic/Agent/elastic-agent') %}
 
+so-elastic-agent-install:
+  file.managed:
+    - name: /usr/sbin/so-elastic-agent-install
+    - source: salt://elasticfleet/tools/sbin/so-elastic-agent-install
+    - user: 947
+    - group: 939
+    - mode: 755
+    - show_changes: False
+
 {% if not AGENT_STATUS or not AGENT_EXISTS %}
 
 pull_agent_installer:
@@ -21,11 +30,9 @@ pull_agent_installer:
 
 run_installer:
   cmd.run:
-    - name: ./so-elastic-agent_linux_amd64 -token={{ GRIDNODETOKEN }} -force
+    - name: /usr/sbin/so-elastic-agent-install "{{ GRIDNODETOKEN }}"
-    - cwd: /opt/so
+    - require:
-    - retry:
+      - file: pull_agent_installer
-        attempts: 3
-        interval: 20
 
 cleanup_agent_installer:
   file.absent:

salt/elasticfleet/manager.sls

@@ -67,8 +67,6 @@ so-elastic-fleet-package-upgrade:
         interval: 30
     - require:
       - http: wait_for_so-kibana
-    - onchanges:
-      - file: /opt/so/state/elastic_fleet_packages.txt
 
 so-elastic-fleet-integrations:
   cmd.run:

salt/elasticfleet/tools/sbin/so-elastic-agent-install

@@ -0,0 +1,100 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-elastic-fleet-common
+
+
+# passed in as arg from elasticfleet/install_agent_grid.sls, else pulled from pillar later
+GRIDNODETOKEN="$1"
+LOGFILE="/opt/so/SO-Elastic-Agent_Installer_Health.log"
+
+check_agent_health() {
+    timeout=300
+    interval=10
+    start=$SECONDS
+
+    while (( SECONDS - start < timeout )); do
+        agent_status=$(elastic-agent status 2>&1)
+        echo -e "\n$(date)\n$agent_status\n" >> "$LOGFILE"
+        if echo "$agent_status" | grep -A1 'elastic-agent$' | grep -q 'status: (HEALTHY)'; then
+            return 0
+        fi
+        echo "The Elastic Agent is not yet healthy. Waiting for ${interval} seconds before checking again..."
+        sleep "$interval"
+    done
+
+    echo "The Elastic Agent did not become healthy within ${timeout} seconds"
+    return 1
+}
+
+uninstall_agent() {
+    if command -v elastic-agent >/dev/null 2>&1; then
+        elastic-agent uninstall -f
+    fi
+}
+
+if [[ -z "$GRIDNODETOKEN" ]]; then
+    noderole=$(so-yaml.py get -r /etc/salt/grains role)
+    if [[ "$noderole" == "so-heavynode" ]]; then
+        GRIDNODETOKEN=$(salt-call pillar.get global:fleet_grid_enrollment_token_heavy --out=newline_values_only)
+    else
+        GRIDNODETOKEN=$(salt-call pillar.get global:fleet_grid_enrollment_token_general --out=newline_values_only)
+    fi
+fi
+
+if [[ -z "$GRIDNODETOKEN" ]]; then
+    echo "Unable to determine Elastic Fleet enrollment token. Exiting."
+
+    exit 1
+fi
+
+if [[ ! -x /opt/so/so-elastic-agent_linux_amd64 ]]; then
+    echo "Downloading so-elastic-agent installer... This could take a while if another Salt job is running."
+
+    # When running outside of elasticfleet/install_agent_grid.sls we need to download the installer independently.
+    #  PYTHONWARNINGS="ignore" to avoid messages like the following when running salt-call:
+    #  '/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/base.py:129: TransportWarning: Unclosed transport! <salt.transport.zeromq.RequestClient object at 0x7fc5f0ee7a30>
+    #    File "/bin/salt-call", line 12, in <module>
+    #      sys.exit(salt_call())'
+
+    PYTHONWARNINGS="ignore" salt-call state.single file.managed name=/opt/so/so-elastic-agent_linux_amd64 source=salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64 mode=755 makedirs=True queue=True
+
+fi
+
+if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then
+    attempts=0
+    cd /opt/so/ || exit 1
+
+    truncate -s 0 "$LOGFILE"
+
+    uninstall_agent
+
+    while [[ $attempts -lt 3 ]]; do
+        if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force && echo "Verifying Elastic Agent health..." && check_agent_health; then
+            rm -f /opt/so/so-elastic-agent_linux_amd64
+            elastic-agent status
+
+            exit 0
+        fi
+
+        attempts=$((attempts + 1))
+
+        if [[ $attempts -lt 3 ]]; then
+            echo "Unable to verify Elastic Agent health... Retrying in 20 seconds..."
+            sleep 20
+        fi
+    done
+
+    uninstall_agent
+    rm -f /opt/so/so-elastic-agent_linux_amd64
+    echo "The so-elastic-agent installer failed after 3 attempts. Exiting."
+
+    exit 1
+else
+    echo "Unable to locate so-elastic-agent installer. Exiting."
+
+    exit 1
+fi

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -9,13 +9,11 @@
 RETURN_CODE=0
 
 if [ ! -f /opt/so/state/eaintegrations.txt ]; then
-  # First, check for any package upgrades
-  /usr/sbin/so-elastic-fleet-package-upgrade
 
-  # Second, update Fleet Server policies
+  # update Fleet Server policies
   /usr/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server
 
-  # Third, configure Elastic Defend Integration seperately
+  # configure Elastic Defend Integration separately
   /usr/sbin/so-elastic-fleet-integration-policy-elastic-defend
 
   # Each group fetches its agent policy once and dispatches create/update writes concurrently.
@@ -32,9 +30,12 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
   elastic_fleet_load_integrations_dir "so-grid-nodes_heavy" \
     /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy "Grid Nodes Policy_Heavy" || RETURN_CODE=1
 
-  # Fleet Server - Optional integrations (one agent policy per FleetServer_* directory)
+  # Fleet Server - Optional integrations (adds integration configuration to a given FleetServer_ policy)
   for FLEET_DIR in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/; do
     [ -d "$FLEET_DIR" ] || continue
+    INTEGRATIONS=("${FLEET_DIR%/}"/*.json)
+    [ -e "${INTEGRATIONS[0]}" ] || continue
+
     FLEET_POLICY=$(basename "$FLEET_DIR")
     elastic_fleet_load_integrations_dir "$FLEET_POLICY" \
       "${FLEET_DIR%/}" "Fleet Server Policy" "elasticsearch-logs" || RETURN_CODE=1

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers

@@ -30,7 +30,7 @@ done
 if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then 
   printf "\nFleet Host URL, Enrollment Token or Elastic Version empty - exiting..."
   printf "\nFleet Host: $FLEETHOST, Enrollment Token: $ENROLLMENTOKEN\n"
-  exit 
+  exit 1
 fi
 
 OSARCH=( "linux-x86_64" "windows-x86_64" "darwin-x86_64" "darwin-aarch64" )
@@ -62,31 +62,54 @@ do
 done 
 
 GOTARGETOS=( "linux" "windows" "darwin" "darwin/arm64" )
-GOARCH="amd64"
 printf "\n### Generating OS packages using the cleaned up tarballs"
-for GOOS in "${GOTARGETOS[@]}"
+for GOOS in "${GOTARGETOS[@]}"; do
-do
+    GOARCH="amd64"
     if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi
     printf "\n\n### Generating $GOOS/$GOARCH Installer...\n"
     docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \
     --mount type=bind,source=/etc/pki/tls/certs/,target=/workspace/files/cert/ \
     --mount type=bind,source=/nsm/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \
-    --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \
+    --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ \
     {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN"  -o /output/so-elastic-agent_${GOOS}_${GOARCH}
     printf "\n### $GOOS/$GOARCH Installer Generated...\n"
 done
 
 printf "\n\n### Generating MSI...\n"
-cp /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe
+cp /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe
 docker run \
---mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ -w /output \
+--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ -w /output \
 {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs
 printf "\n### MSI Generated...\n"
 
+# Verify installers were created
+for GOOS in "${GOTARGETOS[@]}"; do
+    GOARCH="amd64"
+    if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin"; GOARCH="arm64"; fi
+    if [[ ! -f /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_${GOOS}_${GOARCH} ]]; then
+        printf "\n### ERROR: Installer for %s/%s was not generated. Exiting...\n" "$GOOS" "$GOARCH"
+        exit 1
+    fi
+    # After verifying new installer was generated, move it to so_agent-installers directory
+    mv /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_${GOOS}_${GOARCH} /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/
+done
+
+# Verify MSI installer
+if [[ ! -f /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64_msi ]]; then
+    printf "\n### ERROR: Installer MSI was not generated. Exiting...\n"
+    exit 1
+else
+    # After verifying new installer MSI was generated, move it to so_agent-installers directory
+    mv /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64_msi /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/
+fi
+
 printf "\n### Cleaning up temp files \n"
 rm -rf /nsm/elastic-agent-workspace
-rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe
+rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe
 
 printf "\n### Copying so_agent-installers to /nsm/elastic-fleet/ for nginx.\n"
 \cp -vr /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ /nsm/elastic-fleet/
 chmod 644 /nsm/elastic-fleet/so_agent-installers/*
+
+# if we got here all installers have been generated successfully
+exit 0

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade

@@ -12,17 +12,22 @@ PKG_LOAD_FAILURES=0
 PKG_LOAD_FAILURES_NAMES=()
 
 {%- for PACKAGE in SUPPORTED_PACKAGES %}
-echo "Upgrading {{ PACKAGE }} package..."
+if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && LATEST_VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
-if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
+
-    if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
+    if [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then
+        echo "{{ PACKAGE }} integration version $INSTALLED_VERSION is already at the reported latest version $LATEST_VERSION, skipping upgrade."
+    else
+        echo "Upgrading {{ PACKAGE }} package to version $LATEST_VERSION..."
+        if ! elastic_fleet_package_install "{{ PACKAGE }}" "$LATEST_VERSION"; then
             PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
             PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
         fi
+    fi
 else
+    echo "ERROR: Failed to get version information for integration {{ PACKAGE }}"
     PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
     PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
 fi
-echo
 {%- endfor %}
 
 if [ $PKG_LOAD_FAILURES -gt 0 ]; then
@@ -35,6 +40,3 @@ if [ $PKG_LOAD_FAILURES -gt 0 ]; then
 else
     echo "Successfully upgraded all packages."
 fi
-
-echo
-/usr/sbin/so-elasticsearch-templates-load

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -181,6 +181,9 @@ if ! elastic_fleet_policy_create "so-grid-nodes_heavy" "SO Grid Nodes - Heavy No
     exit 1
 fi
 
+# Check for package upgrades
+so-elastic-fleet-package-upgrade
+
 # Load Integrations for default policies
 so-elastic-fleet-integration-policy-load
 
@@ -241,11 +244,37 @@ printf '%s\n'\
     "" >> "$global_pillar_file"
 
 # Call Elastic-Fleet Salt State
-printf "\nApplying elasticfleet state"
+printf "\nApplying elasticfleet state\n"
-salt-call state.apply elasticfleet queue=True
+for state_attempt in {1..3}; do
+    if salt-call state.apply elasticfleet queue=True; then
+        break
+    elif [[ $state_attempt -lt 3 ]]; then
+        printf "\nElasticfleet state did not complete successfully... Attempt (%s/3). Retrying...\n" "$state_attempt"
+        sleep 10
+    else
+        printf "\nFailure(s) in elasticfleet state... Exiting...\n"
+        exit 1
+    fi
+done
 
+printf "\nRunning so-elastic-agent-gen-installers\n"
 # Generate installers & install Elastic Agent on the node
-so-elastic-agent-gen-installers
+for agent_gen_attempt in {1..3}; do
-printf "\nApplying elasticfleet.install_agent_grid state"
+    if so-elastic-agent-gen-installers; then
-salt-call state.apply elasticfleet.install_agent_grid queue=True
+        break
-exit 0
+    elif [[ $agent_gen_attempt -lt 3 ]]; then
+        printf "\nUnable to generate Elastic Agent installers... Attempt (%s/3). Retrying...\n" "$agent_gen_attempt"
+        sleep 10
+    else
+        printf "\nFailed to generate Elastic Agent installers after 3 attempts. Exiting...\n"
+        exit 1
+    fi
+done
+
+printf "\nApplying elasticfleet.install_agent_grid state\n"
+if ! salt-call state.apply elasticfleet.install_agent_grid queue=True; then
+    printf "\nFailure(s) in elasticfleet.install_agent_grid state... Exiting...\n"
+    exit 1
+fi
+
+printf "\nElastic Fleet setup completed successfully\n"

salt/elasticsearch/defaults.yaml

@@ -1,6 +1,6 @@
 elasticsearch:
   enabled: false
-  version: 9.3.3
+  version: 9.3.7
   index_clean: true
   data_retention_method: DLM
   vm:

salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.4

@@ -118,70 +118,70 @@
     {
       "pipeline": {
         "tag": "pipeline_e16851a7",
-        "name": "logs-pfsense.log-1.25.2-firewall",
+        "name": "logs-pfsense.log-1.25.4-firewall",
         "if": "ctx.event.provider == 'filterlog'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_828590b5",
-        "name": "logs-pfsense.log-1.25.2-openvpn",
+        "name": "logs-pfsense.log-1.25.4-openvpn",
         "if": "ctx.event.provider == 'openvpn'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_9d37039c",
-        "name": "logs-pfsense.log-1.25.2-ipsec",
+        "name": "logs-pfsense.log-1.25.4-ipsec",
         "if": "ctx.event.provider == 'charon'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_ad56bbca",
-        "name": "logs-pfsense.log-1.25.2-dhcp",
+        "name": "logs-pfsense.log-1.25.4-dhcp",
         "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\", \"dnsmasq-dhcp\"].contains(ctx.event.provider)"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_dd85553d",
-        "name": "logs-pfsense.log-1.25.2-unbound",
+        "name": "logs-pfsense.log-1.25.4-unbound",
         "if": "ctx.event.provider == 'unbound'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_720ed255",
-        "name": "logs-pfsense.log-1.25.2-haproxy",
+        "name": "logs-pfsense.log-1.25.4-haproxy",
         "if": "ctx.event.provider == 'haproxy'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_456beba5",
-        "name": "logs-pfsense.log-1.25.2-php-fpm",
+        "name": "logs-pfsense.log-1.25.4-php-fpm",
         "if": "ctx.event.provider == 'php-fpm'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_a0d89375",
-        "name": "logs-pfsense.log-1.25.2-squid",
+        "name": "logs-pfsense.log-1.25.4-squid",
         "if": "ctx.event.provider == 'squid'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_c2f1ed55",
-        "name": "logs-pfsense.log-1.25.2-snort",
+        "name": "logs-pfsense.log-1.25.4-snort",
         "if": "ctx.event.provider == 'snort'"
       }
     },
     {
       "pipeline": {
         "tag":"pipeline_33db1c9e",
-        "name": "logs-pfsense.log-1.25.2-suricata",
+        "name": "logs-pfsense.log-1.25.4-suricata",
         "if": "ctx.event.provider == 'suricata'"
       }
     },

salt/elasticsearch/files/ingest/zeek.ssl

@@ -5,6 +5,7 @@
     { "remove":   { "field": ["host"],     "ignore_failure": true } },
     { "json":             { "field": "message",                 "target_field": "message2",             "ignore_failure": true  } },
     { "rename":         { "field": "message2.version",          "target_field": "ssl.version",          "ignore_missing": true  } },
+    { "set":      { "description": "Set transport for the community_id processor", "if": "ctx.ssl?.version == null || !ctx.ssl.version.startsWith('DTLS')", "field": "network.transport", "value": "tcp", "ignore_failure": true } },
     { "rename":         { "field": "message2.cipher",           "target_field": "ssl.cipher",           "ignore_missing": true  } },
     { "rename":         { "field": "message2.curve",            "target_field": "ssl.curve",            "ignore_missing": true  } },
     { "rename":         { "field": "message2.server_name",      "target_field": "ssl.server_name",              "ignore_missing": true  } },

salt/elasticsearch/soc_elasticsearch.yaml

@@ -645,6 +645,7 @@ elasticsearch:
             global: True
             advanced: True
             helpLink: elasticsearch
+    so-logs-soc: *dataStreamSettings
     so-logs-system_x_auth: *dataStreamSettings
     so-logs-system_x_syslog: *dataStreamSettings
     so-logs-system_x_system: *dataStreamSettings

salt/kibana/defaults.yaml

@@ -22,7 +22,7 @@ kibana:
           - default
           - file
     migrations:
-      discardCorruptObjects: "9.3.3"
+      discardCorruptObjects: "9.3.7"
     telemetry:
       enabled: False
     xpack:

salt/kibana/enabled.sls

@@ -69,7 +69,7 @@ wait_for_so-kibana:
     - ssl: True
     - verify_ssl: False
     - status: 200
-    - wait_for: 300
+    - wait_for: 600
     - request_interval: 15
     - require:
       - docker_container: so-kibana

salt/manager/files/mirror-kernel.txt

@@ -0,0 +1,2 @@
+https://repo.securityonion.net/file/so-repo/prod/3/oracle/9-uek8
+https://repo-alt.securityonion.net/prod/3/oracle/9-uek8

salt/manager/files/repodownload.conf

@@ -11,3 +11,8 @@ name=Security Onion Repo repo
 mirrorlist=file:///opt/so/conf/reposync/mirror.txt
 enabled=1
 gpgcheck=1
+[securityonionkernelsync]
+name=Security Onion Kernel Repo repo
+mirrorlist=file:///opt/so/conf/reposync/mirror-kernel.txt
+enabled=1
+gpgcheck=1

salt/manager/init.sls

@@ -86,6 +86,28 @@ repo_dir:
       - group
     - show_changes: False
 
+kernelrepo_dir:
+  file.directory:
+    - name: /nsm/kernelrepo
+    - user: socore
+    - group: socore
+    - recurse:
+      - user
+      - group
+    - show_changes: False
+
+# Ensure /nsm/kernelrepo is always a valid (if empty) repo before it is ever assigned to
+# a client. Without repodata/repomd.xml an enabled file:///nsm/kernelrepo repo makes every
+# dnf operation fail; so-repo-sync only populates it after the highstate, so seed an empty
+# repo here. Only runs when repodata is missing, so it won't clobber a synced repo.
+kernelrepo_init_empty:
+  cmd.run:
+    - name: createrepo /nsm/kernelrepo
+    - unless: 'test -e /nsm/kernelrepo/repodata/repomd.xml'
+    - require:
+      - file: kernelrepo_dir
+      - pkg: install_createrepo
+
 manager_sbin:
   file.recurse:
     - name: /usr/sbin
@@ -122,6 +144,13 @@ so-repo-mirrorlist:
     - user: socore
     - group: socore
 
+so-repo-kernel-mirrorlist:
+  file.managed:
+    - name: /opt/so/conf/reposync/mirror-kernel.txt
+    - source: salt://manager/files/mirror-kernel.txt
+    - user: socore
+    - group: socore
+
 so-repo-sync:
   {%     if MANAGERMERGED.reposync.enabled %}
   cron.present:

salt/manager/tools/sbin/so-repo-sync

@@ -10,5 +10,16 @@ NOROOT=1
 set -e
 
 curl --retry 5 --retry-delay 60 -A "reposync/$(sync_options)" https://sigs.securityonion.net/checkup --output /tmp/checkup
+
 dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/
 createrepo /nsm/repo
+
+# The kernel repo section is deployed to repodownload.conf by the manager highstate, which
+# runs AFTER this script during soup. On the first upgrade to a kernel-aware version the
+# on-disk config still predates the section, so guard on its presence to avoid dnf's
+# "Unknown repo: 'securityonionkernelsync'" aborting the sync (set -e). The next sync after the
+# highstate deploys the section will pick it up.
+if grep -q '^\[securityonionkernelsync\]' /opt/so/conf/reposync/repodownload.conf; then
+  dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernelsync --download-metadata -p /nsm/kernelrepo/
+  createrepo /nsm/kernelrepo
+fi

salt/manager/tools/sbin/soup

@@ -12,7 +12,17 @@
 UPDATE_DIR=/tmp/sogh/securityonion
 DEFAULT_SALT_DIR=/opt/so/saltstack/default
 INSTALLEDVERSION=$(cat /etc/soversion)
+# /etc/sopostversion is a soup-owned marker (no salt state manages it) tracking how
+# far the post-upgrade walk has progressed. Its presence means a prior upgrade did
+# not finish its post-upgrade steps; its contents are the resume point. It is read
+# here before preupgrade_changes mutates INSTALLEDVERSION and before any highstate
+# stamps /etc/soversion from the pillar.
+POSTVERSION_FILE=/etc/sopostversion
+if [ -f "$POSTVERSION_FILE" ]; then
+  POSTVERSION=$(cat "$POSTVERSION_FILE")
+else
   POSTVERSION=$INSTALLEDVERSION
+fi
 INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
 BATCHSIZE=5
 SOUP_LOG=/root/soup.log
@@ -23,6 +33,10 @@ NOTIFYCUSTOMELASTICCONFIG=false
 TOPFILE=/opt/so/saltstack/default/salt/top.sls
 BACKUPTOPFILE=/opt/so/saltstack/default/salt/top.sls.backup
 SALTUPGRADED=false
+# Set true once soup begins modifying the system (past the pre-flight checks), so the
+# EXIT trap can tell the user the update did not finish and must be re-run. Only the
+# pre-flight gates (ES compatibility, disk, network) fail before this is set.
+SOUP_UPGRADE_STARTED=false
 SALT_CLOUD_INSTALLED=false
 SALT_CLOUD_CONFIGURED=false
 # Check if salt-cloud is installed
@@ -123,6 +137,28 @@ check_err() {
 
     echo "SOUP XTRACE debug log (if enabled) at $SOUP_DEBUG_LOG. Re-run soup with SOUP_DEBUG=1 to create $SOUP_DEBUG_LOG"
 
+    # If soup had already started modifying the system, make it unmistakable that the
+    # update is incomplete and must be re-run. soup is resumable: a version upgrade
+    # picks up from the /etc/sopostversion marker, and a hotfix re-applies because
+    # /etc/sohotfix is only advanced after a successful highstate.
+    if [[ "$SOUP_UPGRADE_STARTED" == "true" ]]; then
+      echo ""
+      echo "=============================================================================="
+      echo " UPGRADE INCOMPLETE"
+      echo "=============================================================================="
+      echo " This soup run did NOT finish. Your Security Onion installation may be in a"
+      echo " partially-updated state and is not yet fully upgraded."
+      echo ""
+      echo " Review the error above and $SOUP_LOG, resolve the underlying problem, then"
+      echo " run soup again to resume and complete the update:"
+      echo ""
+      echo "     sudo soup"
+      echo ""
+      echo " soup is resumable -- re-running it continues from where this run stopped."
+      echo "=============================================================================="
+      echo ""
+    fi
+
     exit $exit_code
   fi
 
@@ -131,6 +167,8 @@ check_err() {
 # Collect bash error context before passing off to check_err()
 on_err() {
   local exit_code=$?
+  # Ignore failures in blocks that explicitly disabled errexit with `set +e`.
+  [[ $- == *e* ]] || return $exit_code
   # turn off xtrace to prevent added noise in debug log
   set +x 2>/dev/null || true
 
@@ -243,6 +281,7 @@ check_airgap() {
       UPDATE_DIR=/tmp/soagupdate/SecurityOnion
       AGDOCKER=/tmp/soagupdate/docker
       AGREPO=/tmp/soagupdate/minimal/Packages
+      AGUEKREPO=/tmp/soagupdate/uek/Packages
   else
       is_airgap=1
   fi
@@ -288,6 +327,30 @@ check_pillar_items() {
   fi
 }
 
+check_cluster_health() {
+  echo "Checking Elasticsearch cluster health."
+  # Require a 'green' cluster before upgrading; anything less (yellow, red, or
+  # unreachable) blocks. Modeled on the wait used in so-elasticsearch-roles-load.
+  if so-elasticsearch-query "_cluster/health?wait_for_status=green&timeout=120s" --fail > /dev/null 2>&1; then
+    printf "\nThe Elasticsearch cluster is healthy (green). We can proceed with SOUP.\n\n"
+  else
+    printf "\nThe Elasticsearch cluster is not green. Please resolve the cluster health issue so the cluster is green before running SOUP again.\n\n"
+    exit 0
+  fi
+}
+
+check_fleet_server() {
+  echo "Checking that Elastic Fleet Server is responding."
+  # Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls,
+  # which waits for HTTP 200 from the Fleet Server status API.
+  if curl -sk --fail --retry 3 --retry-delay 10 --max-time 30 "https://localhost:8220/api/status" > /dev/null 2>&1; then
+    printf "\nElastic Fleet Server is responding. We can proceed with SOUP.\n\n"
+  else
+    printf "\nElastic Fleet Server is not responding at https://localhost:8220/api/status. Please ensure Elastic Fleet is healthy before running SOUP again.\n\n"
+    exit 0
+  fi
+}
+
 check_saltmaster_status() {
   set +e
   echo "Waiting on the Salt Master service to be ready."
@@ -374,6 +437,8 @@ get_soup_script_hashes() {
   GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}')
   CURRENTSOFIREWALL=$(md5sum /usr/sbin/so-firewall | awk '{print $1}')
   GITSOFIREWALL=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-firewall | awk '{print $1}')
+  CURRENTSOYAML=$(md5sum /usr/sbin/so-yaml.py | awk '{print $1}')
+  GITSOYAML=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py | awk '{print $1}')
 }
 
 highstate() {
@@ -411,6 +476,13 @@ preupgrade_changes() {
     true
 }
 
+set_postversion() {
+    # Persist post-upgrade walk progress so an interrupted upgrade can resume the
+    # remaining steps on the next soup run (see /etc/sopostversion handling).
+    POSTVERSION="$1"
+    echo "$POSTVERSION" > "$POSTVERSION_FILE"
+}
+
 postupgrade_changes() {
     # This function is to add any new pillar items if needed.
     echo "Running post upgrade processes."
@@ -418,6 +490,8 @@ postupgrade_changes() {
     [[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0
     [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0
     [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0
+    # All applicable post-upgrade steps completed; clear the resume marker.
+    rm -f "$POSTVERSION_FILE"
     true
 }
 
@@ -510,7 +584,7 @@ post_to_3.0.0() {
   # convert yes/no in suricata pillars to true/false
   convert_suricata_yes_no
 
-  POSTVERSION=3.0.0
+  set_postversion 3.0.0
 }
 
 ### 3.0.0 End ###
@@ -737,7 +811,6 @@ fix_logstash_0013_lumberjack_pipeline_name() {
 up_to_3.1.0() {
   ensure_postgres_local_pillar
   ensure_postgres_secret
-  determine_elastic_agent_upgrade
   elasticsearch_backup_index_templates
   # Clear existing component template state file.
   rm -f /opt/so/state/esfleet_component_templates.json
@@ -774,19 +847,30 @@ post_to_3.1.0() {
   # Check for unhealthy / unauthorized integration transform jobs and attempt reauthorizations
   check_transform_health_and_reauthorize || true
 
-  POSTVERSION=3.1.0
+  set_postversion 3.1.0
 }
 
 ### 3.1.0 End ###
 
 ### 3.2.0 Scripts ###
 
+recollate_postgres() {
+  echo ""
+  echo "Recollating PostgreSQL databases. The following output may contain warnings about a version mismatch, followed by a note indicating that the collation version has been changed."
+  for db in postgres securityonion so_telegraf; do
+    docker exec so-postgres psql -U postgres $db -c "reindex database $db"
+    docker exec so-postgres psql -U postgres $db -c "alter database $db refresh collation version"
+  done
+  echo "Recollating PostgreSQL databases complete."
+  echo ""
+}
+
 bootstrap_so_soc_database() {
   # init-db.sh is mounted into so-postgres at /docker-entrypoint-initdb.d/init-db.sh
   # and runs automatically only on a fresh data directory. Hosts upgrading from
   # 3.1.0 already have /nsm/postgres populated, so the so_soc bootstrap block
   # added in 3.2 never fires. Re-run the script explicitly; it's idempotent.
-  echo "Bootstrapping so_soc database via init-db.sh."
+  echo "Bootstrapping database via init-db.sh."
   # The postgres image has no USER directive, so `docker exec` defaults to
   # root, and the container env intentionally omits POSTGRES_USER (the upstream
   # entrypoint defaults it transiently during first-init only). Recreate both
@@ -797,10 +881,13 @@ bootstrap_so_soc_database() {
     return 0
   fi
   if ! $exec_cmd; then
-    FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd")
+    FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the database may not have been bootstrapped. Re-run manually: $exec_cmd")
     return 0
   fi
-  echo "so_soc bootstrap complete."
+  echo "Database bootstrap complete."
+
+  echo "Restarting so-soc container to pick up database changes"
+  docker restart so-soc
 }
 
 # Existing grids should keep ILM unless an admin explicitly opts in to DLM.
@@ -848,7 +935,32 @@ kibana_backport_streams_index_template() {
 
 }
 
+# Runs kafka-features.sh upgrade --release-version $1
+#   Upgrades Kafka KRaft cluster metadata
+update_kafka_metadata() {
+    metadata_version="$1"
+    global_pillar="/opt/so/saltstack/local/pillar/global/soc_global.sls"
+    if PIPELINE=$(so-yaml.py get -r "$global_pillar" global.pipeline 2> /dev/null) && [[ "$PIPELINE" == "KAFKA" ]]; then
+        kafka_nodes_raw=$(salt-call pillar.get kafka:nodes --out=json)
+        if kafka_nodes=$(jq -er '.local | select(type == "object" and length > 0)' <<< "$kafka_nodes_raw"); then
+            bootstrap_servers=$(jq -r '[to_entries[] | select(.value.role | contains("broker")) | "\(.value.ip):9092"] | join(",")' <<< "$kafka_nodes")
+            echo "Upgrading Kafka KRaft cluster version"
+            so-kafka-cli kafka-features.sh --bootstrap-server "$bootstrap_servers" --command-config /opt/kafka/config/kraft/client.properties upgrade --release-version "$metadata_version" 2>/dev/null || true
+
+            return 0
+        else
+            FINAL_MESSAGE_QUEUE+=("WARNING: Unable to automatically perform Kafka KRaft cluster metadata update. This step can be performed manually using the following command (replacing \$BROKER_IP with the ip of atleast 1 available Kafka broker):")
+            FINAL_MESSAGE_QUEUE+=("    - so-kafka-cli kafka-features.sh --bootstrap-server \$BROKER_IP:9092 --command-config /opt/kafka/config/kraft/client.properties upgrade --release-version $metadata_version")
+        fi
+    else
+        echo "Nothing to do!"
+    fi
+}
+
 up_to_3.2.0() {
+  # download 9.3.7 elastic agent packages
+  determine_elastic_agent_upgrade
+
   fix_logstash_0013_lumberjack_pipeline_name
 
   pin_elasticsearch_data_retention_method
@@ -857,15 +969,20 @@ up_to_3.2.0() {
 }
 
 post_to_3.2.0() {
+  # Recollate due to image OS rebase
+  recollate_postgres
+
   bootstrap_so_soc_database
 
-  # Including agent regen script here since it was missed in post_to_3.1.0
+  # Generate 9.3.7 elastic agent installers
   echo "Regenerating Elastic Agent Installers"
   /sbin/so-elastic-agent-gen-installers
 
   kibana_backport_streams_index_template
 
-  POSTVERSION=3.2.0
+  update_kafka_metadata "4.3"
+
+  set_postversion 3.2.0
 }
 
 ### 3.2.0 End ###
@@ -978,13 +1095,19 @@ update_airgap_rules() {
   rsync -a $UPDATE_DIR/agrules/securityonion-resources/* /nsm/securityonion-resources/
 }
 
-update_airgap_repo() {
+update_airgap_repos() {
   # Update the files in the repo
-  echo "Syncing new updates to /nsm/repo"
+  echo "Syncing new updates to /nsm/repo & /nsm/kernelrepo"
-  rsync -a $AGREPO/* /nsm/repo/
+  # Airgap soup copies new files into the local repo, but doesn't remove old packages. Retaining the ability to rollback package updates
-  echo "Creating repo"
+  rsync -a "$AGREPO"/ /nsm/repo/
+  rsync -a "$AGUEKREPO"/ /nsm/kernelrepo/
+
   dnf -y install yum-utils createrepo_c
+
+  echo "Running createrepo for /nsm/repo"
   createrepo /nsm/repo
+  echo "Running createrepo for /nsm/kernelrepo"
+  createrepo /nsm/kernelrepo
 }
 
 update_salt_mine() {
@@ -1011,8 +1134,20 @@ upgrade_check() {
   fi
   [[ -f /etc/sohotfix ]] && CURRENTHOTFIX=$(cat /etc/sohotfix)
   if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then
+    # A leftover post-version marker means a previous upgrade to this version
+    # advanced /etc/soversion (the highstate stamps it from the pillar) but did not
+    # finish its post-upgrade steps. Resume the upgrade instead of reporting "latest".
+    if [ -f "$POSTVERSION_FILE" ] && [ "$(cat "$POSTVERSION_FILE")" != "$NEWVERSION" ]; then
+      echo "A previous upgrade to $NEWVERSION did not complete its post-upgrade steps; resuming."
+      is_hotfix=false
+      return 0
+    fi
     echo "Checking to see if there are hotfixes needed"
     if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then
+      # Reaching here means we are at the target version and NOT resuming (the resume
+      # check above returned otherwise). Clear any stale resume marker so a completed
+      # upgrade is never mistaken for a partial one and re-run on a later invocation.
+      rm -f "$POSTVERSION_FILE"
       echo "You are already running the latest version of Security Onion."
       exit 0
     else
@@ -1091,7 +1226,7 @@ upgrade_salt() {
 
 verify_latest_update_script() {
   get_soup_script_hashes
-  if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then
+  if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" && "$CURRENTSOYAML" == "$GITSOYAML" ]]; then
     echo "This version of the soup script is up to date. Proceeding."
   else
     echo "You are not running the latest soup version. Updating soup and its components. This might take multiple runs to complete."
@@ -1100,7 +1235,7 @@ verify_latest_update_script() {
 
     # Verify that soup scripts updated as expected
     get_soup_script_hashes
-    if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then
+    if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" && "$CURRENTSOYAML" == "$GITSOYAML" ]]; then
       echo "Succesfully updated soup scripts."
     else
       echo "There was a problem updating soup scripts. Trying to rerun script update."
@@ -1124,7 +1259,8 @@ verify_es_version_compatibility() {
         ["8.18.4"]="8.18.6 8.18.8 9.0.8"
 	    ["8.18.6"]="8.18.8 9.0.8"
 	    ["8.18.8"]="9.0.8"
-        ["9.0.8"]="9.3.3"
+        ["9.0.8"]="9.3.3 9.3.7"
+        ["9.3.3"]="9.3.7"
     )
 
     # Elasticsearch MUST upgrade through these versions
@@ -1707,6 +1843,15 @@ main() {
   set_minionid
   MINION_ROLE=$(lookup_role)
   echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
+  # /etc/soversion is stamped to the target version before the upgrade fully
+  # completes, so a lingering resume marker means this grid is only partially
+  # upgraded even though the line above shows the target version. Make that explicit
+  # so it is not mistaken for a finished upgrade.
+  if [ -f "$POSTVERSION_FILE" ] && [ "$(cat "$POSTVERSION_FILE")" != "$INSTALLEDVERSION" ]; then
+    echo ""
+    echo "NOTE: A previous upgrade to $INSTALLEDVERSION did not finish. This grid is"
+    echo "      partially upgraded and this soup run will resume and complete it."
+  fi
   echo ""
   check_minimum_version
 
@@ -1735,12 +1880,18 @@ main() {
   echo "Verifying Elasticsearch version compatibility across the grid before upgrading."
   verify_es_version_compatibility
 
+  # Pre-flight health checks: confirm the grid is in a good state before we change
+  # anything. These run before any modifications, so a failure exits cleanly and the
+  # operator can fix the issue and re-run soup.
+  check_cluster_health
+  check_fleet_server
+
   echo "Checking for Salt Master and Minion updates."
   upgrade_check_salt
   set -e
 
   if [[ $is_airgap -eq 0 ]]; then
-    update_airgap_repo
+    update_airgap_repos
     dnf clean all
     check_os_updates
   elif [[ $OS == 'oracle' ]]; then
@@ -1751,6 +1902,7 @@ main() {
   fi
 
   if [ "$is_hotfix" == "true" ]; then
+    SOUP_UPGRADE_STARTED=true
     echo "Applying $HOTFIXVERSION hotfix"
     # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars
     if [[ ! "$MINION_ROLE" == "import" ]]; then
@@ -1761,10 +1913,16 @@ main() {
     create_local_directories "/opt/so/saltstack/default"
     apply_hotfix
     echo "Hotfix applied"
-    update_version
     enable_highstate
     highstate
+    # Record the hotfix only after the highstate succeeds. /etc/sohotfix is written
+    # solely by soup (no salt state manages it), so deferring the write means a failed
+    # hotfix highstate leaves the old hotfix value and re-running soup re-applies it,
+    # rather than reporting "already latest". The soversion/pillar writes in
+    # update_version are no-ops here since the version is unchanged for a hotfix.
+    update_version
   else
+    SOUP_UPGRADE_STARTED=true
     echo ""
     echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
     echo ""
@@ -1820,6 +1978,10 @@ main() {
     copy_new_files
     echo ""
     create_local_directories "/opt/so/saltstack/default"
+    # Seed the resume marker before the highstate stamps /etc/soversion to the new
+    # version, so an interrupted upgrade is detectable as "not finished" on re-run.
+    # POSTVERSION still holds the pre-upgrade (or prior resume) version here.
+    [ -f "$POSTVERSION_FILE" ] || echo "$POSTVERSION" > "$POSTVERSION_FILE"
     update_version
 
     echo ""

salt/nginx/enabled.sls

@@ -59,6 +59,7 @@ so-nginx:
       - /opt/so/conf/navigator/layers/:/opt/socore/html/navigator/assets/so:ro
       - /opt/so/conf/navigator/config.json:/opt/socore/html/navigator/assets/config.json:ro
       - /nsm/repo:/opt/socore/html/repo:ro
+      - /nsm/kernelrepo:/opt/socore/html/kernelrepo:ro
       - /nsm/rules:/nsm/rules:ro
       {%   if NGINXMERGED.external_suricata %}
       - /opt/so/rules/nids/suri:/surirules:ro

salt/nginx/etc/nginx.conf

@@ -323,6 +323,16 @@ http {
 			autoindex_localtime on;
 		}
 
+		location /kernelrepo/ {
+			allow all;
+			sendfile on;
+			sendfile_max_chunk 1m;
+			autoindex on;
+			autoindex_exact_size off;
+			autoindex_format html;
+			autoindex_localtime on;
+		}
+
 		location /influxdb/ {
 			auth_request          /auth/sessions/whoami;
 			rewrite               /influxdb/api/(.*) /api/$1 break;
@@ -389,7 +399,7 @@ http {
 		error_page 429 = @error429;
 
 		location @error401 {
-			if ($request_uri ~* (^/api/.*|^/connect/.*|^/oauth2/.*|^/.*\.map$)) {
+			if ($request_uri ~* (^.*/api/.*|^/connect/.*|^/oauth2/.*|^/.*\.map$)) {
 				return    401;
 			}
 

salt/registry/enabled.sls

@@ -16,7 +16,7 @@ include:
 # Install the registry container
 so-dockerregistry:
   docker_container.running:
-    - image: ghcr.io/security-onion-solutions/registry:3.0.0
+    - image: ghcr.io/security-onion-solutions/registry:3.1.1
     - hostname: so-registry
     - networks:
       - sobridge:

salt/repo/client/oracle.sls

@@ -6,6 +6,10 @@
 {% from 'repo/client/map.jinja' import REPOPATH with context %}
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 
+{% import_yaml 'salt/minion.defaults.yaml' as saltversion %}
+{% set saltversion = saltversion.salt.minion.version %}
+{% set INSTALLEDSALTVERSION = grains.saltversion %}
+
 {% set role = grains.id.split('_') | last %}
 {% set MANAGER = salt['grains.get']('master') %}
 {% if grains['os'] == 'OEL' %}
@@ -57,6 +61,32 @@ so_repo:
     - enabled: 1
     - gpgcheck: 1
 
+# Only assign the kernel repo once this node's running salt matches the version this
+# SO release ships. During a soup the grid is mid-salt-upgrade; gating here keeps the
+# UEK8 kernel repo (and the kernel update it enables) from activating until the node is
+# fully on the target salt, the same way other states defer across the upgrade window.
+{% if saltversion | string == INSTALLEDSALTVERSION | string %}
+so_kernel_repo:
+  pkgrepo.managed:
+    - name: securityonionkernel
+    - humanname: Security Onion Kernel Repo
+  {% if GLOBALS.is_manager %}
+    - baseurl: file:///nsm/kernelrepo/
+  {% else %}
+    - baseurl: https://{{ GLOBALS.repo_host }}/kernelrepo
+  {% endif %}
+    - enabled: 1
+    - gpgcheck: 1
+    # Supplementary kernel repo: tolerate it being empty/unreachable (e.g. before the
+    # manager has populated /nsm/kernelrepo) so a missing repomd.xml can't make every
+    # dnf/pkg operation on the grid fail.
+    - skip_if_unavailable: 1
+    # Only assign the kernel repo once physical NIC names are pinned by MAC, so the
+    # UEK8 kernel update can't renumber interfaces SO binds by name (see pin_nic_names
+    # in salt/common/init.sls, which drops this marker via /usr/sbin/so-nic-pin).
+    - onlyif: 'test -e /opt/so/state/nic_names_pinned'
+{% endif %}
+
 {% endif %}
   
 # TODO: Add a pillar entry for custom repos

salt/soc/config.sls

@@ -134,6 +134,30 @@ socsigmasopipeline:
     - group: 939
     - mode: 600
 
+socsigmaplaybookpipeline:
+  file.managed:
+    - name: /opt/so/conf/soc/sigma_playbook_pipeline.yaml
+    - source: salt://soc/files/soc/sigma_playbook_pipeline.yaml
+    - user: 939
+    - group: 939
+    - mode: 600
+
+socplaybookplaceholdermap:
+  file.managed:
+    - name: /opt/so/conf/soc/playbook_placeholder_map.yaml
+    - source: salt://soc/files/soc/playbook_placeholder_map.yaml
+    - user: 939
+    - group: 939
+    - mode: 600
+
+socplaybookplaceholdermapcustom:
+  file.managed:
+    - name: /opt/so/conf/soc/playbook_placeholder_map_custom.yaml
+    - source: salt://soc/files/soc/playbook_placeholder_map_custom.yaml
+    - user: 939
+    - group: 939
+    - mode: 600
+
 socbanner:
   file.managed:
     - name: /opt/so/conf/soc/banner.md

salt/soc/defaults.map.jinja

@@ -8,6 +8,7 @@
 {% from 'docker/docker.map.jinja' import DOCKERMERGED -%}
 {% set INFLUXDB_TOKEN = salt['pillar.get']('influxdb:token') %}
 {% import_text 'influxdb/metrics_link.txt' as METRICS_LINK %}
+{% from 'telegraf/map.jinja' import TELEGRAFMERGED %}
 
 {% for module, application_url in GLOBALS.application_urls.items() %}
 {%   do SOCDEFAULTS.soc.config.server.modules[module].update({'hostUrl': application_url}) %}
@@ -24,6 +25,14 @@
 
 {% do SOCDEFAULTS.soc.config.server.modules.elastic.update({'username': GLOBALS.elasticsearch.auth.users.so_elastic_user.user, 'password': GLOBALS.elasticsearch.auth.users.so_elastic_user.pass}) %}
 
+{% if TELEGRAFMERGED.output == 'POSTGRES' %}
+{%   for tool in SOCDEFAULTS.soc.config.server.client.tools %}
+{%     if tool.name == "toolInfluxDb" %}
+{%       do SOCDEFAULTS.soc.config.server.client.tools.remove(tool) %}
+{%     endif %}
+{%   endfor %}
+
+{% else %}
 {%   do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %}
 {%   do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'token': INFLUXDB_TOKEN}) %}
 {%   for tool in SOCDEFAULTS.soc.config.server.client.tools %}
@@ -31,6 +40,7 @@
 {%       do tool.update({'link': METRICS_LINK}) %}
 {%     endif %}
 {%   endfor %}
+{% endif %}
 
 {% do SOCDEFAULTS.soc.config.server.modules.statickeyauth.update({'anonymousCidr': DOCKERMERGED.range, 'apiKey': pillar.sensoroni.config.sensoronikey}) %}
 

salt/soc/defaults.yaml

@@ -1500,15 +1500,23 @@ soc:
           playbookRepos:
             default:
               - repo: https://github.com/Security-Onion-Solutions/securityonion-resources-playbooks
+                rulesetName: sos-playbook-resources
                 branch: main
                 folder: securityonion-normalized
+              - repo: https://github.com/Security-Onion-Solutions/securityonion-resources-playbooks
+                rulesetName: sos-published
+                branch: published
+                folder: sigma
             airgap:
               - repo: file:///nsm/airgap-resources/playbooks/securityonion-resources-playbooks
+                rulesetName: sos-resources-ag
                 branch: main
                 folder: securityonion-normalized
         assistant:
           systemPromptAddendum: ""
           systemPromptAddendumMaxLength: 50000
+          maxSubSessionTokens: 0
+          maxDelegationDepth: 5
           adapters:
             - name: SOAI
               protocol: securityonion_ai_cloud
@@ -1520,6 +1528,11 @@ soc:
               serviceAccountJSON: ""
               serviceAccountLocation: ""
               healthTimeoutSeconds: 5
+          agentic: false
+          agentMapping:
+            Orchestrator: sonnet
+            Investigator: sonnet
+            Detection Engineer: sonnet
         onionconfig:
           saltstackDir: /opt/so/saltstack
           bypassEnabled: false
@@ -1771,13 +1784,13 @@ soc:
               enabled: true
           queries:
             - name: Default Query
-              description: Show all events grouped by the observer host
-              query: '* | groupby observer.name'
-              showSubtitle: true
-            - name: Log Type
               description: Show all events grouped by module and dataset
               query: '* | groupby event.module* event.dataset'
               showSubtitle: true
+            - name: Observer
+              description: Show all events grouped by the observer host
+              query: '* | groupby observer.name'
+              showSubtitle: true
             - name: SOC - Auth
               description: Users authenticated to SOC grouped by IP address and identity
               query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby http.request.headers.x-real-ip user.name'
@@ -2689,6 +2702,8 @@ soc:
           thresholdColorRatioLow: 0.5
           thresholdColorRatioMed: 0.75
           thresholdColorRatioMax: 1
+          toolBusyMaxRetries: 30
+          toolBusyRetryDelayMs: 1000
           availableModels:
             - id: sonnet
               displayName: Claude Sonnet

salt/soc/enabled.sls

@@ -45,7 +45,10 @@ so-soc:
       - /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro
       - /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro
       - /opt/so/conf/soc/sigma_so_pipeline.yaml:/opt/sensoroni/sigma_so_pipeline.yaml:ro
-      - /opt/so/conf/soc/sigma_final_pipeline.yaml:/opt/sensoroni/sigma_final_pipeline.yaml:rw
+      - /opt/so/conf/soc/sigma_playbook_pipeline.yaml:/opt/sensoroni/sigma_playbook_pipeline.yaml:ro
+      - /opt/so/conf/soc/sigma_final_pipeline.yaml:/opt/sensoroni/sigma_final_pipeline.yaml:ro
+      - /opt/so/conf/soc/playbook_placeholder_map.yaml:/opt/sensoroni/playbook_placeholder_map.yaml:ro
+      - /opt/so/conf/soc/playbook_placeholder_map_custom.yaml:/opt/sensoroni/playbook_placeholder_map_custom.yaml:ro
       - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro
       - /opt/so/conf/soc/custom_roles:/opt/sensoroni/rbac/custom_roles:ro
       - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw
@@ -99,6 +102,8 @@ so-soc:
       - file: soccustomroles
       - file: socusersroles
       - file: socclientsroles
+      - file: socplaybookplaceholdermap
+      - file: socplaybookplaceholdermapcustom
 
 delete_so-soc_so-status.disabled:
   file.uncomment:

salt/soc/files/soc/playbook_placeholder_map.yaml

@@ -0,0 +1,49 @@
+# Global Playbook placeholder map: %token% -> event field path.
+#
+# Loaded by the SOC Playbook module and used to resolve `field|expand:%placeholder%` values 
+# from an alert when converting playbook questions to OQL. 
+# Left: the %token% used in a question
+# Right: the event field its value is read from (event_data.-nested or bare; the module
+# tries both). 
+#
+# Example: with `src_ip: source.ip` (below), a question that writes
+# `source.ip|expand: '%src_ip%'` resolves %src_ip% to the alert's source.ip at convert time.
+#
+# This is the global base layer. To add or override tokens edit playbook_placeholder_map_custom.yaml.
+# those entries overlay this map and win on conflict.
+
+CommandLine: process.command_line
+CurrentDirectory: process.working_directory
+Image: process.executable
+ImageLoaded: dll.name
+ParentImage: process.parent.executable
+ParentName: process.parent.name
+ParentProcessGuid: process.parent.entity_id
+ProcessGuid: process.entity_id
+TargetFilename: file.name
+TargetObject: registry.path
+TargetUserName: user.target.name
+User: user.name
+community_id: network.community_id
+dns_resolved_ip: dns.resolved_ip
+document_id: soc_id
+dst_ip: destination.ip
+dst_port: destination.port
+event_data_source_ip: source.ip
+file_path: file.path
+file_dirs: process.file_dirs
+file_name: process.name
+file_paths: process.file_paths
+hostname: host.name
+private_ip: network.private_ip
+public_ip: network.public_ip
+related_hosts: related.hosts
+related_ip: related.ip
+src_ip: source.ip
+dns_query_name: dns.query_name
+flow_id: log.id.uid
+payload: network.data.decoded
+rule_category: rule.category
+rule_name: rule.name
+rule_uuid: rule.uuid
+src_port: source.port

salt/soc/files/soc/playbook_placeholder_map_custom.yaml

@@ -0,0 +1,14 @@
+# Custom Playbook placeholder map: %token% -> event field path.
+#
+#
+# Left: the %token% used in a playbook question.
+# Right: the event field its value is read from (event_data.-nested or bare; the module tries
+# both). Note: a token that is simply named after a flat event field resolves automatically
+# without an entry here - only add a mapping when the token name differs from the field name.
+#
+# Example:
+#
+# account_id: cloudflare.account_id
+#
+# A question that writes
+# `account_id|expand: '%account_id%'` resolves %account_id% from the alert at convert time.

salt/soc/files/soc/sigma_playbook_pipeline.yaml

@@ -0,0 +1,12 @@
+name: Security Onion - Playbook Pipeline
+priority: 97
+transformations:
+    # Route string fields to their lowercase-normalized .caseless subfield so wildcard
+    # matches are case-insensitive.
+    - id: case_insensitive_string_fields
+      type: field_name_mapping
+      mapping:
+        process.executable: process.executable.caseless
+        process.parent.executable: process.parent.executable.caseless
+        process.command_line: process.command_line.caseless
+        process.parent.command_line: process.parent.command_line.caseless

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -63,6 +63,14 @@ transformations:
       rule_conditions:
       - type: logsource
         category: antivirus
+    # OS-agnostic process_creation scoping for product-less (NIDS/host-pivot) rules.
+    - id: process_creation_os_agnostic
+      type: add_condition
+      conditions:
+        event.category: process
+      rule_conditions:
+      - type: logsource
+        category: process_creation
     # Transforms the `Hashes` field to ECS fields
     # ECS fields are used by the hash fields emitted by Elastic Defend
     # If shipped with Elastic Agent, sysmon logs will also have hashes mapped to ECS fields    
@@ -108,6 +116,40 @@ transformations:
       - type: logsource
         product: windows
         category: driver_load
+    - id: ecs_fix_process_creation
+      type: field_name_mapping
+      mapping:
+        # bare `Hashes` (the combined-string case is broken out above)
+        winlog.event_data.Hashes: process.hash.sha256
+        winlog.event_data.IntegrityLevel: process.Ext.token.integrity_level_name
+        winlog.event_data.ParentName: process.parent.name
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: process_creation
+    - id: ecs_fix_registry_set
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.Details: registry.data.strings
+        # field rename only; EventType values (SetValue/CreateKey) still differ from
+        # event.action values (modification/creation)
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: registry_set
+    - id: ecs_fix_image_load
+      type: field_name_mapping
+      mapping:
+        file.path: dll.path
+        file.code_signature.signed: dll.code_signature.exists
+        winlog.event_data.Signature: dll.code_signature.subject_name
+        file.code_signature.status: dll.code_signature.status
+        winlog.event_data.Hashes: dll.hash.sha256
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: image_load
     - id: linux_security_add-fields
       type: add_condition
       conditions:
@@ -281,6 +323,15 @@ transformations:
       rule_conditions:
       - type: logsource
         category: file_event
+    # Scope image_load rules to Elastic Endpoint library events (event.category:library, dll.*
+    # populated).
+    - id: endpoint_image_load_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'library'
+      rule_conditions:
+      - type: logsource
+        category: image_load
     # Maps network rules to all network logs
     # This targets all network logs, all services, generated from endpoints and network
     - id: network_add-fields

salt/soc/merged.map.jinja

@@ -7,6 +7,11 @@
 {% from 'soc/defaults.map.jinja' import SOCDEFAULTS with context %}
 {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
 {% from 'manager/map.jinja' import MANAGERMERGED %}
+{% from 'telegraf/map.jinja' import TELEGRAFMERGED %}
+{%- set PG_ENTRY = salt['pillar.get']('telegraf:postgres_creds:' ~ grains.id, {}) %}
+{%- set PG_USER = PG_ENTRY.get('user', '') %}
+{%- set PG_PASS = PG_ENTRY.get('pass', '') %}
+
 {% set DOCKER_EXTRA_HOSTS = ELASTICSEARCH_NODES %}
 {% do DOCKER_EXTRA_HOSTS.append({GLOBALS.influxdb_host:pillar.node_data[GLOBALS.influxdb_host].ip}) %}
 
@@ -75,6 +80,20 @@
 {%   do SOCMERGED.config.server.update({'airgapEnabled': false}) %}
 {% endif %}
 
+{# Define the postgresmetrics module if telegraf is setup to only use Postgres #}
+{% if TELEGRAFMERGED.output != 'INFLUXDB' and PG_USER and PG_PASS %}
+{%   do SOCMERGED.config.server.modules.update({
+        'postgresmetrics': {
+          'database': 'so_telegraf',
+          'host': GLOBALS.manager_ip,
+          'password': PG_PASS,
+          'port': 5432,
+          'sslMode': 'allow',
+          'user': PG_USER,
+        }
+      }) %}
+{%   do SOCMERGED.config.server.modules.pop('influxdb') %}
+{% endif %}
 
 {# Define the Detections custom ruleset that should always be present #}
 {% set CUSTOM_RULESET = {

salt/soc/soc_soc.yaml

@@ -46,7 +46,15 @@ soc:
         syntax: yaml
         file: True
         global: True
-        advanced: True
+        advanced: False
+        helpLink: security-onion-console-customization
+      playbook_placeholder_map_custom__yaml:
+        title: Playbook Placeholder Map
+        description: Custom mappings of Playbook %placeholder% tokens to event fields.
+        syntax: yaml
+        file: True
+        global: True
+        advanced: False
         helpLink: security-onion-console-customization
   config:
     licenseKey:
@@ -719,6 +727,16 @@ soc:
             description: Maximum length of the system prompt addendum. Longer prompts will be truncated.
             global: True
             advanced: True
+          maxSubSessionTokens:
+            description: Maximum number of output tokens a delegated sub-session may generate across all of its turns. When the budget is reached, the sub-agent is halted and its result is returned to the parent agent. Set to 0 to disable the limit.
+            global: True
+            advanced: True
+            forcedType: int
+          maxDelegationDepth:
+            description: Maximum delegation nesting depth for sub-agents. For example, a value of 2 lets the main agent delegate to a sub-agent that may itself delegate one level deeper. Any deeper delegation is refused and the requesting agent continues without it. Set to 0 to disable the limit.
+            global: True
+            advanced: True
+            forcedType: int
           adapters:
             description: Configuration for AI adapters used by the Onion AI assistant. Please see documentation for help on which fields are required for which protocols.
             global: True
@@ -757,12 +775,32 @@ soc:
               label: Health Timeout Seconds
               required: False
               forcedType: int
+          agentic:
+            description: Indicates if the Assistant Module should operate in agentic mode or not. If true, agents can work together to solve tasks.
+            global: True
+            forcedType: bool
+          agentMapping:
+            Orchestrator:
+              description: The initial agent in most agentic conversations. This agent will delegate requests to specialized agents.
+              global: True
+            Investigator:
+              description: This agent investigates alerts, explains events and records, and hunts through event data. It can also acknowledge alerts and escalate to cases.
+              global: True
+            Detection Engineer:
+              description: This agent manages detections and their overrides, including tuning noisy rules and authoring rule content.
+              global: True
       client:
         assistant:
           enabled:
             description: Set to true to enable the Onion AI assistant in SOC.
             global: True
             forcedType: bool
+          toolBusyMaxRetries:
+            description: How many times to retry auto approving a tool while a tool is already running.
+            global: True
+          toolBusyRetryDelayMs:
+            description: How long in milliseconds to wait between each retry when auto approving a tool.
+            global: True
           investigationPrompt: 
             description: Prompt given to Onion AI when beginning an investigation.
             global: True

salt/suricata/enabled.sls

@@ -69,6 +69,7 @@ surirulereload:
     - name: /usr/sbin/so-suricata-reload-rules >> /opt/so/log/suricata/reload.log 2>&1
     - onchanges:
         - file: surirulesync
+    - onlyif: test -f /opt/so/rules/suricata/all-rulesets.rules
     - require:
         - docker_container: so-suricata
     

salt/suricata/soc_suricata.yaml

@@ -153,12 +153,12 @@ suricata:
       cpu-affinity:
         management-cpu-set:
           cpu:
-            description: Bind management threads to a core or range of cores. This can be a sigle core, list of cores, or list of range of cores. set-cpu-affinity must be set to true for this to be used.
+            description: Bind management threads to a core or range of cores. This can be a single core, list of cores, or list of range of cores. set-cpu-affinity must be set to true for this to be used.
             forcedType: "[]string"
             helpLink: suricata
         worker-cpu-set:
           cpu:
-            description: Bind worker threads to a core or range of cores. This can be a sigle core, list of cores, or list of range of cores. set-cpu-affinity must be set to true for this to be used.
+            description: Bind worker threads to a core or range of cores. This can be a single core, list of cores, or list of range of cores. set-cpu-affinity must be set to true for this to be used.
             forcedType: "[]string"
             helpLink: suricata
     vars:

salt/suricata/tools/sbin/so-suricata-reload-rules

@@ -7,5 +7,59 @@
 
 . /usr/sbin/so-common
 
-retry 60 3 'docker exec so-suricata /opt/suricata/bin/suricatasc -c reload-rules /var/run/suricata/suricata-command.socket' '{"message":"done","return":"OK"}' || fail "The Suricata container was not ready in time."
+RULES_FILE="/opt/so/rules/suricata/all-rulesets.rules"
-retry 60 3 'docker exec so-suricata /opt/suricata/bin/suricatasc -c ruleset-reload-nonblocking /var/run/suricata/suricata-command.socket' '{"message":"done","return":"OK"}' || fail "The Suricata container was not ready in time."
+SOCKET="/var/run/suricata/suricata-command.socket"
+SURICATASC="docker exec so-suricata /opt/suricata/bin/suricatasc"
+
+# Format an epoch as a human-readable local timestamp for log messages.
+fmt_time() { date -d "@$1" '+%Y-%m-%d %H:%M:%S %Z' 2>/dev/null; }
+
+# Prefix each input line with the current timestamp.
+timestamp_lines() { while IFS= read -r line; do printf '%s %s\n' "$(date '+%Y-%m-%d %H:%M:%S %Z')" "$line"; done; }
+
+# Epoch of Suricata's last *completed* ruleset reload; non-zero return on failure.
+suricata_reload_epoch() {
+  local out ts
+  out=$($SURICATASC -c ruleset-reload-time "$SOCKET" 2>/dev/null)
+  ts=$(echo "$out" | jq -r '.message[0].last_reload // empty' 2>/dev/null)
+  [ -n "$ts" ] || return 1
+  date -d "$ts" +%s 2>/dev/null
+}
+
+# Trigger a fresh reload and confirm Suricata is running a ruleset at least as new
+# as the rules file. Returns 0 only when both hold, so retry keeps going until an
+# in-progress reload clears and our own reload completes.
+reload_and_verify() {
+  local out reload_epoch
+  out=$($SURICATASC -c reload-rules "$SOCKET")
+  echo "reload-rules: $out"
+
+  if [[ "$out" =~ "Reload already in progress" ]]; then
+    echo "A reload is already in progress; waiting for it to clear so a fresh reload can load the current ruleset."
+    return 1
+  fi
+  if [[ ! "$out" =~ '{"message":"done","return":"OK"}' ]]; then
+    echo "Suricata not ready or unexpected reload output; will retry."
+    return 1
+  fi
+
+  reload_epoch=$(suricata_reload_epoch) || { echo "Could not read ruleset-reload-time; will retry."; return 1; }
+  if [ "$reload_epoch" -ge "$target_mtime" ]; then
+    echo "Loaded ruleset is current: last reload ($(fmt_time "$reload_epoch")) is newer than rules file ($(fmt_time "$target_mtime"))."
+    return 0
+  fi
+  echo "Loaded ruleset is stale: last reload ($(fmt_time "$reload_epoch")) is older than rules file ($(fmt_time "$target_mtime")); retrying."
+  return 1
+}
+
+# Run the reload/verify, timestamping every line of output (ours and the
+# retry/fail helpers') so reload.log shows when each step ran. The pipeline is
+# synchronous, so the log is fully flushed and ordered before we exit; the
+# script's real exit code is preserved via PIPESTATUS.
+{
+  # Epoch mtime of the ruleset we need Suricata to have loaded. Captured once so
+  # a file update mid-reload does not move the goalpost.
+  target_mtime=$(stat -c %Y "$RULES_FILE") || fail "Could not stat the Suricata rules file: $RULES_FILE"
+  retry 60 3 'reload_and_verify' || fail "Suricata did not load the current ruleset in time."
+} 2>&1 | timestamp_lines
+exit "${PIPESTATUS[0]}"

salt/telegraf/etc/telegraf.conf

@@ -244,6 +244,7 @@
   username = "{{ ES_USER }}"
   password = "{{ ES_PASS }}"
   insecure_skip_verify = true
+  cluster_health = true
 {%- elif grains['role'] in ['so-searchnode']  %}
 [[inputs.elasticsearch]]
   servers = ["https://{{ NODEIP }}:9200"]

salt/telegraf/soc_telegraf.yaml

@@ -5,7 +5,7 @@ telegraf:
     advanced: True
     helpLink: influxdb
   output:
-    description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation.
+    description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation. When set to BOTH, the grid screen's metrics are pulled from Postgres, and the InfluxDB tool link remains visible. When set to POSTGRES, the InfluxDB tool link is removed.
     options:
       - INFLUXDB
       - POSTGRES

salt/top.sls

@@ -83,7 +83,6 @@ base:
     - zeek
     - strelka
     - elastalert
-    - utility
     - elasticfleet
     - pcap.cleanup
 
@@ -113,7 +112,6 @@ base:
     - zeek
     - strelka
     - elastalert
-    - utility
     - elasticfleet
     - stig
     - kafka
@@ -141,7 +139,6 @@ base:
     - elastic-fleet-package-registry
     - kibana
     - elastalert
-    - utility
     - elasticfleet
     - stig
     - kafka
@@ -168,7 +165,6 @@ base:
     - elastic-fleet-package-registry
     - kibana
     - elastalert
-    - utility
     - elasticfleet
     - kafka
 
@@ -198,7 +194,6 @@ base:
     - elastic-fleet-package-registry
     - kibana
     - elastalert
-    - utility
     - elasticfleet
     - stig
     - kafka
@@ -222,7 +217,6 @@ base:
     - elasticsearch
     - elastic-fleet-package-registry
     - kibana
-    - utility
     - suricata
     - zeek
     - elasticfleet

salt/utility/bin/eval

@@ -1,29 +0,0 @@
-#!/bin/bash
-# Wait for ElasticSearch to come up, so that we can query for version infromation
-echo -n "Waiting for ElasticSearch..."
-COUNT=0
-ELASTICSEARCH_CONNECTED="no"
-while [[ "$COUNT" -le 30 ]]; do
-  curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://{{ GLOBALS.manager_ip }}:9200
-  if [ $? -eq 0 ]; then
-    ELASTICSEARCH_CONNECTED="yes"
-    echo "connected!"
-    break
-  else
-    ((COUNT+=1))
-    sleep 1
-    echo -n "."
-  fi
-done
-if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
-  echo
-  echo -e "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'docker ps' \n  -running 'sudo so-elastic-restart'"
-  echo
-
-  exit
-fi
-
-echo "Applying cross cluster search config..."
-    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -XPUT -L https://{{ GLOBALS.manager_ip }}:9200/_cluster/settings \
-         -H 'Content-Type: application/json' \
-         -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ grains.host }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"

salt/utility/init.sls

@@ -1,22 +0,0 @@
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-
-{% if sls in allowed_states %}
-  {% if grains['role'] in ['so-eval', 'so-import'] %}
-fixsearch:
-  cmd.script:
-    - shell: /bin/bash
-    - cwd: /opt/so
-    - source: salt://utility/bin/eval
-    - template: jinja
-    - defaults:
-        GLOBALS: {{ GLOBALS }}
-  {% endif %}
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

setup/so-functions

@@ -29,8 +29,12 @@ title() {
 }
 
 fail_setup() {
+	local err_msg=$1
+	if [[ -n "$err_msg" ]]; then
+		error "$err_msg"
+	fi
 	error "Setup encountered an unrecoverable failure, exiting"
-	touch /root/failure
+	echo "setup incomplete: $err_msg" > /root/failure
 	exit 1
 }
 
@@ -697,7 +701,7 @@ compare_main_nic_ip() {
 			EOM
 
 		  [[ -n $TESTING ]] || whiptail --title "$whiptail_title" --msgbox "$message" 11 75
-		  kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup
+		  kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup "Main IP mismatch"
 	  fi
 	else
 		# Setup uses MAINIP, but since we ignore the equality condition when using a VPN 
@@ -755,8 +759,7 @@ configure_management_bond() {
 	info "Setting up $bond_name management interface with mode $bond_mode"
 
 	if [[ ${#MBNICS[@]} -eq 0 ]]; then
-		error "[ERROR] No management bond NICs were selected."
+		fail_setup "No management bond NICs selected"
-		fail_setup
 	fi
 
 	nmcli -t -f NAME con show | grep -Fxq "$bond_name"
@@ -886,6 +889,7 @@ create_repo() {
 	title "Create the repo directory"
 	logCmd "dnf -y install yum-utils createrepo_c"
 	logCmd "createrepo /nsm/repo"
+	logCmd "createrepo /nsm/kernelrepo"
 }
 
 
@@ -913,8 +917,7 @@ detect_os() {
 		is_rpm=true
 		is_supported=true
 	else
-		info "This OS is not supported. Security Onion requires Oracle Linux 9."
+		fail_setup "This OS is not supported. Security Onion requires Oracle Linux 9."
-		fail_setup
 	fi
 
 	info "Found OS: $OS $OSVER"
@@ -922,7 +925,7 @@ detect_os() {
 
 download_elastic_agent_artifacts() {
 	if ! update_elastic_agent 2>&1 | tee -a "$setup_log"; then
-		fail_setup
+		fail_setup "Failed to update Elastic Agent"
 	fi
 }
 
@@ -1566,7 +1569,7 @@ proxy_validate() {
 		error "Received error: $proxy_test_err"
 		if [[ -n $TESTING ]]; then
 			error "Exiting setup"
-			kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup
+			kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup "Proxy validation failed"
 		fi
 	fi
 	return $ret
@@ -1773,8 +1776,7 @@ ensure_pyyaml() {
 	local result=$?
 	set +o pipefail
 	if [[ $result -ne 0 ]] || ! rpm -q python3-pyyaml >/dev/null 2>&1; then
-		error "Failed to install python3-pyyaml (exit=$result)"
+		fail_setup "Failed to install python3-pyyaml (exit=$result)"
-		fail_setup
 	fi
 	info "python3-pyyaml installed successfully"
 }
@@ -1812,6 +1814,16 @@ securityonion_repo() {
 			echo "mirrorlist=file:///etc/yum/mirror.txt" >> /etc/yum.repos.d/securityonion.repo
 			echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo
 			echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo
+			echo "https://repo.securityonion.net/file/so-repo/prod/3/oracle/9-uek8" > /etc/yum/mirror-kernel.txt
+			echo "https://so-repo-east.s3.us-east-005.backblazeb2.com/prod/3/oracle/9-uek8" >> /etc/yum/mirror-kernel.txt
+			echo "[securityonionkernel]" > /etc/yum.repos.d/securityonionkernel.repo
+			echo "name=Security Onion Kernel Repo repo" >> /etc/yum.repos.d/securityonionkernel.repo
+			echo "mirrorlist=file:///etc/yum/mirror-kernel.txt" >> /etc/yum.repos.d/securityonionkernel.repo
+			echo "enabled=1" >> /etc/yum.repos.d/securityonionkernel.repo
+			echo "gpgcheck=1" >> /etc/yum.repos.d/securityonionkernel.repo
+			# Supplementary kernel repo: tolerate it being empty/unreachable so a missing
+			# repomd.xml can't make every dnf operation fail before the repo is populated.
+			echo "skip_if_unavailable=1" >> /etc/yum.repos.d/securityonionkernel.repo
 			logCmd "dnf repolist"
 		else
 			echo "[securityonion]" > /etc/yum.repos.d/securityonion.repo
@@ -1820,6 +1832,13 @@ securityonion_repo() {
 			echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo
 			echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo
 			echo "sslverify=0" >> /etc/yum.repos.d/securityonion.repo
+			echo "[securityonionkernel]" > /etc/yum.repos.d/securityonionkernel.repo
+			echo "name=Security Onion Kernel Repo" >> /etc/yum.repos.d/securityonionkernel.repo
+			echo "baseurl=https://$MSRV/kernelrepo" >> /etc/yum.repos.d/securityonionkernel.repo
+			echo "enabled=1" >> /etc/yum.repos.d/securityonionkernel.repo
+			echo "gpgcheck=1" >> /etc/yum.repos.d/securityonionkernel.repo
+			echo "sslverify=0" >> /etc/yum.repos.d/securityonionkernel.repo
+			echo "skip_if_unavailable=1" >> /etc/yum.repos.d/securityonionkernel.repo
 			logCmd "dnf repolist"
 		fi
 	elif [[ ! $waitforstate ]]; then
@@ -1829,12 +1848,25 @@ securityonion_repo() {
 		echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo
 		echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo
 		echo "sslverify=0" >> /etc/yum.repos.d/securityonion.repo
+		echo "[securityonionkernel]" > /etc/yum.repos.d/securityonionkernel.repo
+		echo "name=Security Onion Kernel Repo" >> /etc/yum.repos.d/securityonionkernel.repo
+		echo "baseurl=https://$MSRV/kernelrepo" >> /etc/yum.repos.d/securityonionkernel.repo
+		echo "enabled=1" >> /etc/yum.repos.d/securityonionkernel.repo
+		echo "gpgcheck=1" >> /etc/yum.repos.d/securityonionkernel.repo
+		echo "sslverify=0" >> /etc/yum.repos.d/securityonionkernel.repo
+		echo "skip_if_unavailable=1" >> /etc/yum.repos.d/securityonionkernel.repo
 	elif [[ $waitforstate ]]; then
 		echo "[securityonion]" > /etc/yum.repos.d/securityonion.repo
 		echo "name=Security Onion Repo" >> /etc/yum.repos.d/securityonion.repo
 		echo "baseurl=file:///nsm/repo/" >> /etc/yum.repos.d/securityonion.repo
 		echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo
 		echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo
+		echo "[securityonionkernel]" > /etc/yum.repos.d/securityonionkernel.repo
+		echo "name=Security Onion Kernel Repo" >> /etc/yum.repos.d/securityonionkernel.repo
+		echo "baseurl=file:///nsm/kernelrepo/" >> /etc/yum.repos.d/securityonionkernel.repo
+		echo "enabled=1" >> /etc/yum.repos.d/securityonionkernel.repo
+		echo "gpgcheck=1" >> /etc/yum.repos.d/securityonionkernel.repo
+		echo "skip_if_unavailable=1" >> /etc/yum.repos.d/securityonionkernel.repo
 	fi
 	logCmd "dnf repolist all"
 	if [[ $waitforstate ]]; then
@@ -1850,9 +1882,12 @@ repo_sync_local() {
 	# Sync the repo from the SO repo locally.
 	info "Adding Repo Download Configuration"
 	mkdir -p /nsm/repo
+	mkdir -p /nsm/kernelrepo
 	mkdir -p /opt/so/conf/reposync/cache
 	echo "https://repo.securityonion.net/file/so-repo/prod/3/oracle/9" > /opt/so/conf/reposync/mirror.txt
 	echo "https://repo-alt.securityonion.net/prod/3/oracle/9" >> /opt/so/conf/reposync/mirror.txt
+	echo "https://repo.securityonion.net/file/so-repo/prod/3/oracle/9-uek8" > /opt/so/conf/reposync/mirror-kernel.txt
+	echo "https://repo-alt.securityonion.net/prod/3/oracle/9-uek8" >> /opt/so/conf/reposync/mirror-kernel.txt
 	echo "[main]" > /opt/so/conf/reposync/repodownload.conf
 	echo "gpgcheck=1" >> /opt/so/conf/reposync/repodownload.conf
 	echo "installonly_limit=3" >> /opt/so/conf/reposync/repodownload.conf
@@ -1866,12 +1901,18 @@ repo_sync_local() {
 	echo "mirrorlist=file:///opt/so/conf/reposync/mirror.txt" >> /opt/so/conf/reposync/repodownload.conf
 	echo "enabled=1" >> /opt/so/conf/reposync/repodownload.conf
 	echo "gpgcheck=1" >> /opt/so/conf/reposync/repodownload.conf
+	echo "[securityonionkernel]" >> /opt/so/conf/reposync/repodownload.conf
+	echo "name=Security Onion Kernel Repo repo" >> /opt/so/conf/reposync/repodownload.conf
+	echo "mirrorlist=file:///opt/so/conf/reposync/mirror-kernel.txt" >> /opt/so/conf/reposync/repodownload.conf
+	echo "enabled=1" >> /opt/so/conf/reposync/repodownload.conf
+	echo "gpgcheck=1" >> /opt/so/conf/reposync/repodownload.conf
 
 	logCmd "dnf repolist"
 
 	if [[ ! $is_airgap ]]; then
 		curl --retry 5 --retry-delay 60 -A "netinstall/$SOVERSION/$OS/$(uname -r)/1" https://sigs.securityonion.net/checkup --output /tmp/install
-		retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" >> "$setup_log" 2>&1 || fail_setup
+		retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" >> "$setup_log" 2>&1 || fail_setup "Failed to sync repos"
+		retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/" >> "$setup_log" 2>&1 || fail_setup "Failed to sync kernel repos"
 		# After the download is complete run createrepo
 		create_repo
 	fi
@@ -1884,10 +1925,10 @@ saltify() {
 
 	if [[ $waitforstate ]]; then
 		# install all for a manager
-		retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -M -X stable $SALTVERSION" || fail_setup
+		retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -M -X stable $SALTVERSION" || fail_setup "Failed to install salt master"
 	else
 		# just a minion
-		retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -X stable $SALTVERSION" || fail_setup
+		retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -X stable $SALTVERSION" || fail_setup "Failed to install salt minion"
 	fi
 
 	salt_install_module_deps
@@ -1959,7 +2000,7 @@ set_main_ip() {
 			info "MAINIP=$MAINIP"
 			info "MNIC_IP=$MNIC_IP"
 			whiptail_error_message "The management IP could not be determined. Please check the log at /root/sosetup.log and verify the network configuration. Select OK to exit."
-			fail_setup
+			fail_setup "Could not determine MAINIP or MNIC_IP"
 		fi
 		sleep 1
 	done
@@ -2163,7 +2204,7 @@ set_initial_firewall_access() {
 set_management_interface() {
 	title "Setting up the main interface"
 	if [[ $MNIC == "bond1" ]]; then
-		configure_management_bond || fail_setup
+		configure_management_bond || fail_setup "Failed to configure management bond"
 	fi
 
 	if [ "$address_type" = 'DHCP' ]; then
@@ -2228,6 +2269,13 @@ update_sudoers_for_testing() {
 }
 
 update_packages() {
+	# Pin physical NIC names by MAC BEFORE pulling packages, so the UEK8 kernel that
+	# the update below installs can't renumber the interfaces SO binds by name. Doing
+	# it here (instead of waiting for the common highstate) also drops the
+	# /opt/so/state/nic_names_pinned marker that gates the kernel repo, so the kernel
+	# repo is assigned on the very first highstate and the kernel isn't downgraded and
+	# then re-upgraded. Run-once: so-nic-pin no-ops if the marker already exists.
+	logCmd "bash ../salt/common/tools/sbin/so-nic-pin"
 	logCmd "dnf repolist"
 	logCmd "dnf -y update --allowerasing --exclude=salt*,docker*,containerd*"
 	RMREPOFILES=("oracle-linux-ol9.repo" "uek-ol9.repo" "virt-ol9.repo")

setup/so-setup

@@ -9,14 +9,17 @@
 # Make sure you are root before doing anything
 uid="$(id -u)"
 if [ "$uid" -ne 0 ]; then
-	echo "This script must be run using sudo!"
+	echo "This script must be run using sudo!" >&2
-	fail_setup
+	exit 1
 fi
 
 # Save the original argument array since we modify it
 original_args=("$@")
 
-cd "$(dirname "$0")" || fail_setup
+cd "$(dirname "$0")" || {
+	echo "Unable to change to setup directory" >&2
+	exit 1
+}
 
 echo "Getting started..."
 
@@ -87,8 +90,7 @@ if [[ "$setup_type" == 'iso' ]]; then
 	if [[ $is_rpm ]]; then
 		is_iso=true
 	else
-		echo "Only use 'so-setup iso' for an ISO install on Security Onion ISO images. Please run 'so-setup network' instead."
+		fail_setup "Only use 'so-setup iso' for an ISO install on Security Onion ISO images. Please run 'so-setup network' instead."
-		fail_setup
 	fi
 fi
 
@@ -127,7 +129,7 @@ catch() {
 	info "Fatal error occurred at $1 in so-setup, failing setup."
 	grep --color=never "ERROR" "$setup_log" > "$error_log"
 	whiptail_setup_failed
-	fail_setup
+	fail_setup "Fatal error occurred at $1 in so-setup"
 }
 
 # Add the progress function for manager node type installs
@@ -235,8 +237,7 @@ case "$setup_type" in
 		info "Beginning Security Onion $setup_type install"
 		;;
 	*)
-		error "Invalid install type, must be 'iso', 'network' or 'desktop'."
+		fail_setup "Invalid install type, must be 'iso', 'network' or 'desktop'."
-		fail_setup
 		;;
 esac
 
@@ -770,8 +771,7 @@ if ! [[ -f $install_opt_file ]]; then
 		logCmd "salt-call state.apply -l info registry"
 		title "Seeding the docker registry"
 		if ! docker_seed_registry; then
-			error "Failed to seed the docker registry"
+			fail_setup "Failed to seed the docker registry"
-			fail_setup
 		fi
 		title "Applying the manager state"
 		logCmd "salt-call state.apply -l info manager"
@@ -793,9 +793,8 @@ if ! [[ -f $install_opt_file ]]; then
 		logCmd "so-soc-restart"
 		title "Setting up Elastic Fleet"
 		logCmd "salt-call state.apply elasticfleet.config"	
-		if ! logCmd so-elastic-fleet-setup; then
+		if ! so-elastic-fleet-setup; then
-			error "Failed to run so-elastic-fleet-setup"
+			fail_setup "Failed to run so-elastic-fleet-setup"
-			fail_setup
 		fi
 		mark_setup_complete
 		set_initial_firewall_access

setup/so-verify

@@ -143,15 +143,15 @@ main() {
         cat $error_log
         echo "--------------------------"
         exit_code=1
-        touch /root/failure
+        echo "Found setup errors. Check $error_log for details" > /root/failure
     elif using_iso && cron_error_in_mail_spool; then
         echo "WARNING: Unexpected cron job output in mail spool"
         exit_code=1
-        touch /root/failure
+        echo "Unexpected cron job output found in /var/spool/mail/" > /root/failure
     elif is_manager_node && status_failed; then
         echo "WARNING: Containers are not in a healthy state"
         exit_code=1
-        touch /root/failure
+        echo "Containers are not in a healthy state. Check so-status for details" > /root/failure
     else
         echo "Successfully completed setup!"
         touch /root/success