Update baseline agents

ignore file already closed

salt/common/tools/sbin/so-kernel-upgrade

@@ -5,53 +5,239 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 #
-# so-kernel-upgrade — switch the boot default to the installed UEK8 (6.x) kernel.
+# so-kernel-upgrade — install the UEK8 (6.x) kernel and make it the boot default.
 #
-# Security Onion is moving off the EL9 stock kernel / UEK7 (5.x) onto UEK8 (6.x).
-# Installing the kernel-uek-core package adds a UEK8 boot entry but does NOT make it the
-# default: kernel-install/grubby only auto-promote a new kernel within the running
-# kernel's flavor lineage, and we're crossing from a 5.x kernel to the new 6.x UEK flavor.
-# So even with UPDATEDEFAULT=yes and DEFAULTKERNEL=kernel-uek-core the box keeps booting
-# the old kernel. This tool finds the newest installed 6.x UEK kernel and makes it the
-# GRUB default via grubby so the next boot comes up on UEK8.
+# Security Onion is moving off the EL9 stock kernel (RHCK, 5.14) and UEK7 (5.15) onto UEK8
+# (6.x). Three things have to happen, and the tool has to drive each one:
 #
-# Idempotent: if the UEK8 kernel is already the default it does nothing. It only sets the
-# boot default; it does NOT reboot — the admin reboots the node on their own schedule.
+#   1. Populate. The manager mirrors the UEK8 packages into /nsm/kernelrepo via so-repo-sync,
+#      and serves them to the grid over https://<manager>/kernelrepo. Until that sync runs the
+#      repo is valid but EMPTY -- dnf resolves it happily and installs nothing, with no error.
+#   2. Install. A node on RHCK has no kernel-uek* package at all, so there is nothing for
+#      'dnf update' to upgrade. A node on UEK7 does have kernel-uek installed, so
+#      'dnf install kernel-uek' reports "Nothing to do" and exits 0 without installing 6.x.
+#      Both cases need an explicit install of the UEK8 NEVRA.
+#   3. Boot it. Whether a newly installed UEK8 kernel becomes the boot default depends on the
+#      RUNNING kernel's flavor. kernel-install/grubby (with UPDATEDEFAULT=yes) only auto-promote
+#      within the running kernel's flavor lineage:
+#        - From UEK7 (5.x, kernel-uek) the install stays in the kernel-uek lineage and IS
+#          auto-promoted, so no grubby change is needed -- just make sure the repo is populated
+#          and install UEK8.
+#        - From the stock EL9 kernel (RHCK, 5.14, no UEK) it is a flavor CROSS that is NOT
+#          auto-promoted, so the box keeps booting RHCK until grubby is told otherwise.
+#      This tool inspects the running kernel and only runs 'grubby --set-default' for RHCK.
+#
+# Every one of those failure modes is silent by default. This tool handles each case and fails
+# loudly when it cannot, rather than reporting success while changing nothing.
+#
+# Manager vs minion: only the manager owns /nsm/kernelrepo, so only the manager can populate
+# it. If the repo is empty here, a manager runs so-repo-sync itself; a minion has no way to
+# fix it and exits non-zero telling the admin to sync the manager first.
+#
+# Idempotent: an already-installed, already-default UEK8 kernel is left alone. It only sets
+# the boot default; it does NOT reboot -- the admin reboots the node on their own schedule.
+
+. /usr/sbin/so-common
+
+# Client-side repo id (what dnf enables on this node, from repo/client/oracle.sls) vs the
+# reposync-side section in repodownload.conf that the manager mirrors from (mirrors the
+# securityonion/securityonionsync split for the main repo).
+KERNEL_REPO="securityonionkernel"
+KERNEL_REPO_SYNC="securityonionkernelsync"
+KERNEL_PKG="kernel-uek"
+KERNEL_REPO_DIR="/nsm/kernelrepo"
+REPOSYNC_CONF="/opt/so/conf/reposync/repodownload.conf"
+GLOBAL_PILLAR="/opt/so/saltstack/local/pillar/global/soc_global.sls"
 
 log() { echo "[so-kernel-upgrade] $*"; }
+die() { echo "[so-kernel-upgrade] ERROR: $*" >&2; exit 1; }
 
-[ "$(id -u)" -eq 0 ] || { log "must run as root"; exit 1; }
-command -v grubby >/dev/null 2>&1 || { log "grubby not found"; exit 1; }
+command -v grubby >/dev/null 2>&1 || die "grubby not found"
+command -v dnf >/dev/null 2>&1 || die "dnf not found"
+
+ARCH="$(rpm -E '%{_arch}')"
+
+is_airgap() {
+    [ -f "$GLOBAL_PILLAR" ] && grep -q 'airgap: *[Tt]rue' "$GLOBAL_PILLAR"
+}
 
 # Newest installed UEK8 (6.x) kernel known to the bootloader. UEK8 vmlinuz paths look like
-# /boot/vmlinuz-6.12.0-203.76.7.5.el9uek.x86_64; the 5.x UEK7 and 5.14 RHCK won't match.
-target="$(grubby --info=ALL 2>/dev/null \
-    | sed -n 's/^kernel="\(.*\)"$/\1/p' \
-    | grep -E '/vmlinuz-6\.[0-9]+.*uek' \
-    | sort -V | tail -1)"
+# /boot/vmlinuz-6.12.0-204.92.4.2.el9uek.x86_64; UEK7 (5.15) and RHCK (5.14) won't match.
+find_uek8() {
+    grubby --info=ALL 2>/dev/null \
+        | sed -n 's/^kernel="\(.*\)"$/\1/p' \
+        | grep -E '/vmlinuz-6\.[0-9]+.*uek' \
+        | sort -V | tail -1
+}
 
-if [ -z "$target" ]; then
-    log "no installed 6.x UEK (UEK8) kernel found — confirm the kernel repo is assigned and"
-    log "'dnf update' has installed kernel-uek-core. Nothing to do."
+# Classify the RUNNING kernel (uname -r) -- this, not what's installed, is what decides whether
+# a UEK8 install auto-promotes to the boot default:
+#   uek8  6.x UEK    already on the target line; nothing to do
+#   uek7  5.x UEK    a UEK8 install stays in the kernel-uek lineage and auto-promotes (no grubby)
+#   rhck  5.14 EL9   crossing into the UEK flavor does NOT auto-promote (needs grubby --set-default)
+running_flavor() {
+    case "$(uname -r)" in
+        6.*uek*) echo uek8 ;;
+        *uek*)   echo uek7 ;;
+        *)       echo rhck ;;
+    esac
+}
+
+# Newest UEK8 kernel-uek NEVRA offered by the kernel repo, empty if the repo has none.
+# Restricted to the kernel repo so a UEK7 kernel-uek in the main repo can't be picked up,
+# and filtered to 6.x so we never "succeed" by reinstalling the 5.15 we already have.
+uek8_available() {
+    dnf -q repoquery --disablerepo='*' --enablerepo="$KERNEL_REPO" \
+        --arch="$ARCH" --latest-limit=1 \
+        --qf '%{name}-%{evr}.%{arch}\n' "$KERNEL_PKG" 2>/dev/null \
+        | grep -E "^${KERNEL_PKG}-6\." | tail -1
+}
+
+kernelrepo_rpm_count() {
+    find "$KERNEL_REPO_DIR" -maxdepth 1 -name '*.rpm' 2>/dev/null | wc -l
+}
+
+# The kernel repo starts life as valid-but-empty (kernelrepo_init_empty in
+# salt/manager/init.sls) and is filled by so-repo-sync. During a soup, so-repo-sync runs
+# BEFORE the highstate deploys the [securityonionkernelsync] section into repodownload.conf, so
+# the first kernel-aware soup leaves the repo empty until the next nightly sync.
+sync_kernel_repo() {
+    if is_airgap; then
+        log "airgap install: $KERNEL_REPO_DIR is populated from the airgap ISO, not by so-repo-sync."
+        return 1
+    fi
+    if ! grep -q "^\[${KERNEL_REPO_SYNC}\]" "$REPOSYNC_CONF" 2>/dev/null; then
+        log "$REPOSYNC_CONF has no [${KERNEL_REPO_SYNC}] section -- run a highstate to deploy it."
+        return 1
+    fi
+
+    log "populating $KERNEL_REPO_DIR with so-repo-sync (mirrors upstream; can take several minutes)"
+    su socore -c '/usr/sbin/so-repo-sync' || { log "so-repo-sync failed"; return 1; }
+
+    dnf -q clean expire-cache >/dev/null 2>&1
+    return 0
+}
+
+# Make the kernel repo actually able to serve a UEK8 package, or fail trying.
+ensure_kernel_repo() {
+    # The repo is assigned by the repo.client highstate, and only once NICs are pinned by MAC
+    # (/opt/so/state/nic_names_pinned) so the kernel swap can't renumber interfaces SO binds
+    # by name. skip_if_unavailable=1 means a broken repo is silently ignored, so check first.
+    if ! dnf -q repolist --enabled 2>/dev/null | awk '{print $1}' | grep -qx "$KERNEL_REPO"; then
+        log "repo '$KERNEL_REPO' is not enabled on this node."
+        log "Run a highstate first; the repo is skipped until /opt/so/state/nic_names_pinned"
+        log "exists (run so-nic-pin) and this node's salt matches the version this release ships."
+        die "kernel repo unavailable"
+    fi
+
+    [ -n "$(uek8_available)" ] && return 0
+
+    log "repo '$KERNEL_REPO' is enabled but offers no UEK8 $KERNEL_PKG package"
+
+    if ! is_manager_node; then
+        log "This is a minion; it consumes the kernel repo from the manager and cannot populate it."
+        log "On the manager, run:  su socore -c /usr/sbin/so-repo-sync"
+        log "then re-run this script here."
+        die "manager's kernel repo is empty"
+    fi
+
+    log "this is a manager and $KERNEL_REPO_DIR holds $(kernelrepo_rpm_count) rpm(s)"
+    sync_kernel_repo || die "could not populate $KERNEL_REPO_DIR"
+
+    [ -n "$(uek8_available)" ] \
+        || die "so-repo-sync completed but $KERNEL_REPO still offers no UEK8 $KERNEL_PKG"
+}
+
+reboot_notice() {
+    [ "$(uname -r)" = "$(basename "$1" | sed 's/^vmlinuz-//')" ] \
+        || log "REBOOT REQUIRED to start using the UEK8 kernel (currently running $(uname -r))."
+}
+
+# Keep future kernel updates on the UEK line rather than falling back to RHCK. Oracle ships
+# /etc/sysconfig/kernel; only rewrite it when it's actually pointing somewhere else.
+set_default_kernel_conf() {
+    if [ -f /etc/sysconfig/kernel ] && ! grep -q '^DEFAULTKERNEL=kernel-uek-core$' /etc/sysconfig/kernel; then
+        log "setting DEFAULTKERNEL=kernel-uek-core in /etc/sysconfig/kernel"
+        sed -i 's/^DEFAULTKERNEL=.*/DEFAULTKERNEL=kernel-uek-core/' /etc/sysconfig/kernel
+    fi
+}
+
+# Make sure a UEK8 kernel is installed, leaving its boot entry in INSTALLED_UEK8. If one is
+# already present we leave the repo alone -- it may be disabled or empty and we don't need it
+# just to flip the boot default. Otherwise install the explicit NEVRA, not the bare package
+# name: on a UEK7 node 'dnf install kernel-uek' sees 5.15 already present, prints "Nothing to
+# do" and exits 0 without installing 6.x.
+ensure_uek8_installed() {
+    INSTALLED_UEK8="$(find_uek8)"
+    if [ -n "$INSTALLED_UEK8" ]; then
+        log "UEK8 kernel already installed: $INSTALLED_UEK8"
+        return 0
+    fi
+
+    ensure_kernel_repo
+    local nevra; nevra="$(uek8_available)"
+    log "installing $nevra from $KERNEL_REPO"
+    dnf -y install "$nevra" || die "failed to install $nevra"
+
+    INSTALLED_UEK8="$(find_uek8)"
+    [ -n "$INSTALLED_UEK8" ] || die "$nevra installed but no 6.x UEK boot entry appeared -- check 'grubby --info=ALL'"
+    log "installed UEK8 kernel: $INSTALLED_UEK8"
+}
+
+case "$(running_flavor)" in
+uek8)
+    # Already on the 6.x UEK line. A plain 'dnf update' keeps this node current within the
+    # lineage and auto-promotes newer builds, so there is nothing for this tool to do.
+    log "already running a UEK8 kernel ($(uname -r)); nothing to do."
     exit 0
-fi
+    ;;
 
-current="$(grubby --default-kernel 2>/dev/null)"
-if [ "$current" = "$target" ]; then
-    log "UEK8 kernel is already the boot default: $target"
-    exit 0
-fi
+uek7)
+    # On a 5.x UEK kernel. Installing UEK8 stays inside the kernel-uek lineage, so dnf/grubby
+    # (UPDATEDEFAULT=yes) auto-promote it and we do NOT touch grubby. A node still on UEK7
+    # usually means the kernel repo was empty when it last updated, so populate it and install.
+    log "running UEK7 kernel ($(uname -r)); the kernel repo was likely not yet populated when"
+    log "this node last updated. Populating it and installing UEK8 -- the update stays on the"
+    log "kernel-uek line, so it becomes the boot default automatically (no grubby change needed)."
+    set_default_kernel_conf
+    ensure_uek8_installed
 
-log "current default kernel: ${current:-unknown}"
-log "switching boot default to UEK8 kernel: $target"
-grubby --set-default="$target" || { log "ERROR: grubby --set-default failed for $target"; exit 1; }
+    now="$(grubby --default-kernel 2>/dev/null)"
+    if [ "$now" = "$INSTALLED_UEK8" ]; then
+        log "boot default auto-promoted to UEK8 kernel: $INSTALLED_UEK8"
+    else
+        log "WARNING: expected the UEK8 kernel to auto-promote but the default is still"
+        log "'${now:-unknown}'. Run 'grubby --set-default=$INSTALLED_UEK8' to force it."
+    fi
+    reboot_notice "$INSTALLED_UEK8"
+    ;;
 
-# Verify the change actually took before claiming success.
-now="$(grubby --default-kernel 2>/dev/null)"
-if [ "$now" != "$target" ]; then
-    log "ERROR: default kernel is still '${now:-unknown}' after set-default"
-    exit 1
-fi
+rhck)
+    # On the stock EL9 kernel (5.14, no UEK installed). Crossing from RHCK into the UEK flavor
+    # does NOT auto-promote -- kernel-install/grubby only auto-promote within the running
+    # kernel's flavor lineage -- so after installing we must set the boot default explicitly.
+    log "running stock EL9 (RHCK) kernel ($(uname -r)); installing UEK8 and setting it as the"
+    log "boot default explicitly (a RHCK->UEK flavor change does not auto-promote)."
+    set_default_kernel_conf
+    ensure_uek8_installed
+    target="$INSTALLED_UEK8"
 
-log "boot default is now $target"
-log "REBOOT REQUIRED to start using the UEK8 kernel (currently running $(uname -r))."
+    current="$(grubby --default-kernel 2>/dev/null)"
+    if [ "$current" = "$target" ]; then
+        log "UEK8 kernel is already the boot default: $target"
+        reboot_notice "$target"
+        exit 0
+    fi
+
+    log "current default kernel: ${current:-unknown}"
+    log "switching boot default to UEK8 kernel: $target"
+    grubby --set-default="$target" || die "grubby --set-default failed for $target"
+
+    # Verify the change actually took before claiming success.
+    now="$(grubby --default-kernel 2>/dev/null)"
+    [ "$now" = "$target" ] || die "default kernel is still '${now:-unknown}' after set-default"
+
+    log "boot default is now $target"
+    reboot_notice "$target"
+    ;;
+esac

salt/common/tools/sbin/so-restart

@@ -35,6 +35,9 @@ case $1 in
     "elastic-fleet"|"elasticfleet")
         docker_check_running "elastic-fleet" "--stop"
         docker rm "so-elastic-fleet" 2> /dev/null
+        # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy
+        rm -rf /opt/so/conf/elastic-fleet/state
+
         salt-call state.apply elasticfleet queue=True
         ;;
     *)

salt/common/tools/sbin/so-stop

@@ -29,6 +29,8 @@ case $1 in
 	"elasticfleet"|"elastic-fleet")
         docker_check_running "elastic-fleet" "--stop"
         docker rm "so-elastic-fleet" 2> /dev/null
+        # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy
+        rm -rf /opt/so/conf/elastic-fleet/state
         ;;
     *)
         docker_check_running "$1" "--stop"

salt/elasticfleet/defaults.yaml

@@ -1,6 +1,5 @@
 elasticfleet:
   enabled: False
-  patch_version: 9.3.3+build202604082258 # Elastic Agent specific patch release.
   enable_manager_output: True
   config:
     server:

salt/elasticfleet/enabled.sls

@@ -11,6 +11,10 @@
 
 {#   This value is generated during node install and stored in minion pillar #}
 {%   set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %}
+{#   Prevent Elastic Agent from re-enrolling with a new agent.id everytime the container starts up.
+       - if a fresh enrollment is needed use 'so-stop elasticfleet'
+       #}
+{%   set ENROLLED = salt['file.file_exists']('/opt/so/conf/elastic-fleet/state/fleet.enc') %}
 
 include:
   - ca
@@ -65,6 +69,7 @@ so-elastic-fleet:
       - /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro
       - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro
       - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro
+      - /opt/so/conf/elastic-fleet/state:/usr/share/elastic-agent/state
       - /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs 
      {% if DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
         {% for BIND in DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
@@ -72,6 +77,7 @@ so-elastic-fleet:
         {% endfor %}
       {% endif %}      
     - environment:
+      {% if not ENROLLED %}
       - FLEET_SERVER_ENABLE=true
       - FLEET_URL=https://{{ GLOBALS.hostname }}:8220
       - FLEET_SERVER_ELASTICSEARCH_HOST=https://{{ GLOBALS.manager }}:9200
@@ -81,6 +87,9 @@ so-elastic-fleet:
       - FLEET_SERVER_CERT_KEY=/etc/pki/elasticfleet-server.key
       - FLEET_CA=/etc/pki/tls/certs/intca.crt     
       - FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt
+      {% endif %}
+      - STATE_PATH=/usr/share/elastic-agent/state
+      - CONFIG_PATH=/usr/share/elastic-agent/state
       - LOGS_PATH=logs
       {% if DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
         {% for XTRAENV in DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
@@ -99,6 +108,7 @@ so-elastic-fleet:
       - x509: etc_elasticfleet_crt
     - require:
       - file: trusttheca
+      - file: eastatedir
       - x509: etc_elasticfleet_key
       - x509: etc_elasticfleet_crt
 

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -5,7 +5,7 @@
   "package": {
     "name": "endpoint",
     "title": "Elastic Defend",
-    "version": "9.3.0",
+    "version": "9.3.1",
     "requires_root": true
   },
   "enabled": true,

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -29,7 +29,7 @@
               "\\.gz$"
             ],
             "include_files": [],
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.15.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.15.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.15.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.20.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.3\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.20.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.20.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.3\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
             ],

salt/elasticfleet/install_agent_grid.sls

@@ -10,6 +10,15 @@
 {% set AGENT_STATUS = salt['service.available']('elastic-agent') %}
 {% set AGENT_EXISTS = salt['file.file_exists']('/opt/Elastic/Agent/elastic-agent') %}
 
+so-elastic-agent-install:
+  file.managed:
+    - name: /usr/sbin/so-elastic-agent-install
+    - source: salt://elasticfleet/tools/sbin/so-elastic-agent-install
+    - user: 947
+    - group: 939
+    - mode: 755
+    - show_changes: False
+
 {% if not AGENT_STATUS or not AGENT_EXISTS %}
 
 pull_agent_installer:
@@ -21,11 +30,9 @@ pull_agent_installer:
 
 run_installer:
   cmd.run:
-    - name: ./so-elastic-agent_linux_amd64 -token={{ GRIDNODETOKEN }} -force
-    - cwd: /opt/so
-    - retry:
-        attempts: 3
-        interval: 20
+    - name: /usr/sbin/so-elastic-agent-install "{{ GRIDNODETOKEN }}"
+    - require:
+      - file: pull_agent_installer
 
 cleanup_agent_installer:
   file.absent:

salt/elasticfleet/tools/sbin/so-elastic-agent-install

@@ -0,0 +1,100 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-elastic-fleet-common
+
+
+# passed in as arg from elasticfleet/install_agent_grid.sls, else pulled from pillar later
+GRIDNODETOKEN="$1"
+LOGFILE="/opt/so/SO-Elastic-Agent_Installer_Health.log"
+
+check_agent_health() {
+    timeout=300
+    interval=10
+    start=$SECONDS
+
+    while (( SECONDS - start < timeout )); do
+        agent_status=$(elastic-agent status 2>&1)
+        echo -e "\n$(date)\n$agent_status\n" >> "$LOGFILE"
+        if echo "$agent_status" | grep -A1 'elastic-agent$' | grep -q 'status: (HEALTHY)'; then
+            return 0
+        fi
+        echo "The Elastic Agent is not yet healthy. Waiting for ${interval} seconds before checking again..."
+        sleep "$interval"
+    done
+
+    echo "The Elastic Agent did not become healthy within ${timeout} seconds"
+    return 1
+}
+
+uninstall_agent() {
+    if command -v elastic-agent >/dev/null 2>&1; then
+        elastic-agent uninstall -f
+    fi
+}
+
+if [[ -z "$GRIDNODETOKEN" ]]; then
+    noderole=$(so-yaml.py get -r /etc/salt/grains role)
+    if [[ "$noderole" == "so-heavynode" ]]; then
+        GRIDNODETOKEN=$(salt-call pillar.get global:fleet_grid_enrollment_token_heavy --out=newline_values_only)
+    else
+        GRIDNODETOKEN=$(salt-call pillar.get global:fleet_grid_enrollment_token_general --out=newline_values_only)
+    fi
+fi
+
+if [[ -z "$GRIDNODETOKEN" ]]; then
+    echo "Unable to determine Elastic Fleet enrollment token. Exiting."
+
+    exit 1
+fi
+
+if [[ ! -x /opt/so/so-elastic-agent_linux_amd64 ]]; then
+    echo "Downloading so-elastic-agent installer... This could take a while if another Salt job is running."
+
+    # When running outside of elasticfleet/install_agent_grid.sls we need to download the installer independently.
+    #  PYTHONWARNINGS="ignore" to avoid messages like the following when running salt-call:
+    #  '/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/base.py:129: TransportWarning: Unclosed transport! <salt.transport.zeromq.RequestClient object at 0x7fc5f0ee7a30>
+    #    File "/bin/salt-call", line 12, in <module>
+    #      sys.exit(salt_call())'
+
+    PYTHONWARNINGS="ignore" salt-call state.single file.managed name=/opt/so/so-elastic-agent_linux_amd64 source=salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64 mode=755 makedirs=True queue=True
+
+fi
+
+if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then
+    attempts=0
+    cd /opt/so/ || exit 1
+
+    truncate -s 0 "$LOGFILE"
+
+    uninstall_agent
+
+    while [[ $attempts -lt 3 ]]; do
+        if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force && echo "Verifying Elastic Agent health..." && check_agent_health; then
+            rm -f /opt/so/so-elastic-agent_linux_amd64
+            elastic-agent status
+
+            exit 0
+        fi
+
+        attempts=$((attempts + 1))
+
+        if [[ $attempts -lt 3 ]]; then
+            echo "Unable to verify Elastic Agent health... Retrying in 20 seconds..."
+            sleep 20
+        fi
+    done
+
+    uninstall_agent
+    rm -f /opt/so/so-elastic-agent_linux_amd64
+    echo "The so-elastic-agent installer failed after 3 attempts. Exiting."
+
+    exit 1
+else
+    echo "Unable to locate so-elastic-agent installer. Exiting."
+
+    exit 1
+fi

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers

@@ -30,7 +30,7 @@ done
 if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then 
   printf "\nFleet Host URL, Enrollment Token or Elastic Version empty - exiting..."
   printf "\nFleet Host: $FLEETHOST, Enrollment Token: $ENROLLMENTOKEN\n"
-  exit 
+  exit 1
 fi
 
 OSARCH=( "linux-x86_64" "windows-x86_64" "darwin-x86_64" "darwin-aarch64" )
@@ -62,31 +62,54 @@ do
 done 
 
 GOTARGETOS=( "linux" "windows" "darwin" "darwin/arm64" )
-GOARCH="amd64"
 printf "\n### Generating OS packages using the cleaned up tarballs"
-for GOOS in "${GOTARGETOS[@]}"
-do
+for GOOS in "${GOTARGETOS[@]}"; do
+    GOARCH="amd64"
     if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi
     printf "\n\n### Generating $GOOS/$GOARCH Installer...\n"
     docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \
     --mount type=bind,source=/etc/pki/tls/certs/,target=/workspace/files/cert/ \
     --mount type=bind,source=/nsm/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \
-    --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \
+    --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ \
     {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN"  -o /output/so-elastic-agent_${GOOS}_${GOARCH}
     printf "\n### $GOOS/$GOARCH Installer Generated...\n"
 done
 
 printf "\n\n### Generating MSI...\n"
-cp /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe
+cp /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe
 docker run \
---mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ -w /output \
+--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ -w /output \
 {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs
 printf "\n### MSI Generated...\n"
 
+# Verify installers were created
+for GOOS in "${GOTARGETOS[@]}"; do
+    GOARCH="amd64"
+    if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin"; GOARCH="arm64"; fi
+    if [[ ! -f /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_${GOOS}_${GOARCH} ]]; then
+        printf "\n### ERROR: Installer for %s/%s was not generated. Exiting...\n" "$GOOS" "$GOARCH"
+        exit 1
+    fi
+    # After verifying new installer was generated, move it to so_agent-installers directory
+    mv /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_${GOOS}_${GOARCH} /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/
+done
+
+# Verify MSI installer
+if [[ ! -f /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64_msi ]]; then
+    printf "\n### ERROR: Installer MSI was not generated. Exiting...\n"
+    exit 1
+else
+    # After verifying new installer MSI was generated, move it to so_agent-installers directory
+    mv /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64_msi /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/
+fi
+
 printf "\n### Cleaning up temp files \n"
 rm -rf /nsm/elastic-agent-workspace
-rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe
+rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe
 
 printf "\n### Copying so_agent-installers to /nsm/elastic-fleet/ for nginx.\n"
 \cp -vr /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ /nsm/elastic-fleet/
 chmod 644 /nsm/elastic-fleet/so_agent-installers/*
+
+# if we got here all installers have been generated successfully
+exit 0

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -244,11 +244,37 @@ printf '%s\n'\
     "" >> "$global_pillar_file"
 
 # Call Elastic-Fleet Salt State
-printf "\nApplying elasticfleet state"
-salt-call state.apply elasticfleet queue=True
+printf "\nApplying elasticfleet state\n"
+for state_attempt in {1..3}; do
+    if salt-call state.apply elasticfleet queue=True; then
+        break
+    elif [[ $state_attempt -lt 3 ]]; then
+        printf "\nElasticfleet state did not complete successfully... Attempt (%s/3). Retrying...\n" "$state_attempt"
+        sleep 10
+    else
+        printf "\nFailure(s) in elasticfleet state... Exiting...\n"
+        exit 1
+    fi
+done
 
+printf "\nRunning so-elastic-agent-gen-installers\n"
 # Generate installers & install Elastic Agent on the node
-so-elastic-agent-gen-installers
-printf "\nApplying elasticfleet.install_agent_grid state"
-salt-call state.apply elasticfleet.install_agent_grid queue=True
-exit 0
+for agent_gen_attempt in {1..3}; do
+    if so-elastic-agent-gen-installers; then
+        break
+    elif [[ $agent_gen_attempt -lt 3 ]]; then
+        printf "\nUnable to generate Elastic Agent installers... Attempt (%s/3). Retrying...\n" "$agent_gen_attempt"
+        sleep 10
+    else
+        printf "\nFailed to generate Elastic Agent installers after 3 attempts. Exiting...\n"
+        exit 1
+    fi
+done
+
+printf "\nApplying elasticfleet.install_agent_grid state\n"
+if ! salt-call state.apply elasticfleet.install_agent_grid queue=True; then
+    printf "\nFailure(s) in elasticfleet.install_agent_grid state... Exiting...\n"
+    exit 1
+fi
+
+printf "\nElastic Fleet setup completed successfully\n"

salt/elasticsearch/defaults.yaml

@@ -1,6 +1,6 @@
 elasticsearch:
   enabled: false
-  version: 9.3.3
+  version: 9.3.7
   index_clean: true
   data_retention_method: DLM
   vm:

salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.4

@@ -118,70 +118,70 @@
     {
       "pipeline": {
         "tag": "pipeline_e16851a7",
-        "name": "logs-pfsense.log-1.25.2-firewall",
+        "name": "logs-pfsense.log-1.25.4-firewall",
         "if": "ctx.event.provider == 'filterlog'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_828590b5",
-        "name": "logs-pfsense.log-1.25.2-openvpn",
+        "name": "logs-pfsense.log-1.25.4-openvpn",
         "if": "ctx.event.provider == 'openvpn'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_9d37039c",
-        "name": "logs-pfsense.log-1.25.2-ipsec",
+        "name": "logs-pfsense.log-1.25.4-ipsec",
         "if": "ctx.event.provider == 'charon'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_ad56bbca",
-        "name": "logs-pfsense.log-1.25.2-dhcp",
+        "name": "logs-pfsense.log-1.25.4-dhcp",
         "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\", \"dnsmasq-dhcp\"].contains(ctx.event.provider)"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_dd85553d",
-        "name": "logs-pfsense.log-1.25.2-unbound",
+        "name": "logs-pfsense.log-1.25.4-unbound",
         "if": "ctx.event.provider == 'unbound'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_720ed255",
-        "name": "logs-pfsense.log-1.25.2-haproxy",
+        "name": "logs-pfsense.log-1.25.4-haproxy",
         "if": "ctx.event.provider == 'haproxy'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_456beba5",
-        "name": "logs-pfsense.log-1.25.2-php-fpm",
+        "name": "logs-pfsense.log-1.25.4-php-fpm",
         "if": "ctx.event.provider == 'php-fpm'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_a0d89375",
-        "name": "logs-pfsense.log-1.25.2-squid",
+        "name": "logs-pfsense.log-1.25.4-squid",
         "if": "ctx.event.provider == 'squid'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_c2f1ed55",
-        "name": "logs-pfsense.log-1.25.2-snort",
+        "name": "logs-pfsense.log-1.25.4-snort",
         "if": "ctx.event.provider == 'snort'"
       }
     },
     {
       "pipeline": {
         "tag":"pipeline_33db1c9e",
-        "name": "logs-pfsense.log-1.25.2-suricata",
+        "name": "logs-pfsense.log-1.25.4-suricata",
         "if": "ctx.event.provider == 'suricata'"
       }
     },

salt/elasticsearch/soc_elasticsearch.yaml

@@ -28,14 +28,14 @@ elasticsearch:
       description: The maximum number of memory map areas a process may use. Elasticsearch uses a mmapfs directory by default to store its indices. The default operating system limits on mmap counts could be too low, which may result in out of memory exceptions.
       forcedType: int
       helpLink: elasticsearch
-  retention: 
+  retention:
     retention_pct:
       decription: Total percentage of space used by Elasticsearch for multi node clusters
       helpLink: elasticsearch
       global: True
   config:
     cluster:
-      name: 
+      name:
         description: The name of the Security Onion Elasticsearch cluster, for identification purposes.
         readonly: True
         global: True
@@ -55,13 +55,13 @@ elasticsearch:
               forcedType: bool
               helpLink: elasticsearch
             watermark:
-              low: 
+              low:
                 description: The lower percentage of used disk space representing a healthy node.
                 helpLink: elasticsearch
-              high: 
+              high:
                 description: The higher percentage of used disk space representing an unhealthy node.
                 helpLink: elasticsearch
-              flood_stage: 
+              flood_stage:
                 description: The max percentage of used disk space that will cause the node to take protective actions, such as blocking incoming events.
                 helpLink: elasticsearch
     action:
@@ -172,11 +172,11 @@ elasticsearch:
                 forcedType: int
                 global: True
                 helpLink: elasticsearch
-              refresh_interval: 
+              refresh_interval:
                 description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation.
                 global: True
                 helpLink: elasticsearch
-              number_of_shards: 
+              number_of_shards:
                 description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs.
                 global: True
                 helpLink: elasticsearch
@@ -269,7 +269,7 @@ elasticsearch:
                   global: True
                   advanced: True
           warm:
-            min_age: 
+            min_age:
               description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
               regex: ^[0-9]{1,5}d$
               forcedType: string
@@ -370,7 +370,7 @@ elasticsearch:
         template:
           settings:
             index:
-              number_of_replicas: 
+              number_of_replicas:
                 description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs.
                 forcedType: int
                 global: True
@@ -391,12 +391,12 @@ elasticsearch:
                     global: True
                     advanced: True
                     helpLink: elasticsearch
-              refresh_interval: 
+              refresh_interval:
                 description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation.
                 global: True
                 advanced: True
                 helpLink: elasticsearch
-              number_of_shards: 
+              number_of_shards:
                 description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs.
                 global: True
                 advanced: True
@@ -645,6 +645,7 @@ elasticsearch:
             global: True
             advanced: True
             helpLink: elasticsearch
+    so-logs-soc: *dataStreamSettings
     so-logs-system_x_auth: *dataStreamSettings
     so-logs-system_x_syslog: *dataStreamSettings
     so-logs-system_x_system: *dataStreamSettings

salt/kibana/defaults.yaml

@@ -22,7 +22,7 @@ kibana:
           - default
           - file
     migrations:
-      discardCorruptObjects: "9.3.3"
+      discardCorruptObjects: "9.3.7"
     telemetry:
       enabled: False
     xpack:

salt/manager/tools/sbin/soup

@@ -12,7 +12,17 @@
 UPDATE_DIR=/tmp/sogh/securityonion
 DEFAULT_SALT_DIR=/opt/so/saltstack/default
 INSTALLEDVERSION=$(cat /etc/soversion)
-POSTVERSION=$INSTALLEDVERSION
+# /etc/sopostversion is a soup-owned marker (no salt state manages it) tracking how
+# far the post-upgrade walk has progressed. Its presence means a prior upgrade did
+# not finish its post-upgrade steps; its contents are the resume point. It is read
+# here before preupgrade_changes mutates INSTALLEDVERSION and before any highstate
+# stamps /etc/soversion from the pillar.
+POSTVERSION_FILE=/etc/sopostversion
+if [ -f "$POSTVERSION_FILE" ]; then
+  POSTVERSION=$(cat "$POSTVERSION_FILE")
+else
+  POSTVERSION=$INSTALLEDVERSION
+fi
 INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
 BATCHSIZE=5
 SOUP_LOG=/root/soup.log
@@ -23,6 +33,10 @@ NOTIFYCUSTOMELASTICCONFIG=false
 TOPFILE=/opt/so/saltstack/default/salt/top.sls
 BACKUPTOPFILE=/opt/so/saltstack/default/salt/top.sls.backup
 SALTUPGRADED=false
+# Set true once soup begins modifying the system (past the pre-flight checks), so the
+# EXIT trap can tell the user the update did not finish and must be re-run. Only the
+# pre-flight gates (ES compatibility, disk, network) fail before this is set.
+SOUP_UPGRADE_STARTED=false
 SALT_CLOUD_INSTALLED=false
 SALT_CLOUD_CONFIGURED=false
 # Check if salt-cloud is installed
@@ -123,6 +137,28 @@ check_err() {
 
     echo "SOUP XTRACE debug log (if enabled) at $SOUP_DEBUG_LOG. Re-run soup with SOUP_DEBUG=1 to create $SOUP_DEBUG_LOG"
 
+    # If soup had already started modifying the system, make it unmistakable that the
+    # update is incomplete and must be re-run. soup is resumable: a version upgrade
+    # picks up from the /etc/sopostversion marker, and a hotfix re-applies because
+    # /etc/sohotfix is only advanced after a successful highstate.
+    if [[ "$SOUP_UPGRADE_STARTED" == "true" ]]; then
+      echo ""
+      echo "=============================================================================="
+      echo " UPGRADE INCOMPLETE"
+      echo "=============================================================================="
+      echo " This soup run did NOT finish. Your Security Onion installation may be in a"
+      echo " partially-updated state and is not yet fully upgraded."
+      echo ""
+      echo " Review the error above and $SOUP_LOG, resolve the underlying problem, then"
+      echo " run soup again to resume and complete the update:"
+      echo ""
+      echo "     sudo soup"
+      echo ""
+      echo " soup is resumable -- re-running it continues from where this run stopped."
+      echo "=============================================================================="
+      echo ""
+    fi
+
     exit $exit_code
   fi
 
@@ -291,6 +327,30 @@ check_pillar_items() {
   fi
 }
 
+check_cluster_health() {
+  echo "Checking Elasticsearch cluster health."
+  # Require a 'green' cluster before upgrading; anything less (yellow, red, or
+  # unreachable) blocks. Modeled on the wait used in so-elasticsearch-roles-load.
+  if so-elasticsearch-query "_cluster/health?wait_for_status=green&timeout=120s" --fail > /dev/null 2>&1; then
+    printf "\nThe Elasticsearch cluster is healthy (green). We can proceed with SOUP.\n\n"
+  else
+    printf "\nThe Elasticsearch cluster is not green. Please resolve the cluster health issue so the cluster is green before running SOUP again.\n\n"
+    exit 0
+  fi
+}
+
+check_fleet_server() {
+  echo "Checking that Elastic Fleet Server is responding."
+  # Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls,
+  # which waits for HTTP 200 from the Fleet Server status API.
+  if curl -sk --fail --retry 3 --retry-delay 10 --max-time 30 "https://localhost:8220/api/status" > /dev/null 2>&1; then
+    printf "\nElastic Fleet Server is responding. We can proceed with SOUP.\n\n"
+  else
+    printf "\nElastic Fleet Server is not responding at https://localhost:8220/api/status. Please ensure Elastic Fleet is healthy before running SOUP again.\n\n"
+    exit 0
+  fi
+}
+
 check_saltmaster_status() {
   set +e
   echo "Waiting on the Salt Master service to be ready."
@@ -377,6 +437,8 @@ get_soup_script_hashes() {
   GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}')
   CURRENTSOFIREWALL=$(md5sum /usr/sbin/so-firewall | awk '{print $1}')
   GITSOFIREWALL=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-firewall | awk '{print $1}')
+  CURRENTSOYAML=$(md5sum /usr/sbin/so-yaml.py | awk '{print $1}')
+  GITSOYAML=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py | awk '{print $1}')
 }
 
 highstate() {
@@ -414,6 +476,13 @@ preupgrade_changes() {
     true
 }
 
+set_postversion() {
+    # Persist post-upgrade walk progress so an interrupted upgrade can resume the
+    # remaining steps on the next soup run (see /etc/sopostversion handling).
+    POSTVERSION="$1"
+    echo "$POSTVERSION" > "$POSTVERSION_FILE"
+}
+
 postupgrade_changes() {
     # This function is to add any new pillar items if needed.
     echo "Running post upgrade processes."
@@ -421,6 +490,8 @@ postupgrade_changes() {
     [[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0
     [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0
     [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0
+    # All applicable post-upgrade steps completed; clear the resume marker.
+    rm -f "$POSTVERSION_FILE"
     true
 }
 
@@ -513,7 +584,7 @@ post_to_3.0.0() {
   # convert yes/no in suricata pillars to true/false
   convert_suricata_yes_no
 
-  POSTVERSION=3.0.0
+  set_postversion 3.0.0
 }
 
 ### 3.0.0 End ###
@@ -740,7 +811,6 @@ fix_logstash_0013_lumberjack_pipeline_name() {
 up_to_3.1.0() {
   ensure_postgres_local_pillar
   ensure_postgres_secret
-  determine_elastic_agent_upgrade
   elasticsearch_backup_index_templates
   # Clear existing component template state file.
   rm -f /opt/so/state/esfleet_component_templates.json
@@ -777,7 +847,7 @@ post_to_3.1.0() {
   # Check for unhealthy / unauthorized integration transform jobs and attempt reauthorizations
   check_transform_health_and_reauthorize || true
 
-  POSTVERSION=3.1.0
+  set_postversion 3.1.0
 }
 
 ### 3.1.0 End ###
@@ -888,6 +958,9 @@ update_kafka_metadata() {
 }
 
 up_to_3.2.0() {
+  # download 9.3.7 elastic agent packages
+  determine_elastic_agent_upgrade
+
   fix_logstash_0013_lumberjack_pipeline_name
 
   pin_elasticsearch_data_retention_method
@@ -901,7 +974,7 @@ post_to_3.2.0() {
 
   bootstrap_so_soc_database
 
-  # Including agent regen script here since it was missed in post_to_3.1.0
+  # Generate 9.3.7 elastic agent installers
   echo "Regenerating Elastic Agent Installers"
   /sbin/so-elastic-agent-gen-installers
 
@@ -909,7 +982,7 @@ post_to_3.2.0() {
 
   update_kafka_metadata "4.3"
 
-  POSTVERSION=3.2.0
+  set_postversion 3.2.0
 }
 
 ### 3.2.0 End ###
@@ -1061,8 +1134,20 @@ upgrade_check() {
   fi
   [[ -f /etc/sohotfix ]] && CURRENTHOTFIX=$(cat /etc/sohotfix)
   if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then
+    # A leftover post-version marker means a previous upgrade to this version
+    # advanced /etc/soversion (the highstate stamps it from the pillar) but did not
+    # finish its post-upgrade steps. Resume the upgrade instead of reporting "latest".
+    if [ -f "$POSTVERSION_FILE" ] && [ "$(cat "$POSTVERSION_FILE")" != "$NEWVERSION" ]; then
+      echo "A previous upgrade to $NEWVERSION did not complete its post-upgrade steps; resuming."
+      is_hotfix=false
+      return 0
+    fi
     echo "Checking to see if there are hotfixes needed"
     if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then
+      # Reaching here means we are at the target version and NOT resuming (the resume
+      # check above returned otherwise). Clear any stale resume marker so a completed
+      # upgrade is never mistaken for a partial one and re-run on a later invocation.
+      rm -f "$POSTVERSION_FILE"
       echo "You are already running the latest version of Security Onion."
       exit 0
     else
@@ -1141,7 +1226,7 @@ upgrade_salt() {
 
 verify_latest_update_script() {
   get_soup_script_hashes
-  if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then
+  if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" && "$CURRENTSOYAML" == "$GITSOYAML" ]]; then
     echo "This version of the soup script is up to date. Proceeding."
   else
     echo "You are not running the latest soup version. Updating soup and its components. This might take multiple runs to complete."
@@ -1150,7 +1235,7 @@ verify_latest_update_script() {
 
     # Verify that soup scripts updated as expected
     get_soup_script_hashes
-    if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then
+    if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" && "$CURRENTSOYAML" == "$GITSOYAML" ]]; then
       echo "Succesfully updated soup scripts."
     else
       echo "There was a problem updating soup scripts. Trying to rerun script update."
@@ -1174,7 +1259,8 @@ verify_es_version_compatibility() {
         ["8.18.4"]="8.18.6 8.18.8 9.0.8"
 	    ["8.18.6"]="8.18.8 9.0.8"
 	    ["8.18.8"]="9.0.8"
-        ["9.0.8"]="9.3.3"
+        ["9.0.8"]="9.3.3 9.3.7"
+        ["9.3.3"]="9.3.7"
     )
 
     # Elasticsearch MUST upgrade through these versions
@@ -1757,6 +1843,15 @@ main() {
   set_minionid
   MINION_ROLE=$(lookup_role)
   echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
+  # /etc/soversion is stamped to the target version before the upgrade fully
+  # completes, so a lingering resume marker means this grid is only partially
+  # upgraded even though the line above shows the target version. Make that explicit
+  # so it is not mistaken for a finished upgrade.
+  if [ -f "$POSTVERSION_FILE" ] && [ "$(cat "$POSTVERSION_FILE")" != "$INSTALLEDVERSION" ]; then
+    echo ""
+    echo "NOTE: A previous upgrade to $INSTALLEDVERSION did not finish. This grid is"
+    echo "      partially upgraded and this soup run will resume and complete it."
+  fi
   echo ""
   check_minimum_version
 
@@ -1785,6 +1880,12 @@ main() {
   echo "Verifying Elasticsearch version compatibility across the grid before upgrading."
   verify_es_version_compatibility
 
+  # Pre-flight health checks: confirm the grid is in a good state before we change
+  # anything. These run before any modifications, so a failure exits cleanly and the
+  # operator can fix the issue and re-run soup.
+  check_cluster_health
+  check_fleet_server
+
   echo "Checking for Salt Master and Minion updates."
   upgrade_check_salt
   set -e
@@ -1801,6 +1902,7 @@ main() {
   fi
 
   if [ "$is_hotfix" == "true" ]; then
+    SOUP_UPGRADE_STARTED=true
     echo "Applying $HOTFIXVERSION hotfix"
     # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars
     if [[ ! "$MINION_ROLE" == "import" ]]; then
@@ -1811,10 +1913,16 @@ main() {
     create_local_directories "/opt/so/saltstack/default"
     apply_hotfix
     echo "Hotfix applied"
-    update_version
     enable_highstate
     highstate
+    # Record the hotfix only after the highstate succeeds. /etc/sohotfix is written
+    # solely by soup (no salt state manages it), so deferring the write means a failed
+    # hotfix highstate leaves the old hotfix value and re-running soup re-applies it,
+    # rather than reporting "already latest". The soversion/pillar writes in
+    # update_version are no-ops here since the version is unchanged for a hotfix.
+    update_version
   else
+    SOUP_UPGRADE_STARTED=true
     echo ""
     echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
     echo ""
@@ -1870,6 +1978,10 @@ main() {
     copy_new_files
     echo ""
     create_local_directories "/opt/so/saltstack/default"
+    # Seed the resume marker before the highstate stamps /etc/soversion to the new
+    # version, so an interrupted upgrade is detectable as "not finished" on re-run.
+    # POSTVERSION still holds the pre-upgrade (or prior resume) version here.
+    [ -f "$POSTVERSION_FILE" ] || echo "$POSTVERSION" > "$POSTVERSION_FILE"
     update_version
 
     echo ""

salt/soc/defaults.yaml

@@ -1531,7 +1531,8 @@ soc:
           agentic: false
           agentMapping:
             Orchestrator: sonnet
-            Hunter: sonnet
+            Investigator: sonnet
+            Detection Engineer: sonnet
         onionconfig:
           saltstackDir: /opt/so/saltstack
           bypassEnabled: false

salt/soc/soc_soc.yaml

@@ -783,8 +783,11 @@ soc:
             Orchestrator:
               description: The initial agent in most agentic conversations. This agent will delegate requests to specialized agents.
               global: True
-            Hunter:
-              description: This agent is specialized in querying events.
+            Investigator:
+              description: This agent investigates alerts, explains events and records, and hunts through event data. It can also acknowledge alerts and escalate to cases.
+              global: True
+            Detection Engineer:
+              description: This agent manages detections and their overrides, including tuning noisy rules and authoring rule content.
               global: True
       client:
         assistant:

salt/suricata/soc_suricata.yaml

@@ -153,12 +153,12 @@ suricata:
       cpu-affinity:
         management-cpu-set:
           cpu:
-            description: Bind management threads to a core or range of cores. This can be a sigle core, list of cores, or list of range of cores. set-cpu-affinity must be set to true for this to be used.
+            description: Bind management threads to a core or range of cores. This can be a single core, list of cores, or list of range of cores. set-cpu-affinity must be set to true for this to be used.
             forcedType: "[]string"
             helpLink: suricata
         worker-cpu-set:
           cpu:
-            description: Bind worker threads to a core or range of cores. This can be a sigle core, list of cores, or list of range of cores. set-cpu-affinity must be set to true for this to be used.
+            description: Bind worker threads to a core or range of cores. This can be a single core, list of cores, or list of range of cores. set-cpu-affinity must be set to true for this to be used.
             forcedType: "[]string"
             helpLink: suricata
     vars:

setup/so-setup

@@ -793,7 +793,7 @@ if ! [[ -f $install_opt_file ]]; then
 		logCmd "so-soc-restart"
 		title "Setting up Elastic Fleet"
 		logCmd "salt-call state.apply elasticfleet.config"	
-		if ! logCmd so-elastic-fleet-setup; then
+		if ! so-elastic-fleet-setup; then
 			fail_setup "Failed to run so-elastic-fleet-setup"
 		fi
 		mark_setup_complete