clear stale docker network endpoints before starting containers

docker_container.running fails with an address-already-in-use error when a
container holds a stale sobridge endpoint - either after its IP changes in
pillar or after an unclean shutdown leaves the endpoint behind.

Add the clear_stale_endpoint macro and call it from every enabled.sls whose
container attaches to sobridge with a static IP. The macro force-disconnects
the container from the network only when its current endpoint IP does not
match the one salt is about to assign, so it is a no-op on a normal highstate.

The strelka states use state IDs that differ from their container names, so
those calls pass state_id explicitly. nginx resolves its IP through
container_config, which varies by role.

pillar/data/addtotab.sh

@@ -1,59 +0,0 @@
-#!/usr/bin/env bash
-
-# This script adds sensors/nodes/etc to the nodes tab
-default_salt_dir=/opt/so/saltstack/default
-local_salt_dir=/opt/so/saltstack/local
-TYPE=$1
-NAME=$2
-IPADDRESS=$3
-CPUS=$4
-GUID=$5
-MANINT=$6
-ROOTFS=$7
-NSM=$8
-MONINT=$9
-#NODETYPE=$10
-#HOTNAME=$11
-
-echo "Seeing if this host is already in here. If so delete it"
-if grep -q $NAME "$local_salt_dir/pillar/data/$TYPE.sls"; then
-  echo "Node Already Present - Let's re-add it"
-  awk -v blah="  $NAME:" 'BEGIN{ print_flag=1 }
-{
-    if( $0 ~ blah )
-    {
-       print_flag=0;
-       next
-    }
-    if( $0 ~ /^  [a-zA-Z0-9]+:$/ )
-    {
-        print_flag=1;
-    }
-    if ( print_flag == 1 )
-        print $0
-
-} ' $local_salt_dir/pillar/data/$TYPE.sls > $local_salt_dir/pillar/data/tmp.$TYPE.sls
-mv $local_salt_dir/pillar/data/tmp.$TYPE.sls $local_salt_dir/pillar/data/$TYPE.sls
-echo "Deleted $NAME from the tab. Now adding it in again with updated info"
-fi
-echo "  $NAME:" >> $local_salt_dir/pillar/data/$TYPE.sls
-echo "    ip: $IPADDRESS" >> $local_salt_dir/pillar/data/$TYPE.sls
-echo "    manint: $MANINT" >> $local_salt_dir/pillar/data/$TYPE.sls
-echo "    totalcpus: $CPUS" >> $local_salt_dir/pillar/data/$TYPE.sls
-echo "    guid: $GUID" >> $local_salt_dir/pillar/data/$TYPE.sls
-echo "    rootfs: $ROOTFS" >> $local_salt_dir/pillar/data/$TYPE.sls
-echo "    nsmfs: $NSM" >> $local_salt_dir/pillar/data/$TYPE.sls
-if [ $TYPE == 'sensorstab' ]; then
-  echo "    monint: bond0" >> $local_salt_dir/pillar/data/$TYPE.sls
-fi
-if [ $TYPE == 'evaltab' ] || [ $TYPE == 'standalonetab' ]; then
-  echo "    monint: bond0" >> $local_salt_dir/pillar/data/$TYPE.sls
-  if [ ! $10 ]; then
-    salt-call state.apply utility queue=True
-  fi
-fi
-if [ $TYPE == 'nodestab' ]; then
-  salt-call state.apply elasticsearch queue=True
-#  echo "    nodetype: $NODETYPE" >> $local_salt_dir/pillar/data/$TYPE.sls
-#  echo "    hotname: $HOTNAME" >> $local_salt_dir/pillar/data/$TYPE.sls
-fi

salt/allowed_states.map.jinja

@@ -37,8 +37,7 @@
     'elasticfleet',
     'elasticfleet.manager',
     'elasticsearch.cluster',
-    'elastic-fleet-package-registry',
+    'elastic-fleet-package-registry'
-    'utility'
 ] %}
 
 {% set sensor_states = [

salt/common/tools/sbin/so-restart

@@ -35,6 +35,9 @@ case $1 in
     "elastic-fleet"|"elasticfleet")
         docker_check_running "elastic-fleet" "--stop"
         docker rm "so-elastic-fleet" 2> /dev/null
+        # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy
+        rm -rf /opt/so/conf/elastic-fleet/state
+
         salt-call state.apply elasticfleet queue=True
         ;;
     *)

salt/common/tools/sbin/so-stop

@@ -29,6 +29,8 @@ case $1 in
 	"elasticfleet"|"elastic-fleet")
         docker_check_running "elastic-fleet" "--stop"
         docker rm "so-elastic-fleet" 2> /dev/null
+        # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy
+        rm -rf /opt/so/conf/elastic-fleet/state
         ;;
     *)
         docker_check_running "$1" "--stop"

salt/docker/macros/docker_endpoint.jinja

@@ -0,0 +1,12 @@
+{% macro clear_stale_endpoint(container, network, ipv4, state_id=None) %}
+{{ container }}_{{ network }}_stale_endpoint:
+  cmd.run:
+    - name: docker network disconnect -f {{ network }} {{ container }} || true
+    - onlyif: docker inspect {{ container }}
+    - unless: >-
+        docker inspect -f
+        '{{ '{{' }} with index .NetworkSettings.Networks "{{ network }}" {{ '}}' }}{{ '{{' }} .IPAMConfig.IPv4Address {{ '}}' }}{{ '{{' }} end {{ '}}' }}'
+        {{ container }} 2>/dev/null | grep -qx '{{ ipv4 }}'
+    - require_in:
+      - docker_container: {{ state_id or container }}
+{% endmacro %}

salt/elastalert/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-elastalert', 'sobridge', DOCKERMERGED.containers['so-elastalert'].ip) }}
 
 include:
   - elastalert.config

salt/elastic-fleet-package-registry/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-elastic-fleet-package-registry', 'sobridge', DOCKERMERGED.containers['so-elastic-fleet-package-registry'].ip) }}
 
 include:
   - elastic-fleet-package-registry.config

salt/elasticagent/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-elastic-agent', 'sobridge', DOCKERMERGED.containers['so-elastic-agent'].ip) }}
 
 include:
   - ca

salt/elasticfleet/enabled.sls

@@ -8,9 +8,14 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
 
 {#   This value is generated during node install and stored in minion pillar #}
 {%   set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %}
+{#   Prevent Elastic Agent from re-enrolling with a new agent.id everytime the container starts up.
+       - if a fresh enrollment is needed use 'so-stop elasticfleet'
+       #}
+{%   set ENROLLED = salt['file.file_exists']('/opt/so/conf/elastic-fleet/state/fleet.enc') %}
 
 include:
   - ca
@@ -39,6 +44,8 @@ elasticagent_syncartifacts:
 {% endif %}
 
 {%   if SERVICETOKEN != '' %}
+{{ clear_stale_endpoint('so-elastic-fleet', 'sobridge', DOCKERMERGED.containers['so-elastic-fleet'].ip) }}
+
 so-elastic-fleet:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent:{{ GLOBALS.so_version }}
@@ -65,6 +72,7 @@ so-elastic-fleet:
       - /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro
       - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro
       - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro
+      - /opt/so/conf/elastic-fleet/state:/usr/share/elastic-agent/state
       - /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs 
      {% if DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
         {% for BIND in DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
@@ -72,6 +80,7 @@ so-elastic-fleet:
         {% endfor %}
       {% endif %}      
     - environment:
+      {% if not ENROLLED %}
       - FLEET_SERVER_ENABLE=true
       - FLEET_URL=https://{{ GLOBALS.hostname }}:8220
       - FLEET_SERVER_ELASTICSEARCH_HOST=https://{{ GLOBALS.manager }}:9200
@@ -81,6 +90,9 @@ so-elastic-fleet:
       - FLEET_SERVER_CERT_KEY=/etc/pki/elasticfleet-server.key
       - FLEET_CA=/etc/pki/tls/certs/intca.crt     
       - FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt
+      {% endif %}
+      - STATE_PATH=/usr/share/elastic-agent/state
+      - CONFIG_PATH=/usr/share/elastic-agent/state
       - LOGS_PATH=logs
       {% if DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
         {% for XTRAENV in DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
@@ -99,6 +111,7 @@ so-elastic-fleet:
       - x509: etc_elasticfleet_crt
     - require:
       - file: trusttheca
+      - file: eastatedir
       - x509: etc_elasticfleet_key
       - x509: etc_elasticfleet_crt
 

salt/elasticfleet/install_agent_grid.sls

@@ -10,6 +10,15 @@
 {% set AGENT_STATUS = salt['service.available']('elastic-agent') %}
 {% set AGENT_EXISTS = salt['file.file_exists']('/opt/Elastic/Agent/elastic-agent') %}
 
+so-elastic-agent-install:
+  file.managed:
+    - name: /usr/sbin/so-elastic-agent-install
+    - source: salt://elasticfleet/tools/sbin/so-elastic-agent-install
+    - user: 947
+    - group: 939
+    - mode: 755
+    - show_changes: False
+
 {% if not AGENT_STATUS or not AGENT_EXISTS %}
 
 pull_agent_installer:
@@ -21,11 +30,9 @@ pull_agent_installer:
 
 run_installer:
   cmd.run:
-    - name: ./so-elastic-agent_linux_amd64 -token={{ GRIDNODETOKEN }} -force
+    - name: /usr/sbin/so-elastic-agent-install "{{ GRIDNODETOKEN }}"
-    - cwd: /opt/so
+    - require:
-    - retry:
+      - file: pull_agent_installer
-        attempts: 3
-        interval: 20
 
 cleanup_agent_installer:
   file.absent:

salt/elasticfleet/tools/sbin/so-elastic-agent-install

@@ -0,0 +1,100 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-elastic-fleet-common
+
+
+# passed in as arg from elasticfleet/install_agent_grid.sls, else pulled from pillar later
+GRIDNODETOKEN="$1"
+LOGFILE="/opt/so/SO-Elastic-Agent_Installer_Health.log"
+
+check_agent_health() {
+    timeout=300
+    interval=10
+    start=$SECONDS
+
+    while (( SECONDS - start < timeout )); do
+        agent_status=$(elastic-agent status 2>&1)
+        echo -e "\n$(date)\n$agent_status\n" >> "$LOGFILE"
+        if echo "$agent_status" | grep -A1 'elastic-agent$' | grep -q 'status: (HEALTHY)'; then
+            return 0
+        fi
+        echo "The Elastic Agent is not yet healthy. Waiting for ${interval} seconds before checking again..."
+        sleep "$interval"
+    done
+
+    echo "The Elastic Agent did not become healthy within ${timeout} seconds"
+    return 1
+}
+
+uninstall_agent() {
+    if command -v elastic-agent >/dev/null 2>&1; then
+        elastic-agent uninstall -f
+    fi
+}
+
+if [[ -z "$GRIDNODETOKEN" ]]; then
+    noderole=$(so-yaml.py get -r /etc/salt/grains role)
+    if [[ "$noderole" == "so-heavynode" ]]; then
+        GRIDNODETOKEN=$(salt-call pillar.get global:fleet_grid_enrollment_token_heavy --out=newline_values_only)
+    else
+        GRIDNODETOKEN=$(salt-call pillar.get global:fleet_grid_enrollment_token_general --out=newline_values_only)
+    fi
+fi
+
+if [[ -z "$GRIDNODETOKEN" ]]; then
+    echo "Unable to determine Elastic Fleet enrollment token. Exiting."
+
+    exit 1
+fi
+
+if [[ ! -x /opt/so/so-elastic-agent_linux_amd64 ]]; then
+    echo "Downloading so-elastic-agent installer... This could take a while if another Salt job is running."
+
+    # When running outside of elasticfleet/install_agent_grid.sls we need to download the installer independently.
+    #  PYTHONWARNINGS="ignore" to avoid messages like the following when running salt-call:
+    #  '/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/base.py:129: TransportWarning: Unclosed transport! <salt.transport.zeromq.RequestClient object at 0x7fc5f0ee7a30>
+    #    File "/bin/salt-call", line 12, in <module>
+    #      sys.exit(salt_call())'
+
+    PYTHONWARNINGS="ignore" salt-call state.single file.managed name=/opt/so/so-elastic-agent_linux_amd64 source=salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64 mode=755 makedirs=True queue=True
+
+fi
+
+if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then
+    attempts=0
+    cd /opt/so/ || exit 1
+
+    truncate -s 0 "$LOGFILE"
+
+    uninstall_agent
+
+    while [[ $attempts -lt 3 ]]; do
+        if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force && echo "Verifying Elastic Agent health..." && check_agent_health; then
+            rm -f /opt/so/so-elastic-agent_linux_amd64
+            elastic-agent status
+
+            exit 0
+        fi
+
+        attempts=$((attempts + 1))
+
+        if [[ $attempts -lt 3 ]]; then
+            echo "Unable to verify Elastic Agent health... Retrying in 20 seconds..."
+            sleep 20
+        fi
+    done
+
+    uninstall_agent
+    rm -f /opt/so/so-elastic-agent_linux_amd64
+    echo "The so-elastic-agent installer failed after 3 attempts. Exiting."
+
+    exit 1
+else
+    echo "Unable to locate so-elastic-agent installer. Exiting."
+
+    exit 1
+fi

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers

@@ -30,7 +30,7 @@ done
 if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then 
   printf "\nFleet Host URL, Enrollment Token or Elastic Version empty - exiting..."
   printf "\nFleet Host: $FLEETHOST, Enrollment Token: $ENROLLMENTOKEN\n"
-  exit 
+  exit 1
 fi
 
 OSARCH=( "linux-x86_64" "windows-x86_64" "darwin-x86_64" "darwin-aarch64" )
@@ -62,31 +62,54 @@ do
 done 
 
 GOTARGETOS=( "linux" "windows" "darwin" "darwin/arm64" )
-GOARCH="amd64"
 printf "\n### Generating OS packages using the cleaned up tarballs"
-for GOOS in "${GOTARGETOS[@]}"
+for GOOS in "${GOTARGETOS[@]}"; do
-do
+    GOARCH="amd64"
     if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi
     printf "\n\n### Generating $GOOS/$GOARCH Installer...\n"
     docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \
     --mount type=bind,source=/etc/pki/tls/certs/,target=/workspace/files/cert/ \
     --mount type=bind,source=/nsm/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \
-    --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \
+    --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ \
     {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN"  -o /output/so-elastic-agent_${GOOS}_${GOARCH}
     printf "\n### $GOOS/$GOARCH Installer Generated...\n"
 done
 
 printf "\n\n### Generating MSI...\n"
-cp /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe
+cp /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe
 docker run \
---mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ -w /output \
+--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ -w /output \
 {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs
 printf "\n### MSI Generated...\n"
 
+# Verify installers were created
+for GOOS in "${GOTARGETOS[@]}"; do
+    GOARCH="amd64"
+    if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin"; GOARCH="arm64"; fi
+    if [[ ! -f /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_${GOOS}_${GOARCH} ]]; then
+        printf "\n### ERROR: Installer for %s/%s was not generated. Exiting...\n" "$GOOS" "$GOARCH"
+        exit 1
+    fi
+    # After verifying new installer was generated, move it to so_agent-installers directory
+    mv /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_${GOOS}_${GOARCH} /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/
+done
+
+# Verify MSI installer
+if [[ ! -f /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64_msi ]]; then
+    printf "\n### ERROR: Installer MSI was not generated. Exiting...\n"
+    exit 1
+else
+    # After verifying new installer MSI was generated, move it to so_agent-installers directory
+    mv /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64_msi /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/
+fi
+
 printf "\n### Cleaning up temp files \n"
 rm -rf /nsm/elastic-agent-workspace
-rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe
+rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe
 
 printf "\n### Copying so_agent-installers to /nsm/elastic-fleet/ for nginx.\n"
 \cp -vr /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ /nsm/elastic-fleet/
 chmod 644 /nsm/elastic-fleet/so_agent-installers/*
+
+# if we got here all installers have been generated successfully
+exit 0

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -244,11 +244,37 @@ printf '%s\n'\
     "" >> "$global_pillar_file"
 
 # Call Elastic-Fleet Salt State
-printf "\nApplying elasticfleet state"
+printf "\nApplying elasticfleet state\n"
-salt-call state.apply elasticfleet queue=True
+for state_attempt in {1..3}; do
+    if salt-call state.apply elasticfleet queue=True; then
+        break
+    elif [[ $state_attempt -lt 3 ]]; then
+        printf "\nElasticfleet state did not complete successfully... Attempt (%s/3). Retrying...\n" "$state_attempt"
+        sleep 10
+    else
+        printf "\nFailure(s) in elasticfleet state... Exiting...\n"
+        exit 1
+    fi
+done
 
+printf "\nRunning so-elastic-agent-gen-installers\n"
 # Generate installers & install Elastic Agent on the node
-so-elastic-agent-gen-installers
+for agent_gen_attempt in {1..3}; do
-printf "\nApplying elasticfleet.install_agent_grid state"
+    if so-elastic-agent-gen-installers; then
-salt-call state.apply elasticfleet.install_agent_grid queue=True
+        break
-exit 0
+    elif [[ $agent_gen_attempt -lt 3 ]]; then
+        printf "\nUnable to generate Elastic Agent installers... Attempt (%s/3). Retrying...\n" "$agent_gen_attempt"
+        sleep 10
+    else
+        printf "\nFailed to generate Elastic Agent installers after 3 attempts. Exiting...\n"
+        exit 1
+    fi
+done
+
+printf "\nApplying elasticfleet.install_agent_grid state\n"
+if ! salt-call state.apply elasticfleet.install_agent_grid queue=True; then
+    printf "\nFailure(s) in elasticfleet.install_agent_grid state... Exiting...\n"
+    exit 1
+fi
+
+printf "\nElastic Fleet setup completed successfully\n"

salt/elasticsearch/enabled.sls

@@ -10,6 +10,9 @@
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-elasticsearch', 'sobridge', DOCKERMERGED.containers['so-elasticsearch'].ip) }}
 
 include:
   - ca

salt/elasticsearch/soc_elasticsearch.yaml

@@ -645,6 +645,7 @@ elasticsearch:
             global: True
             advanced: True
             helpLink: elasticsearch
+    so-logs-soc: *dataStreamSettings
     so-logs-system_x_auth: *dataStreamSettings
     so-logs-system_x_syslog: *dataStreamSettings
     so-logs-system_x_system: *dataStreamSettings

salt/hydra/enabled.sls

@@ -14,6 +14,9 @@
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   if 'api' in salt['pillar.get']('features', []) %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-hydra', 'sobridge', DOCKERMERGED.containers['so-hydra'].ip) }}
 
 include:
   - hydra.config

salt/influxdb/enabled.sls

@@ -9,6 +9,9 @@
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   set PASSWORD = salt['pillar.get']('secrets:influx_pass') %}
 {%   set TOKEN = salt['pillar.get']('influxdb:token') %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-influxdb', 'sobridge', DOCKERMERGED.containers['so-influxdb'].ip) }}
 
 include:
   - influxdb.ssl

salt/kafka/enabled.sls

@@ -16,6 +16,9 @@
 {%   set KAFKANODES = salt['pillar.get']('kafka:nodes') %}
 {%   set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %}
 {%   if 'gmd' in salt['pillar.get']('features', []) %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-kafka', 'sobridge', DOCKERMERGED.containers['so-kafka'].ip) }}
 
 include:
   - kafka.ca

salt/kibana/enabled.sls

@@ -8,6 +8,9 @@
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-kibana', 'sobridge', DOCKERMERGED.containers['so-kibana'].ip) }}
 
 include:
   - kibana.config
@@ -69,7 +72,7 @@ wait_for_so-kibana:
     - ssl: True
     - verify_ssl: False
     - status: 200
-    - wait_for: 300
+    - wait_for: 600
     - request_interval: 15
     - require:
       - docker_container: so-kibana

salt/kratos/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-kratos', 'sobridge', DOCKERMERGED.containers['so-kratos'].ip) }}
 
 include:
   - kratos.config

salt/logstash/enabled.sls

@@ -10,6 +10,9 @@
 {%   from 'logstash/map.jinja' import LOGSTASH_MERGED %}
 {%   from 'logstash/map.jinja' import LOGSTASH_NODES %}
 {%   set lsheap = LOGSTASH_MERGED.settings.lsheap %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-logstash', 'sobridge', DOCKERMERGED.containers['so-logstash'].ip) }}
 
 include:
   - ca

salt/manager/files/repodownload.conf

@@ -11,8 +11,8 @@ name=Security Onion Repo repo
 mirrorlist=file:///opt/so/conf/reposync/mirror.txt
 enabled=1
 gpgcheck=1
-[securityonionkernel]
+[securityonionkernelsync]
-name=Security Onion Repo repo
+name=Security Onion Kernel Repo repo
 mirrorlist=file:///opt/so/conf/reposync/mirror-kernel.txt
 enabled=1
 gpgcheck=1

salt/manager/tools/sbin/so-repo-sync

@@ -17,9 +17,9 @@ createrepo /nsm/repo
 # The kernel repo section is deployed to repodownload.conf by the manager highstate, which
 # runs AFTER this script during soup. On the first upgrade to a kernel-aware version the
 # on-disk config still predates the section, so guard on its presence to avoid dnf's
-# "Unknown repo: 'securityonionkernel'" aborting the sync (set -e). The next sync after the
+# "Unknown repo: 'securityonionkernelsync'" aborting the sync (set -e). The next sync after the
 # highstate deploys the section will pick it up.
-if grep -q '^\[securityonionkernel\]' /opt/so/conf/reposync/repodownload.conf; then
+if grep -q '^\[securityonionkernelsync\]' /opt/so/conf/reposync/repodownload.conf; then
-  dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/
+  dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernelsync --download-metadata -p /nsm/kernelrepo/
   createrepo /nsm/kernelrepo
 fi

salt/manager/tools/sbin/soup

@@ -245,6 +245,7 @@ check_airgap() {
       UPDATE_DIR=/tmp/soagupdate/SecurityOnion
       AGDOCKER=/tmp/soagupdate/docker
       AGREPO=/tmp/soagupdate/minimal/Packages
+      AGUEKREPO=/tmp/soagupdate/uek/Packages
   else
       is_airgap=1
   fi
@@ -783,12 +784,23 @@ post_to_3.1.0() {
 
 ### 3.2.0 Scripts ###
 
+recollate_postgres() {
+  echo ""
+  echo "Recollating PostgreSQL databases. The following output may contain warnings about a version mismatch, followed by a note indicating that the collation version has been changed."
+  for db in postgres securityonion so_telegraf; do
+    docker exec so-postgres psql -U postgres $db -c "reindex database $db"
+    docker exec so-postgres psql -U postgres $db -c "alter database $db refresh collation version"
+  done
+  echo "Recollating PostgreSQL databases complete."
+  echo ""
+}
+
 bootstrap_so_soc_database() {
   # init-db.sh is mounted into so-postgres at /docker-entrypoint-initdb.d/init-db.sh
   # and runs automatically only on a fresh data directory. Hosts upgrading from
   # 3.1.0 already have /nsm/postgres populated, so the so_soc bootstrap block
   # added in 3.2 never fires. Re-run the script explicitly; it's idempotent.
-  echo "Bootstrapping so_soc database via init-db.sh."
+  echo "Bootstrapping database via init-db.sh."
   # The postgres image has no USER directive, so `docker exec` defaults to
   # root, and the container env intentionally omits POSTGRES_USER (the upstream
   # entrypoint defaults it transiently during first-init only). Recreate both
@@ -799,10 +811,13 @@ bootstrap_so_soc_database() {
     return 0
   fi
   if ! $exec_cmd; then
-    FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd")
+    FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the database may not have been bootstrapped. Re-run manually: $exec_cmd")
     return 0
   fi
-  echo "so_soc bootstrap complete."
+  echo "Database bootstrap complete."
+
+  echo "Restarting so-soc container to pick up database changes"
+  docker restart so-soc
 }
 
 # Existing grids should keep ILM unless an admin explicitly opts in to DLM.
@@ -850,6 +865,28 @@ kibana_backport_streams_index_template() {
 
 }
 
+# Runs kafka-features.sh upgrade --release-version $1
+#   Upgrades Kafka KRaft cluster metadata
+update_kafka_metadata() {
+    metadata_version="$1"
+    global_pillar="/opt/so/saltstack/local/pillar/global/soc_global.sls"
+    if PIPELINE=$(so-yaml.py get -r "$global_pillar" global.pipeline 2> /dev/null) && [[ "$PIPELINE" == "KAFKA" ]]; then
+        kafka_nodes_raw=$(salt-call pillar.get kafka:nodes --out=json)
+        if kafka_nodes=$(jq -er '.local | select(type == "object" and length > 0)' <<< "$kafka_nodes_raw"); then
+            bootstrap_servers=$(jq -r '[to_entries[] | select(.value.role | contains("broker")) | "\(.value.ip):9092"] | join(",")' <<< "$kafka_nodes")
+            echo "Upgrading Kafka KRaft cluster version"
+            so-kafka-cli kafka-features.sh --bootstrap-server "$bootstrap_servers" --command-config /opt/kafka/config/kraft/client.properties upgrade --release-version "$metadata_version" 2>/dev/null || true
+
+            return 0
+        else
+            FINAL_MESSAGE_QUEUE+=("WARNING: Unable to automatically perform Kafka KRaft cluster metadata update. This step can be performed manually using the following command (replacing \$BROKER_IP with the ip of atleast 1 available Kafka broker):")
+            FINAL_MESSAGE_QUEUE+=("    - so-kafka-cli kafka-features.sh --bootstrap-server \$BROKER_IP:9092 --command-config /opt/kafka/config/kraft/client.properties upgrade --release-version $metadata_version")
+        fi
+    else
+        echo "Nothing to do!"
+    fi
+}
+
 up_to_3.2.0() {
   fix_logstash_0013_lumberjack_pipeline_name
 
@@ -859,6 +896,9 @@ up_to_3.2.0() {
 }
 
 post_to_3.2.0() {
+  # Recollate due to image OS rebase
+  recollate_postgres
+
   bootstrap_so_soc_database
 
   # Including agent regen script here since it was missed in post_to_3.1.0
@@ -867,6 +907,8 @@ post_to_3.2.0() {
 
   kibana_backport_streams_index_template
 
+  update_kafka_metadata "4.3"
+
   POSTVERSION=3.2.0
 }
 
@@ -980,13 +1022,19 @@ update_airgap_rules() {
   rsync -a $UPDATE_DIR/agrules/securityonion-resources/* /nsm/securityonion-resources/
 }
 
-update_airgap_repo() {
+update_airgap_repos() {
   # Update the files in the repo
-  echo "Syncing new updates to /nsm/repo"
+  echo "Syncing new updates to /nsm/repo & /nsm/kernelrepo"
-  rsync -a $AGREPO/* /nsm/repo/
+  # Airgap soup copies new files into the local repo, but doesn't remove old packages. Retaining the ability to rollback package updates
-  echo "Creating repo"
+  rsync -a "$AGREPO"/ /nsm/repo/
+  rsync -a "$AGUEKREPO"/ /nsm/kernelrepo/
+
   dnf -y install yum-utils createrepo_c
+
+  echo "Running createrepo for /nsm/repo"
   createrepo /nsm/repo
+  echo "Running createrepo for /nsm/kernelrepo"
+  createrepo /nsm/kernelrepo
 }
 
 update_salt_mine() {
@@ -1742,7 +1790,7 @@ main() {
   set -e
 
   if [[ $is_airgap -eq 0 ]]; then
-    update_airgap_repo
+    update_airgap_repos
     dnf clean all
     check_os_updates
   elif [[ $OS == 'oracle' ]]; then

salt/nginx/enabled.sls

@@ -8,6 +8,7 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'nginx/map.jinja' import NGINXMERGED %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
 
 include:
   - nginx.ssl
@@ -31,6 +32,8 @@ make-rule-dir-nginx:
 {%     set container_config = 'so-nginx-fleet-node' %}
 {%   endif %}
 
+{{ clear_stale_endpoint('so-nginx', 'sobridge', DOCKERMERGED.containers[container_config].ip) }}
+
 so-nginx:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-nginx:{{ GLOBALS.so_version }}

salt/postgres/enabled.sls

@@ -8,6 +8,9 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   set SO_POSTGRES_USER = salt['pillar.get']('postgres:auth:users:so_postgres_user:user', 'so_postgres') %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-postgres', 'sobridge', DOCKERMERGED.containers['so-postgres'].ip) }}
 
 include:
   - postgres.auth

salt/redis/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-redis', 'sobridge', DOCKERMERGED.containers['so-redis'].ip) }}
 
 include:
   - ca

salt/registry/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-dockerregistry', 'sobridge', DOCKERMERGED.containers['so-dockerregistry'].ip) }}
 
 include:
   - registry.ssl

salt/soc/config.sls

@@ -134,6 +134,30 @@ socsigmasopipeline:
     - group: 939
     - mode: 600
 
+socsigmaplaybookpipeline:
+  file.managed:
+    - name: /opt/so/conf/soc/sigma_playbook_pipeline.yaml
+    - source: salt://soc/files/soc/sigma_playbook_pipeline.yaml
+    - user: 939
+    - group: 939
+    - mode: 600
+
+socplaybookplaceholdermap:
+  file.managed:
+    - name: /opt/so/conf/soc/playbook_placeholder_map.yaml
+    - source: salt://soc/files/soc/playbook_placeholder_map.yaml
+    - user: 939
+    - group: 939
+    - mode: 600
+
+socplaybookplaceholdermapcustom:
+  file.managed:
+    - name: /opt/so/conf/soc/playbook_placeholder_map_custom.yaml
+    - source: salt://soc/files/soc/playbook_placeholder_map_custom.yaml
+    - user: 939
+    - group: 939
+    - mode: 600
+
 socbanner:
   file.managed:
     - name: /opt/so/conf/soc/banner.md

salt/soc/defaults.map.jinja

@@ -8,6 +8,7 @@
 {% from 'docker/docker.map.jinja' import DOCKERMERGED -%}
 {% set INFLUXDB_TOKEN = salt['pillar.get']('influxdb:token') %}
 {% import_text 'influxdb/metrics_link.txt' as METRICS_LINK %}
+{% from 'telegraf/map.jinja' import TELEGRAFMERGED %}
 
 {% for module, application_url in GLOBALS.application_urls.items() %}
 {%   do SOCDEFAULTS.soc.config.server.modules[module].update({'hostUrl': application_url}) %}
@@ -24,13 +25,22 @@
 
 {% do SOCDEFAULTS.soc.config.server.modules.elastic.update({'username': GLOBALS.elasticsearch.auth.users.so_elastic_user.user, 'password': GLOBALS.elasticsearch.auth.users.so_elastic_user.pass}) %}
 
-{% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %}
+{% if TELEGRAFMERGED.output == 'POSTGRES' %}
-{% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'token': INFLUXDB_TOKEN}) %}
+{%   for tool in SOCDEFAULTS.soc.config.server.client.tools %}
-{% for tool in SOCDEFAULTS.soc.config.server.client.tools %}
+{%     if tool.name == "toolInfluxDb" %}
-{%   if tool.name == "toolInfluxDb" and METRICS_LINK | length > 0 %}
+{%       do SOCDEFAULTS.soc.config.server.client.tools.remove(tool) %}
-{%     do tool.update({'link': METRICS_LINK}) %}
+{%     endif %}
-{%   endif %}
+{%   endfor %}
-{% endfor %}
+
+{% else %}
+{%   do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %}
+{%   do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'token': INFLUXDB_TOKEN}) %}
+{%   for tool in SOCDEFAULTS.soc.config.server.client.tools %}
+{%     if tool.name == "toolInfluxDb" and METRICS_LINK | length > 0 %}
+{%       do tool.update({'link': METRICS_LINK}) %}
+{%     endif %}
+{%   endfor %}
+{% endif %}
 
 {% do SOCDEFAULTS.soc.config.server.modules.statickeyauth.update({'anonymousCidr': DOCKERMERGED.range, 'apiKey': pillar.sensoroni.config.sensoronikey}) %}
 

salt/soc/defaults.yaml

@@ -1500,15 +1500,23 @@ soc:
           playbookRepos:
             default:
               - repo: https://github.com/Security-Onion-Solutions/securityonion-resources-playbooks
+                rulesetName: sos-playbook-resources
                 branch: main
                 folder: securityonion-normalized
+              - repo: https://github.com/Security-Onion-Solutions/securityonion-resources-playbooks
+                rulesetName: sos-published
+                branch: published
+                folder: sigma
             airgap:
               - repo: file:///nsm/airgap-resources/playbooks/securityonion-resources-playbooks
+                rulesetName: sos-resources-ag
                 branch: main
                 folder: securityonion-normalized
         assistant:
           systemPromptAddendum: ""
           systemPromptAddendumMaxLength: 50000
+          maxSubSessionTokens: 0
+          maxDelegationDepth: 5
           adapters:
             - name: SOAI
               protocol: securityonion_ai_cloud
@@ -1520,6 +1528,10 @@ soc:
               serviceAccountJSON: ""
               serviceAccountLocation: ""
               healthTimeoutSeconds: 5
+          agentic: false
+          agentMapping:
+            Orchestrator: sonnet
+            Hunter: sonnet
         onionconfig:
           saltstackDir: /opt/so/saltstack
           bypassEnabled: false
@@ -1771,13 +1783,13 @@ soc:
               enabled: true
           queries:
             - name: Default Query
-              description: Show all events grouped by the observer host
-              query: '* | groupby observer.name'
-              showSubtitle: true
-            - name: Log Type
               description: Show all events grouped by module and dataset
               query: '* | groupby event.module* event.dataset'
               showSubtitle: true
+            - name: Observer
+              description: Show all events grouped by the observer host
+              query: '* | groupby observer.name'
+              showSubtitle: true
             - name: SOC - Auth
               description: Users authenticated to SOC grouped by IP address and identity
               query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby http.request.headers.x-real-ip user.name'
@@ -2689,6 +2701,8 @@ soc:
           thresholdColorRatioLow: 0.5
           thresholdColorRatioMed: 0.75
           thresholdColorRatioMax: 1
+          toolBusyMaxRetries: 30
+          toolBusyRetryDelayMs: 1000
           availableModels:
             - id: sonnet
               displayName: Claude Sonnet

salt/soc/enabled.sls

@@ -10,6 +10,9 @@
 {%   from 'soc/merged.map.jinja' import DOCKER_EXTRA_HOSTS %}
 {%   from 'soc/merged.map.jinja' import SOCMERGED %}
 
+{% from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+{{ clear_stale_endpoint('so-soc', 'sobridge', DOCKERMERGED.containers['so-soc'].ip) }}
+
 include:
   - ca
   - soc.config
@@ -45,7 +48,10 @@ so-soc:
       - /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro
       - /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro
       - /opt/so/conf/soc/sigma_so_pipeline.yaml:/opt/sensoroni/sigma_so_pipeline.yaml:ro
-      - /opt/so/conf/soc/sigma_final_pipeline.yaml:/opt/sensoroni/sigma_final_pipeline.yaml:rw
+      - /opt/so/conf/soc/sigma_playbook_pipeline.yaml:/opt/sensoroni/sigma_playbook_pipeline.yaml:ro
+      - /opt/so/conf/soc/sigma_final_pipeline.yaml:/opt/sensoroni/sigma_final_pipeline.yaml:ro
+      - /opt/so/conf/soc/playbook_placeholder_map.yaml:/opt/sensoroni/playbook_placeholder_map.yaml:ro
+      - /opt/so/conf/soc/playbook_placeholder_map_custom.yaml:/opt/sensoroni/playbook_placeholder_map_custom.yaml:ro
       - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro
       - /opt/so/conf/soc/custom_roles:/opt/sensoroni/rbac/custom_roles:ro
       - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw
@@ -99,6 +105,8 @@ so-soc:
       - file: soccustomroles
       - file: socusersroles
       - file: socclientsroles
+      - file: socplaybookplaceholdermap
+      - file: socplaybookplaceholdermapcustom
 
 delete_so-soc_so-status.disabled:
   file.uncomment:

salt/soc/files/soc/playbook_placeholder_map.yaml

@@ -0,0 +1,49 @@
+# Global Playbook placeholder map: %token% -> event field path.
+#
+# Loaded by the SOC Playbook module and used to resolve `field|expand:%placeholder%` values 
+# from an alert when converting playbook questions to OQL. 
+# Left: the %token% used in a question
+# Right: the event field its value is read from (event_data.-nested or bare; the module
+# tries both). 
+#
+# Example: with `src_ip: source.ip` (below), a question that writes
+# `source.ip|expand: '%src_ip%'` resolves %src_ip% to the alert's source.ip at convert time.
+#
+# This is the global base layer. To add or override tokens edit playbook_placeholder_map_custom.yaml.
+# those entries overlay this map and win on conflict.
+
+CommandLine: process.command_line
+CurrentDirectory: process.working_directory
+Image: process.executable
+ImageLoaded: dll.name
+ParentImage: process.parent.executable
+ParentName: process.parent.name
+ParentProcessGuid: process.parent.entity_id
+ProcessGuid: process.entity_id
+TargetFilename: file.name
+TargetObject: registry.path
+TargetUserName: user.target.name
+User: user.name
+community_id: network.community_id
+dns_resolved_ip: dns.resolved_ip
+document_id: soc_id
+dst_ip: destination.ip
+dst_port: destination.port
+event_data_source_ip: source.ip
+file_path: file.path
+file_dirs: process.file_dirs
+file_name: process.name
+file_paths: process.file_paths
+hostname: host.name
+private_ip: network.private_ip
+public_ip: network.public_ip
+related_hosts: related.hosts
+related_ip: related.ip
+src_ip: source.ip
+dns_query_name: dns.query_name
+flow_id: log.id.uid
+payload: network.data.decoded
+rule_category: rule.category
+rule_name: rule.name
+rule_uuid: rule.uuid
+src_port: source.port

salt/soc/files/soc/playbook_placeholder_map_custom.yaml

@@ -0,0 +1,14 @@
+# Custom Playbook placeholder map: %token% -> event field path.
+#
+#
+# Left: the %token% used in a playbook question.
+# Right: the event field its value is read from (event_data.-nested or bare; the module tries
+# both). Note: a token that is simply named after a flat event field resolves automatically
+# without an entry here - only add a mapping when the token name differs from the field name.
+#
+# Example:
+#
+# account_id: cloudflare.account_id
+#
+# A question that writes
+# `account_id|expand: '%account_id%'` resolves %account_id% from the alert at convert time.

salt/soc/files/soc/sigma_playbook_pipeline.yaml

@@ -0,0 +1,12 @@
+name: Security Onion - Playbook Pipeline
+priority: 97
+transformations:
+    # Route string fields to their lowercase-normalized .caseless subfield so wildcard
+    # matches are case-insensitive.
+    - id: case_insensitive_string_fields
+      type: field_name_mapping
+      mapping:
+        process.executable: process.executable.caseless
+        process.parent.executable: process.parent.executable.caseless
+        process.command_line: process.command_line.caseless
+        process.parent.command_line: process.parent.command_line.caseless

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -63,6 +63,14 @@ transformations:
       rule_conditions:
       - type: logsource
         category: antivirus
+    # OS-agnostic process_creation scoping for product-less (NIDS/host-pivot) rules.
+    - id: process_creation_os_agnostic
+      type: add_condition
+      conditions:
+        event.category: process
+      rule_conditions:
+      - type: logsource
+        category: process_creation
     # Transforms the `Hashes` field to ECS fields
     # ECS fields are used by the hash fields emitted by Elastic Defend
     # If shipped with Elastic Agent, sysmon logs will also have hashes mapped to ECS fields    
@@ -108,6 +116,40 @@ transformations:
       - type: logsource
         product: windows
         category: driver_load
+    - id: ecs_fix_process_creation
+      type: field_name_mapping
+      mapping:
+        # bare `Hashes` (the combined-string case is broken out above)
+        winlog.event_data.Hashes: process.hash.sha256
+        winlog.event_data.IntegrityLevel: process.Ext.token.integrity_level_name
+        winlog.event_data.ParentName: process.parent.name
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: process_creation
+    - id: ecs_fix_registry_set
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.Details: registry.data.strings
+        # field rename only; EventType values (SetValue/CreateKey) still differ from
+        # event.action values (modification/creation)
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: registry_set
+    - id: ecs_fix_image_load
+      type: field_name_mapping
+      mapping:
+        file.path: dll.path
+        file.code_signature.signed: dll.code_signature.exists
+        winlog.event_data.Signature: dll.code_signature.subject_name
+        file.code_signature.status: dll.code_signature.status
+        winlog.event_data.Hashes: dll.hash.sha256
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: image_load
     - id: linux_security_add-fields
       type: add_condition
       conditions:
@@ -281,6 +323,15 @@ transformations:
       rule_conditions:
       - type: logsource
         category: file_event
+    # Scope image_load rules to Elastic Endpoint library events (event.category:library, dll.*
+    # populated).
+    - id: endpoint_image_load_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'library'
+      rule_conditions:
+      - type: logsource
+        category: image_load
     # Maps network rules to all network logs
     # This targets all network logs, all services, generated from endpoints and network
     - id: network_add-fields

salt/soc/merged.map.jinja

@@ -7,6 +7,11 @@
 {% from 'soc/defaults.map.jinja' import SOCDEFAULTS with context %}
 {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
 {% from 'manager/map.jinja' import MANAGERMERGED %}
+{% from 'telegraf/map.jinja' import TELEGRAFMERGED %}
+{%- set PG_ENTRY = salt['pillar.get']('telegraf:postgres_creds:' ~ grains.id, {}) %}
+{%- set PG_USER = PG_ENTRY.get('user', '') %}
+{%- set PG_PASS = PG_ENTRY.get('pass', '') %}
+
 {% set DOCKER_EXTRA_HOSTS = ELASTICSEARCH_NODES %}
 {% do DOCKER_EXTRA_HOSTS.append({GLOBALS.influxdb_host:pillar.node_data[GLOBALS.influxdb_host].ip}) %}
 
@@ -75,6 +80,20 @@
 {%   do SOCMERGED.config.server.update({'airgapEnabled': false}) %}
 {% endif %}
 
+{# Define the postgresmetrics module if telegraf is setup to only use Postgres #}
+{% if TELEGRAFMERGED.output != 'INFLUXDB' and PG_USER and PG_PASS %}
+{%   do SOCMERGED.config.server.modules.update({
+        'postgresmetrics': {
+          'database': 'so_telegraf',
+          'host': GLOBALS.manager_ip,
+          'password': PG_PASS,
+          'port': 5432,
+          'sslMode': 'allow',
+          'user': PG_USER,
+        }
+      }) %}
+{%   do SOCMERGED.config.server.modules.pop('influxdb') %}
+{% endif %}
 
 {# Define the Detections custom ruleset that should always be present #}
 {% set CUSTOM_RULESET = {

salt/soc/soc_soc.yaml

@@ -46,7 +46,15 @@ soc:
         syntax: yaml
         file: True
         global: True
-        advanced: True
+        advanced: False
+        helpLink: security-onion-console-customization
+      playbook_placeholder_map_custom__yaml:
+        title: Playbook Placeholder Map
+        description: Custom mappings of Playbook %placeholder% tokens to event fields.
+        syntax: yaml
+        file: True
+        global: True
+        advanced: False
         helpLink: security-onion-console-customization
   config:
     licenseKey:
@@ -719,6 +727,16 @@ soc:
             description: Maximum length of the system prompt addendum. Longer prompts will be truncated.
             global: True
             advanced: True
+          maxSubSessionTokens:
+            description: Maximum number of output tokens a delegated sub-session may generate across all of its turns. When the budget is reached, the sub-agent is halted and its result is returned to the parent agent. Set to 0 to disable the limit.
+            global: True
+            advanced: True
+            forcedType: int
+          maxDelegationDepth:
+            description: Maximum delegation nesting depth for sub-agents. For example, a value of 2 lets the main agent delegate to a sub-agent that may itself delegate one level deeper. Any deeper delegation is refused and the requesting agent continues without it. Set to 0 to disable the limit.
+            global: True
+            advanced: True
+            forcedType: int
           adapters:
             description: Configuration for AI adapters used by the Onion AI assistant. Please see documentation for help on which fields are required for which protocols.
             global: True
@@ -757,12 +775,29 @@ soc:
               label: Health Timeout Seconds
               required: False
               forcedType: int
+          agentic:
+            description: Indicates if the Assistant Module should operate in agentic mode or not. If true, agents can work together to solve tasks.
+            global: True
+            forcedType: bool
+          agentMapping:
+            Orchestrator:
+              description: The initial agent in most agentic conversations. This agent will delegate requests to specialized agents.
+              global: True
+            Hunter:
+              description: This agent is specialized in querying events.
+              global: True
       client:
         assistant:
           enabled:
             description: Set to true to enable the Onion AI assistant in SOC.
             global: True
             forcedType: bool
+          toolBusyMaxRetries:
+            description: How many times to retry auto approving a tool while a tool is already running.
+            global: True
+          toolBusyRetryDelayMs:
+            description: How long in milliseconds to wait between each retry when auto approving a tool.
+            global: True
           investigationPrompt: 
             description: Prompt given to Onion AI when beginning an investigation.
             global: True

salt/strelka/backend/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-strelka-backend', 'sobridge', DOCKERMERGED.containers['so-strelka-backend'].ip, state_id='strelka_backend') }}
 
 include:
   - strelka.backend.config

salt/strelka/coordinator/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-strelka-coordinator', 'sobridge', DOCKERMERGED.containers['so-strelka-coordinator'].ip, state_id='strelka_coordinator') }}
 
 include:
   - strelka.coordinator.config

salt/strelka/filestream/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-strelka-filestream', 'sobridge', DOCKERMERGED.containers['so-strelka-filestream'].ip, state_id='strelka_filestream') }}
 
 include:
   - strelka.filestream.config

salt/strelka/frontend/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-strelka-frontend', 'sobridge', DOCKERMERGED.containers['so-strelka-frontend'].ip, state_id='strelka_frontend') }}
 
 include:
   - strelka.frontend.config

salt/strelka/gatekeeper/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-strelka-gatekeeper', 'sobridge', DOCKERMERGED.containers['so-strelka-gatekeeper'].ip, state_id='strelka_gatekeeper') }}
 
 include:
   - strelka.gatekeeper.config

salt/strelka/manager/enabled.sls

@@ -7,6 +7,9 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/macros/docker_endpoint.jinja' import clear_stale_endpoint %}
+
+{{ clear_stale_endpoint('so-strelka-manager', 'sobridge', DOCKERMERGED.containers['so-strelka-manager'].ip, state_id='strelka_manager') }}
 
 include:
   - strelka.manager.config

salt/telegraf/etc/telegraf.conf

@@ -244,6 +244,7 @@
   username = "{{ ES_USER }}"
   password = "{{ ES_PASS }}"
   insecure_skip_verify = true
+  cluster_health = true
 {%- elif grains['role'] in ['so-searchnode']  %}
 [[inputs.elasticsearch]]
   servers = ["https://{{ NODEIP }}:9200"]

salt/telegraf/soc_telegraf.yaml

@@ -5,7 +5,7 @@ telegraf:
     advanced: True
     helpLink: influxdb
   output:
-    description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation.
+    description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation. When set to BOTH, the grid screen's metrics are pulled from Postgres, and the InfluxDB tool link remains visible. When set to POSTGRES, the InfluxDB tool link is removed.
     options:
       - INFLUXDB
       - POSTGRES

salt/top.sls

@@ -83,7 +83,6 @@ base:
     - zeek
     - strelka
     - elastalert
-    - utility
     - elasticfleet
     - pcap.cleanup
 
@@ -113,7 +112,6 @@ base:
     - zeek
     - strelka
     - elastalert
-    - utility
     - elasticfleet
     - stig
     - kafka
@@ -141,7 +139,6 @@ base:
     - elastic-fleet-package-registry
     - kibana
     - elastalert
-    - utility
     - elasticfleet
     - stig
     - kafka
@@ -168,7 +165,6 @@ base:
     - elastic-fleet-package-registry
     - kibana
     - elastalert
-    - utility
     - elasticfleet
     - kafka
 
@@ -198,7 +194,6 @@ base:
     - elastic-fleet-package-registry
     - kibana
     - elastalert
-    - utility
     - elasticfleet
     - stig
     - kafka
@@ -222,7 +217,6 @@ base:
     - elasticsearch
     - elastic-fleet-package-registry
     - kibana
-    - utility
     - suricata
     - zeek
     - elasticfleet

salt/utility/bin/eval

@@ -1,29 +0,0 @@
-#!/bin/bash
-# Wait for ElasticSearch to come up, so that we can query for version infromation
-echo -n "Waiting for ElasticSearch..."
-COUNT=0
-ELASTICSEARCH_CONNECTED="no"
-while [[ "$COUNT" -le 30 ]]; do
-  curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://{{ GLOBALS.manager_ip }}:9200
-  if [ $? -eq 0 ]; then
-    ELASTICSEARCH_CONNECTED="yes"
-    echo "connected!"
-    break
-  else
-    ((COUNT+=1))
-    sleep 1
-    echo -n "."
-  fi
-done
-if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
-  echo
-  echo -e "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'docker ps' \n  -running 'sudo so-elastic-restart'"
-  echo
-
-  exit
-fi
-
-echo "Applying cross cluster search config..."
-    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -XPUT -L https://{{ GLOBALS.manager_ip }}:9200/_cluster/settings \
-         -H 'Content-Type: application/json' \
-         -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ grains.host }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"

salt/utility/init.sls

@@ -1,22 +0,0 @@
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-
-{% if sls in allowed_states %}
-  {% if grains['role'] in ['so-eval', 'so-import'] %}
-fixsearch:
-  cmd.script:
-    - shell: /bin/bash
-    - cwd: /opt/so
-    - source: salt://utility/bin/eval
-    - template: jinja
-    - defaults:
-        GLOBALS: {{ GLOBALS }}
-  {% endif %}
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

setup/so-functions

@@ -29,8 +29,12 @@ title() {
 }
 
 fail_setup() {
+	local err_msg=$1
+	if [[ -n "$err_msg" ]]; then
+		error "$err_msg"
+	fi
 	error "Setup encountered an unrecoverable failure, exiting"
-	touch /root/failure
+	echo "setup incomplete: $err_msg" > /root/failure
 	exit 1
 }
 
@@ -697,7 +701,7 @@ compare_main_nic_ip() {
 			EOM
 
 		  [[ -n $TESTING ]] || whiptail --title "$whiptail_title" --msgbox "$message" 11 75
-		  kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup
+		  kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup "Main IP mismatch"
 	  fi
 	else
 		# Setup uses MAINIP, but since we ignore the equality condition when using a VPN 
@@ -755,8 +759,7 @@ configure_management_bond() {
 	info "Setting up $bond_name management interface with mode $bond_mode"
 
 	if [[ ${#MBNICS[@]} -eq 0 ]]; then
-		error "[ERROR] No management bond NICs were selected."
+		fail_setup "No management bond NICs selected"
-		fail_setup
 	fi
 
 	nmcli -t -f NAME con show | grep -Fxq "$bond_name"
@@ -914,8 +917,7 @@ detect_os() {
 		is_rpm=true
 		is_supported=true
 	else
-		info "This OS is not supported. Security Onion requires Oracle Linux 9."
+		fail_setup "This OS is not supported. Security Onion requires Oracle Linux 9."
-		fail_setup
 	fi
 
 	info "Found OS: $OS $OSVER"
@@ -923,7 +925,7 @@ detect_os() {
 
 download_elastic_agent_artifacts() {
 	if ! update_elastic_agent 2>&1 | tee -a "$setup_log"; then
-		fail_setup
+		fail_setup "Failed to update Elastic Agent"
 	fi
 }
 
@@ -1567,7 +1569,7 @@ proxy_validate() {
 		error "Received error: $proxy_test_err"
 		if [[ -n $TESTING ]]; then
 			error "Exiting setup"
-			kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup
+			kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup "Proxy validation failed"
 		fi
 	fi
 	return $ret
@@ -1774,8 +1776,7 @@ ensure_pyyaml() {
 	local result=$?
 	set +o pipefail
 	if [[ $result -ne 0 ]] || ! rpm -q python3-pyyaml >/dev/null 2>&1; then
-		error "Failed to install python3-pyyaml (exit=$result)"
+		fail_setup "Failed to install python3-pyyaml (exit=$result)"
-		fail_setup
 	fi
 	info "python3-pyyaml installed successfully"
 }
@@ -1910,8 +1911,8 @@ repo_sync_local() {
 
 	if [[ ! $is_airgap ]]; then
 		curl --retry 5 --retry-delay 60 -A "netinstall/$SOVERSION/$OS/$(uname -r)/1" https://sigs.securityonion.net/checkup --output /tmp/install
-		retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" >> "$setup_log" 2>&1 || fail_setup
+		retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" >> "$setup_log" 2>&1 || fail_setup "Failed to sync repos"
-		retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/" >> "$setup_log" 2>&1 || fail_setup
+		retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/" >> "$setup_log" 2>&1 || fail_setup "Failed to sync kernel repos"
 		# After the download is complete run createrepo
 		create_repo
 	fi
@@ -1924,10 +1925,10 @@ saltify() {
 
 	if [[ $waitforstate ]]; then
 		# install all for a manager
-		retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -M -X stable $SALTVERSION" || fail_setup
+		retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -M -X stable $SALTVERSION" || fail_setup "Failed to install salt master"
 	else
 		# just a minion
-		retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -X stable $SALTVERSION" || fail_setup
+		retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -X stable $SALTVERSION" || fail_setup "Failed to install salt minion"
 	fi
 
 	salt_install_module_deps
@@ -1999,7 +2000,7 @@ set_main_ip() {
 			info "MAINIP=$MAINIP"
 			info "MNIC_IP=$MNIC_IP"
 			whiptail_error_message "The management IP could not be determined. Please check the log at /root/sosetup.log and verify the network configuration. Select OK to exit."
-			fail_setup
+			fail_setup "Could not determine MAINIP or MNIC_IP"
 		fi
 		sleep 1
 	done
@@ -2203,7 +2204,7 @@ set_initial_firewall_access() {
 set_management_interface() {
 	title "Setting up the main interface"
 	if [[ $MNIC == "bond1" ]]; then
-		configure_management_bond || fail_setup
+		configure_management_bond || fail_setup "Failed to configure management bond"
 	fi
 
 	if [ "$address_type" = 'DHCP' ]; then

setup/so-setup

@@ -90,8 +90,7 @@ if [[ "$setup_type" == 'iso' ]]; then
 	if [[ $is_rpm ]]; then
 		is_iso=true
 	else
-		echo "Only use 'so-setup iso' for an ISO install on Security Onion ISO images. Please run 'so-setup network' instead."
+		fail_setup "Only use 'so-setup iso' for an ISO install on Security Onion ISO images. Please run 'so-setup network' instead."
-		fail_setup
 	fi
 fi
 
@@ -130,7 +129,7 @@ catch() {
 	info "Fatal error occurred at $1 in so-setup, failing setup."
 	grep --color=never "ERROR" "$setup_log" > "$error_log"
 	whiptail_setup_failed
-	fail_setup
+	fail_setup "Fatal error occurred at $1 in so-setup"
 }
 
 # Add the progress function for manager node type installs
@@ -238,8 +237,7 @@ case "$setup_type" in
 		info "Beginning Security Onion $setup_type install"
 		;;
 	*)
-		error "Invalid install type, must be 'iso', 'network' or 'desktop'."
+		fail_setup "Invalid install type, must be 'iso', 'network' or 'desktop'."
-		fail_setup
 		;;
 esac
 
@@ -773,8 +771,7 @@ if ! [[ -f $install_opt_file ]]; then
 		logCmd "salt-call state.apply -l info registry"
 		title "Seeding the docker registry"
 		if ! docker_seed_registry; then
-			error "Failed to seed the docker registry"
+			fail_setup "Failed to seed the docker registry"
-			fail_setup
 		fi
 		title "Applying the manager state"
 		logCmd "salt-call state.apply -l info manager"
@@ -796,9 +793,8 @@ if ! [[ -f $install_opt_file ]]; then
 		logCmd "so-soc-restart"
 		title "Setting up Elastic Fleet"
 		logCmd "salt-call state.apply elasticfleet.config"	
-		if ! logCmd so-elastic-fleet-setup; then
+		if ! so-elastic-fleet-setup; then
-			error "Failed to run so-elastic-fleet-setup"
+			fail_setup "Failed to run so-elastic-fleet-setup"
-			fail_setup
 		fi
 		mark_setup_complete
 		set_initial_firewall_access

setup/so-verify

@@ -143,15 +143,15 @@ main() {
         cat $error_log
         echo "--------------------------"
         exit_code=1
-        touch /root/failure
+        echo "Found setup errors. Check $error_log for details" > /root/failure
     elif using_iso && cron_error_in_mail_spool; then
         echo "WARNING: Unexpected cron job output in mail spool"
         exit_code=1
-        touch /root/failure
+        echo "Unexpected cron job output found in /var/spool/mail/" > /root/failure
     elif is_manager_node && status_failed; then
         echo "WARNING: Containers are not in a healthy state"
         exit_code=1
-        touch /root/failure
+        echo "Containers are not in a healthy state. Check so-status for details" > /root/failure
     else
         echo "Successfully completed setup!"
         touch /root/success