From 9652a2053b7169be5e98cc31042a4f10fa20b665 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 10 Sep 2026 11:38:05 -0400 Subject: [PATCH 01/18] Stop the Zeek container gracefully zeekctl's post-terminate archives the final logs in the background and returns immediately unless StopWait is set, so the container exits and takes the archiving with it, stranding unarchived logs in /nsm/zeek/spool/tmp on every restart. Docker's default 10s grace is also too tight for the entrypoint's SIGTERM trap; overrunning it means SIGKILL and crash directories on the next start. Both are needed. StopWait alone gives the stop more work to do inside the same 10s window, which was measured ending in SIGKILL with logs stranded in the spool. disabled.sls used docker rm -f, which never delivers SIGTERM, so stop the container before removing it. Reported in discussion #16174. --- salt/zeek/defaults.yaml | 1 + salt/zeek/disabled.sls | 10 ++++++++++ salt/zeek/enabled.sls | 4 ++++ salt/zeek/soc_zeek.yaml | 12 ++++++++++++ 4 files changed, 27 insertions(+) diff --git a/salt/zeek/defaults.yaml b/salt/zeek/defaults.yaml index c68339584..24e1d8ca0 100644 --- a/salt/zeek/defaults.yaml +++ b/salt/zeek/defaults.yaml @@ -18,6 +18,7 @@ zeek: StatsLogEnable: 0 StatsLogExpireInterval: 0 StatusCmdShowAll: 0 + StopWait: 1 CrashExpireInterval: 0 SitePolicyScripts: local.zeek LogDir: /nsm/zeek/logs diff --git a/salt/zeek/disabled.sls b/salt/zeek/disabled.sls index c75cdd7b1..2025327fd 100644 --- a/salt/zeek/disabled.sls +++ b/salt/zeek/disabled.sls @@ -9,9 +9,19 @@ include: - zeek.sostatus +# Stop first so the entrypoint's SIGTERM trap can archive the final logs; docker_container.absent +# with force is a 'docker rm -f', which never delivers SIGTERM. force stays so the state still +# converges if the stop overruns. +so-zeek_stopped: + docker_container.stopped: + - name: so-zeek + - error_on_absent: False + so-zeek: docker_container.absent: - force: True + - require: + - docker_container: so-zeek_stopped so-zeek_so-status.disabled: file.comment: diff --git a/salt/zeek/enabled.sls b/salt/zeek/enabled.sls index 453990acb..ec8148e90 100644 --- a/salt/zeek/enabled.sls +++ b/salt/zeek/enabled.sls @@ -19,6 +19,10 @@ so-zeek: - restart_policy: unless-stopped - start: True - privileged: True + # Docker's default 10s grace is not enough for the entrypoint's SIGTERM trap to run + # 'zeekctl stop' and let StopWait archive the final logs. Overrunning it means SIGKILL, + # which strands those logs in spool/tmp and marks every node crashed on the next start. + - stop_timeout: 180 {% if DOCKERMERGED.containers['so-zeek'].ulimits %} - ulimits: {% for ULIMIT in DOCKERMERGED.containers['so-zeek'].ulimits %} diff --git a/salt/zeek/soc_zeek.yaml b/salt/zeek/soc_zeek.yaml index 1a12f7f5f..f1cc9e14d 100644 --- a/salt/zeek/soc_zeek.yaml +++ b/salt/zeek/soc_zeek.yaml @@ -99,6 +99,18 @@ zeek: regexFailureMessage: You must enter a whole number of days, or 0 to keep crash directories forever. helpLink: zeek advanced: True + StopWait: + description: >- + Set to 1 to make "zeekctl stop" wait for the final logs to be archived instead of + letting that finish in the background. Security Onion stops Zeek by stopping its + container, so anything still running in the background is killed when the container + exits - without this, the last logs of each run are stranded unarchived in + /nsm/zeek/spool/tmp and never reach Elasticsearch. It is read only for that reason. + regex: ^[01]$ + regexFailureMessage: You must enter 0 or 1. + helpLink: zeek + advanced: True + readonly: True MinDiskSpace: description: >- Percentage of free disk space below which ZeekControl reports a warning, or 0 to disable the check From 8f253d17a6bfebb3ff2d79449e6b57a83a76fc41 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Thu, 10 Sep 2026 15:18:35 -0600 Subject: [PATCH 02/18] Default Memory to Disabled Gives users a chance to reconfigure embed model before messages they send to the OnionAI get sent to SOAI by default. --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index d14ca3e08..6352ea371 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1537,7 +1537,7 @@ soc: Orchestrator: sonnet@SOAI Investigator: gemma@SOAI DetectionEngineer: gemma@SOAI - useMemory: true + useMemory: false useMemoryScanner: false dontScanBefore: "" memoryScanIntervalSeconds: 300 From 66e78633366d24dea6cde600a1a9b65b58694d59 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 11 Sep 2026 08:56:00 -0400 Subject: [PATCH 03/18] Update HOTFIX version to 3.3.0-20260911 --- HOTFIX | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HOTFIX b/HOTFIX index 8b1378917..b17d2ca5d 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1 +1 @@ - +3.3.0-20260911 From e4e8b90b9c7b7bf6fe1f5970a8e8c36dfcc71052 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 11 Sep 2026 09:03:24 -0400 Subject: [PATCH 04/18] Update version number in HOTFIX file --- HOTFIX | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HOTFIX b/HOTFIX index b17d2ca5d..59ed0b69b 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1 +1 @@ -3.3.0-20260911 +20260911 From cb489095789df76370df1d3f12e3eb460ad5abd7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 11 Sep 2026 10:12:24 -0400 Subject: [PATCH 05/18] Add x86-64-v3 CPU pre-flight check to soup Upstream Elastic now ships binaries built for the x86-64-v3 micro-architecture level. This is not a Security Onion choice: nodes whose CPUs predate x86-64-v3 can no longer run Elastic's own builds, so those nodes break once they are upgraded. Check for support before soup modifies anything, and require the operator to type "override" to proceed when a node is unsupported or offline. Runs after upgrade_check so a grid that is already current exits without prompting. Targets only the roles that run a container built from the so-elastic-agent image, using the role lists already maintained in salt/reactor/pillar_push_map.yaml. Adds --skip-cpu-check to bypass the gate for automation, and exit code 162 when the operator declines to override. --- salt/manager/tools/sbin/soup | 96 +++++++++++++++++++++++++++++++++++- 1 file changed, 95 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 353158b7e..7ab6d7598 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -120,6 +120,9 @@ check_err() { 161) echo 'Required intermediate Elasticsearch upgrade not complete' ;; + 162) + echo 'One or more Elastic Agent nodes do not support the x86-64-v3 CPU instruction set' + ;; 170) echo "Intermediate upgrade completed successfully to $next_step_so_version, but next soup to Security Onion $originally_requested_so_version could not be started automatically." echo "Start soup again manually to continue the upgrade to Security Onion $originally_requested_so_version." @@ -347,6 +350,83 @@ check_cluster_health() { exit 0 } +no_soup_for_you() { + echo "" + echo "No soup for you!" + exit 162 +} + +check_cpu_compatibility() { + # Roles running a container built from the so-elastic-agent image; mirrors the + # elasticagent and elasticfleet entries in salt/reactor/pillar_push_map.yaml. + local cpu_target='G@role:so-heavynode or G@role:so-eval or G@role:so-fleet or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone' + local expected_nodes cpu_results node result confirm + local -a unsupported=() offline=() + + echo "Checking that Elastic Agent nodes support the x86-64-v3 CPU instruction set now required by Elastic." + + if [[ "$SKIP_CPU_CHECK" == "true" ]]; then + printf "\nSkipping the x86-64-v3 CPU check because --skip-cpu-check was specified.\n\n" + return + fi + + # Nodes that never answer are absent from the results, so diff against who should have. + expected_nodes=$(salt -C "$cpu_target" --preview-target --out=json 2>/dev/null | jq -r '.[]?') || true + if [[ -z "$expected_nodes" ]]; then + printf "\nCould not determine which nodes run the Elastic Agent, so the x86-64-v3 CPU check cannot run.\n" + no_soup_for_you + fi + + cpu_results=$(salt -t 30 -C "$cpu_target" cmd.run "/lib64/ld-linux-x86-64.so.2 --help | grep x86-64-v3" --out=json 2>/dev/null) || true + + while IFS= read -r node; do + [[ -z "$node" ]] && continue + result=$(jq -r --arg node "$node" '.[$node] // empty' <<< "$cpu_results" 2>/dev/null) + if [[ -z "$result" || "$result" == *"did not return"* ]]; then + offline+=("$node") + elif [[ "$result" != *"x86-64-v3 (supported"* ]]; then + # glibc appends "(supported, searched)" only when supported; the open paren keeps + # this from matching a future "(unsupported". + unsupported+=("$node") + fi + done <<< "$expected_nodes" + + if [[ ${#unsupported[@]} -eq 0 && ${#offline[@]} -eq 0 ]]; then + printf "\nAll Elastic Agent nodes support x86-64-v3. We can proceed with SOUP.\n\n" + return + fi + + echo "" + if [[ ${#unsupported[@]} -gt 0 ]]; then + echo "The following node(s) do NOT support the x86-64-v3 CPU instruction set:" + printf ' %s\n' "${unsupported[@]}" + echo "" + echo "Upstream Elastic now builds its binaries for x86-64-v3, so these nodes can no" + echo "longer run Elastic. Upgrading them WILL BREAK them." + echo "" + fi + if [[ ${#offline[@]} -gt 0 ]]; then + echo "The following node(s) did not respond and could not be checked:" + printf ' %s\n' "${offline[@]}" + echo "" + echo "These nodes are offline, so we cannot confirm they support x86-64-v3, which" + echo "upstream Elastic now requires." + echo "" + fi + + if [[ -n $UNATTENDED ]]; then + echo "Unattended mode cannot prompt for an override. Re-run soup interactively, or pass --skip-cpu-check to bypass this check." + no_soup_for_you + fi + + read -rp "Type 'override' to continue anyway, or press Enter to exit: " confirm + if [[ "${confirm,,}" == "override" ]]; then + printf "\nOverride accepted. Continuing at your own risk.\n\n" + else + no_soup_for_you + fi +} + check_fleet_server() { echo "Checking that Elastic Fleet Server is responding." # Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls, @@ -1977,6 +2057,9 @@ main() { echo "Let's see if we need to update Security Onion." upgrade_check + + check_cpu_compatibility + upgrade_space echo "Verifying Elasticsearch version compatibility across the grid before upgrading." @@ -2255,6 +2338,17 @@ fi echo "### soup has been served at $(date) ###" } +SKIP_CPU_CHECK=false +declare -a SOUP_ARGS=() +for arg in "$@"; do + if [[ "$arg" == "--skip-cpu-check" ]]; then + SKIP_CPU_CHECK=true + else + SOUP_ARGS+=("$arg") + fi +done +set -- "${SOUP_ARGS[@]}" + while getopts ":b:f:y" opt; do case ${opt} in b ) @@ -2278,7 +2372,7 @@ while getopts ":b:f:y" opt; do ISOLOC="$OPTARG" ;; \? ) - echo "Usage: soup [-b] [-y] [-f ]" + echo "Usage: soup [-b] [-y] [-f ] [--skip-cpu-check]" exit 1 ;; : ) From 1624e8c094f269199146ca2b93ce5a5c3eb8b17b Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Fri, 11 Sep 2026 10:54:20 -0600 Subject: [PATCH 06/18] filterEventFields Previously hard coded array of fields is now a config option with the hard coded value as the default value. --- salt/soc/defaults.yaml | 63 ++++++++++++++++++++++++++++++++++++++++++ salt/soc/soc_soc.yaml | 5 ++++ 2 files changed, 68 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index d14ca3e08..d0c60285f 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1556,6 +1556,69 @@ soc: reconcilePersona: "" toolUseTurnAttempts: 12 toolUseTurnDelayMs: 175 + tools: + filterEventFields: + - "@timestamp" + - "client.name" + - "destination.ip" + - "destination.port" + - "destination.geo.country_name" + - "dns.query.name" + - "dns.query_name" + - "event.action" + - "event.category" + - "event.module" + - "event.dataset" + - "event.outcome" + - "event.severity" + - "event.severity_label" + - "event.type" + - "event_data.agent.name" + - "event_data.host.os.name" + - "file.mime_type" + - "file.name" + - "hash.md5" + - "hash.sha1" + - "host.mac" + - "host.name" + - "host.os.name" + - "http.method" + - "http.useragent" + - "http.virtual_host" + - "log.id.uid" + - "network.community_id" + - "network.protocol" + - "network.transport" + - "notice.message" + - "observer.name" + - "process.name" + - "process.executable" + - "process.entity_id" + - "process.command_line" + - "process.Ext.ancestry" + - "process.parent.entity_id" + - "process.parent.command_line" + - "rule.category" + - "rule.name" + - "rule.uuid" + - "software.name" + - "software.type" + - "software.version.unparsed" + - "source.ip" + - "source.port" + - "source.geo.country_name" + - "ssh.cypher_algorithm" + - "ssh.client" + - "ssh.server" + - "ssl.cipher" + - "ssl.server_name" + - "ssl.version" + - "system.auth.sudo.command" + - "user.name" + - "user.domain" + - "user.effective.name" + - "weird.name" + - "tags" onionconfig: saltstackDir: /opt/so/saltstack bypassEnabled: false diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 995d97493..2f730bf42 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -916,6 +916,11 @@ soc: description: The number of times to retry extracting memories from a session if errors occur. global: True advanced: True + tools: + filterEventFields: + description: A whitelist of fields to return when OnionAI uses the query_events tool. All other fields are removed. One field per line. + global: True + multiline: True client: assistant: enabled: From bee03d5baedc127868f38fee1f7319d8d9c996e1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 11 Sep 2026 16:13:09 -0400 Subject: [PATCH 07/18] 3.3.0 hotfix --- DOWNLOAD_AND_VERIFY_ISO.md | 22 +++++++++++----------- sigs/securityonion-3.3.0-20260911.iso.sig | Bin 0 -> 566 bytes 2 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 sigs/securityonion-3.3.0-20260911.iso.sig diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index 284b89ae3..11aff3ebb 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,17 +1,17 @@ -### 3.3.0-20260908 ISO image released on 2026/09/08 +### 3.3.0-20260911ISO image released on 2026/09/11 ### Download and Verify -3.3.0-20260908 ISO image: -https://download.securityonion.net/file/securityonion/securityonion-3.3.0-20260908.iso +3.3.0-20260911ISO image: +https://download.securityonion.net/file/securityonion/securityonion-3.3.0-20260911.iso -MD5: 5A2C42D0083F2D7B4DC2178C30EBC05F -SHA1: 505220A8A3315AFEE601C13772996018425CCD29 -SHA256: 6EB8401296A1D051FEC558C520D2D4AB1912A72D351A2426FBF8C87FEE2BA844 +MD5: 12B18433D3A2198A185892FF79CF638F +SHA1: 2B3C2E1FA7A78ED1F956E7EDCC12E32593C14EEE +SHA256: 0938C73B76CE30EC9E4394D312C79EA7CAC721B6818541697279A6221F7D870D Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.3.0-20260908.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.3.0-20260911.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS @@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/ Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.3.0-20260908.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.3.0-20260911.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-3.3.0-20260908.iso +wget https://download.securityonion.net/file/securityonion/securityonion-3.3.0-20260911.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-3.3.0-20260908.iso.sig securityonion-3.3.0-20260908.iso +gpg --verify securityonion-3.3.0-20260911.iso.sig securityonion-3.3.0-20260911.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Tue 08 Sep 2026 10:07:12 AM EDT using RSA key ID FE507013 +gpg: Signature made Fri 11 Sep 2026 11:23:56 AM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-3.3.0-20260911.iso.sig b/sigs/securityonion-3.3.0-20260911.iso.sig new file mode 100644 index 0000000000000000000000000000000000000000..3f90eed8b3f48ff85ab9b9da243b7831ea3ec98b GIT binary patch literal 566 zcmV-60?GY}0y6{v0SEvc79j-41gSkXz6^6dp_W8^5Ma0dP;e6k0&1ik3;+rV5PT3| zxBgIY6S@l!0G8Xbjr@wW&r0HdFoxB~)^;I_h>)VMDjG#yN4MES%e{%h3y^xv++}G< z>v39FTb=Mf`)-m2ndknBBSI`rTSpYvK%$)1OgU;P!v|A`s|>{?x!S_y{#OWm{~*U~ zo1=<8-n3zc7(nei5V-&^32DCg~w2f0iO1n8wWvf;t| z9r~+bctW#nlFMl7;-x6W-c?4wLu0!Dh~K_KCwxxJCB5m1!U7u!QKKmrSyFlE3qs~( z{w5-rYJ8 Date: Fri, 11 Sep 2026 16:16:01 -0400 Subject: [PATCH 08/18] 3.3.0 hotfix --- DOWNLOAD_AND_VERIFY_ISO.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index 11aff3ebb..a9732437e 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -3,7 +3,7 @@ ### Download and Verify -3.3.0-20260911ISO image: +3.3.0-20260911 ISO image: https://download.securityonion.net/file/securityonion/securityonion-3.3.0-20260911.iso MD5: 12B18433D3A2198A185892FF79CF638F From 7245843a3c0323fe61c97f11c1adbcde1b2b0258 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 11 Sep 2026 16:17:34 -0400 Subject: [PATCH 09/18] 3.3.0 hotfix --- DOWNLOAD_AND_VERIFY_ISO.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index a9732437e..68ae7f2c4 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,4 +1,4 @@ -### 3.3.0-20260911ISO image released on 2026/09/11 +### 3.3.0-20260911 ISO image released on 2026/09/11 ### Download and Verify From 52037314be44de745da9969f02547f01f221b8d6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 11 Sep 2026 16:47:13 -0400 Subject: [PATCH 10/18] Remove outdated HOTFIX version number --- HOTFIX | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HOTFIX b/HOTFIX index 59ed0b69b..8b1378917 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1 +1 @@ -20260911 + From 84cd9667367727cb8714e1839a79a1d9b201f1eb Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 14 Sep 2026 12:14:13 -0400 Subject: [PATCH 11/18] FIX: prevent so-sensor-clean runaway loop and concurrent instances The cleanup loop had no progress check or pass limit, so once /nsm was over threshold with nothing left to reclaim it spun at full speed, writing 1.1 GB / 12.5M lines to sensor_clean.log in five hours. Stop when a pass removes nothing, when a pass frees no space, or at MAX_PASSES, and drop the per-pass "no old files" logging in favor of one actionable line. Replace the pgrep guard with flock -n. A find|while read subshell inherits the parent's argv, so pgrep -cf counted one instance as hundreds; it was also check-then-act, which let cron stack up overlapping runs. Paths now derive from SENSOR_DIR with env-overridable LOG/LOCK so the over-threshold path can be tested against a scratch filesystem. --- salt/common/tools/sbin/so-sensor-clean | 125 +++++++++++++++---------- 1 file changed, 78 insertions(+), 47 deletions(-) diff --git a/salt/common/tools/sbin/so-sensor-clean b/salt/common/tools/sbin/so-sensor-clean index 083a316b9..ef061beff 100755 --- a/salt/common/tools/sbin/so-sensor-clean +++ b/salt/common/tools/sbin/so-sensor-clean @@ -8,21 +8,37 @@ # Elastic License 2.0. -SENSOR_DIR='/nsm' +SENSOR_DIR="${SENSOR_DIR:-/nsm}" CRIT_DISK_USAGE=90 -CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %) -LOG="/opt/so/log/sensor_clean.log" -TODAY=$(date -u "+%Y-%m-%d") +LOG="${LOG:-/opt/so/log/sensor_clean.log}" +LOCK="${LOCK:-/var/tmp/so-sensor-clean.lock}" +MAX_PASSES=100 +ZEEK_LOGS="$SENSOR_DIR/zeek/logs" +STRELKA_FILES="$SENSOR_DIR/strelka/processed" +SURICATA_LOGS="$SENSOR_DIR/suricata" +PCAPS="$SENSOR_DIR/pcapout" + +log() { + echo "$(date) - $*" >>"$LOG" +} + +disk_usage() { + df -P "$SENSOR_DIR" | tail -1 | awk '{print $5}' | tr -d % +} + +disk_avail() { + df -P "$SENSOR_DIR" | tail -1 | awk '{print $4}' +} + +# sets REMOVED=1 if anything was actually deleted clean() { ## find the oldest Zeek logs directory - OLDEST_DIR=$(ls /nsm/zeek/logs/ | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1) - if [ -z "$OLDEST_DIR" -o "$OLDEST_DIR" == ".." -o "$OLDEST_DIR" == "." ]; then - echo "$(date) - No old Zeek logs available to clean up in /nsm/zeek/logs/" >>$LOG - #exit 0 - else - echo "$(date) - Removing directory: /nsm/zeek/logs/$OLDEST_DIR" >>$LOG - rm -rf /nsm/zeek/logs/"$OLDEST_DIR" + OLDEST_DIR=$(ls "$ZEEK_LOGS" 2>/dev/null | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1) + if [ -n "$OLDEST_DIR" ]; then + log "Removing directory: $ZEEK_LOGS/$OLDEST_DIR" + rm -rf "$ZEEK_LOGS/$OLDEST_DIR" + REMOVED=1 fi ## Remarking for now, as we are moving extracted files to /nsm/strelka/processed @@ -43,58 +59,73 @@ clean() { #fi ## Clean up Zeek extracted files processed by Strelka - STRELKA_FILES='/nsm/strelka/processed' - OLDEST_STRELKA=$(find $STRELKA_FILES -type f -printf '%T+ %p\n' | sort -n | head -n 1) - if [ -z "$OLDEST_STRELKA" -o "$OLDEST_STRELKA" == ".." -o "$OLDEST_STRELKA" == "." ]; then - echo "$(date) - No old files available to clean up in $STRELKA_FILES" >>$LOG - else + OLDEST_STRELKA=$(find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1) + if [ -n "$OLDEST_STRELKA" ]; then OLDEST_STRELKA_DATE=$(echo $OLDEST_STRELKA | awk '{print $1}' | cut -d+ -f1) - OLDEST_STRELKA_FILE=$(echo $OLDEST_STRELKA | awk '{print $2}') - echo "$(date) - Removing extracted files for $OLDEST_STRELKA_DATE" >>$LOG - find $STRELKA_FILES -type f -printf '%T+ %p\n' | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do - echo "$(date) - Removing file: $FILE" >>$LOG + log "Removing extracted files for $OLDEST_STRELKA_DATE" + REMOVED=1 + find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do + log "Removing file: $FILE" rm -f "$FILE" done fi ## Clean up Suricata log files - SURICATA_LOGS='/nsm/suricata' - OLDEST_SURICATA=$(find $SURICATA_LOGS -type f -printf '%T+ %p\n' | sort -n | head -n 1) - if [[ -z "$OLDEST_SURICATA" ]] || [[ "$OLDEST_SURICATA" == ".." ]] || [[ "$OLDEST_SURICATA" == "." ]]; then - echo "$(date) - No old files available to clean up in $SURICATA_LOGS" >>$LOG - else + OLDEST_SURICATA=$(find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1) + if [ -n "$OLDEST_SURICATA" ]; then OLDEST_SURICATA_DATE=$(echo $OLDEST_SURICATA | awk '{print $1}' | cut -d+ -f1) - OLDEST_SURICATA_FILE=$(echo $OLDEST_SURICATA | awk '{print $2}') - echo "$(date) - Removing logs for $OLDEST_SURICATA_DATE" >>$LOG - find $SURICATA_LOGS -type f -printf '%T+ %p\n' | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do - echo "$(date) - Removing file: $FILE" >>$LOG + log "Removing logs for $OLDEST_SURICATA_DATE" + REMOVED=1 + find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do + log "Removing file: $FILE" rm -f "$FILE" done fi ## Clean up extracted pcaps - PCAPS='/nsm/pcapout' - OLDEST_PCAP=$(find $PCAPS -type f -printf '%T+ %p\n' | sort -n | head -n 1) - if [ -z "$OLDEST_PCAP" -o "$OLDEST_PCAP" == ".." -o "$OLDEST_PCAP" == "." ]; then - echo "$(date) - No old files available to clean up in $PCAPS" >>$LOG - else + OLDEST_PCAP=$(find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1) + if [ -n "$OLDEST_PCAP" ]; then OLDEST_PCAP_DATE=$(echo $OLDEST_PCAP | awk '{print $1}' | cut -d+ -f1) - OLDEST_PCAP_FILE=$(echo $OLDEST_PCAP | awk '{print $2}') - echo "$(date) - Removing extracted files for $OLDEST_PCAP_DATE" >>$LOG - find $PCAPS -type f -printf '%T+ %p\n' | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do - echo "$(date) - Removing file: $FILE" >>$LOG + log "Removing extracted files for $OLDEST_PCAP_DATE" + REMOVED=1 + find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do + log "Removing file: $FILE" rm -f "$FILE" done fi } -# Check to see if we are already running -NUM_RUNNING=$(pgrep -cf "/bin/bash /usr/sbin/so-sensor-clean") -[ "$NUM_RUNNING" -gt 1 ] && echo "$(date) - $NUM_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0 - -if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then - while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do - clean - CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %) - done +# Only one instance at a time; the lock is the fd, so it releases on any exit +exec 9>"$LOCK" || exit 1 +if ! flock -n 9; then + log "another so-sensor-clean is already running (lock $LOCK held); exiting" + exit 0 fi + +CUR_USAGE=$(disk_usage) +[ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ] || exit 0 + +log "$SENSOR_DIR at ${CUR_USAGE}% (threshold ${CRIT_DISK_USAGE}%); starting cleanup" + +PASS=0 +while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do + PASS=$((PASS + 1)) + if [ "$PASS" -gt "$MAX_PASSES" ]; then + log "stopping after $MAX_PASSES passes; $SENSOR_DIR still at ${CUR_USAGE}%" + break + fi + + REMOVED=0 + BEFORE=$(disk_avail) + clean + CUR_USAGE=$(disk_usage) + + if [ "$REMOVED" -eq 0 ]; then + log "nothing left to remove in $ZEEK_LOGS, $STRELKA_FILES, $SURICATA_LOGS, $PCAPS; $SENSOR_DIR still at ${CUR_USAGE}% - space is consumed outside of NSM cleanup scope" + break + fi + if [ "$(disk_avail)" -le "$BEFORE" ]; then + log "pass $PASS freed no space; $SENSOR_DIR still at ${CUR_USAGE}% - stopping until next run" + break + fi +done From b018277d68f07242b4ca53e23a8e288ac41e9309 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 15 Sep 2026 09:58:38 -0400 Subject: [PATCH 12/18] FIX: prevent joining minion from executing code as root on the manager so-minion exported every line of the minion-controlled /opt/so/install.txt into its root shell and wrote the values unescaped into a Jinja-rendered pillar, allowing a rogue node to redefine PILLARFILE or run code on the master at the next pillar compile. pcapspace also fed minion-returned disk.usage output into bash arithmetic, which evaluates array subscripts. - Parse install.txt against an allowlist of known keys; never export - Validate MINION_ID before building pillar paths; make them readonly - Validate node type, IP, interface, hostname, heap and core values before any pillar is written; strip braces and control chars from the free-text node description - Require a numeric disk size before pcapspace arithmetic - Refuse manager node types on add/addVM so a remote node cannot rewrite the CA pillar; only setup may create them --- salt/manager/tools/sbin/so-minion | 87 ++++++++++++++++++++++++++++--- 1 file changed, 81 insertions(+), 6 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index ea93cee43..34bb6f8b1 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -121,8 +121,14 @@ for i in "$@"; do esac done -PILLARFILE=/opt/so/saltstack/local/pillar/minions/$MINION_ID.sls -ADVPILLARFILE=/opt/so/saltstack/local/pillar/minions/adv_$MINION_ID.sls +if [[ -n "$MINION_ID" && ! "$MINION_ID" =~ ^[A-Za-z0-9._-]{1,253}$ ]]; then + echo "Invalid minion id: $MINION_ID" + log "ERROR" "Invalid minion id: $MINION_ID" + exit 1 +fi + +readonly PILLARFILE=/opt/so/saltstack/local/pillar/minions/$MINION_ID.sls +readonly ADVPILLARFILE=/opt/so/saltstack/local/pillar/minions/adv_$MINION_ID.sls function getinstallinfo() { log "INFO" "Getting install info for minion $MINION_ID" @@ -133,10 +139,23 @@ function getinstallinfo() { return 1 fi - while read -r var; do export "$var"; done <<< "$INSTALLVARS" - if [ $? -ne 0 ]; then - log "ERROR" "Failed to source install variables" - return 1 + # install.txt is controlled by the minion; only accept known keys and never eval or export them + local line key + while IFS= read -r line; do + [[ "$line" == *=* ]] || continue + key=${line%%=*} + case "$key" in + MAINIP|MNIC|NODE_DESCRIPTION|ES_HEAP_SIZE|PATCHSCHEDULENAME|INTERFACE|NODETYPE|CORECOUNT|LSHOSTNAME|LSHEAP|CPUCORES|IDH_MGTRESTRICT|IDH_SERVICES) + printf -v "$key" '%s' "${line#*=}" + ;; + *) + log "WARN" "Ignoring unexpected install var from $MINION_ID: ${key:0:64}" + ;; + esac + done <<< "$INSTALLVARS" + + if [[ "$NODE_DESCRIPTION" == \'*\' ]]; then + NODE_DESCRIPTION=${NODE_DESCRIPTION:1:-1} fi log "INFO" "Fetched install info for $MINION_ID (node type: ${NODETYPE:-unset})" @@ -176,6 +195,12 @@ function pcapspace() { fi fi + # Must be checked before arithmetic expansion, which evaluates array subscripts + if [[ ! "$SPACESIZE" =~ ^[0-9]+$ ]]; then + log "ERROR" "Invalid disk size for $MINION_ID: ${SPACESIZE:0:64}" + return 1 + fi + local s=$(( $SPACESIZE / 1000000 )) local s1=$(( $s / 4 * $PCAP_PERCENTAGE )) @@ -1050,6 +1075,54 @@ function updateMineAndApplyStates() { fi } +# Values end up in a Jinja-rendered pillar and in bash, and may come from the minion +function validate_minion_vars() { + local error_msg="" + + case "$NODETYPE" in + EVAL|STANDALONE|MANAGER|MANAGERSEARCH|MANAGERHYPE|IMPORT) + # Manager pillars also rewrite the CA pillar, so never accept them from a remote node + [[ "$OPERATION" == "setup" ]] || error_msg="Node type $NODETYPE can only be configured during setup" + ;; + FLEET|IDH|HEAVYNODE|SENSOR|SEARCHNODE|RECEIVER|HYPERVISOR|DESKTOP) + ;; + *) + error_msg="Invalid node type: ${NODETYPE:0:64}" + ;; + esac + + if [[ -z "$error_msg" ]]; then + if ! valid_ip4 "$MAINIP"; then + error_msg="Invalid MAINIP: ${MAINIP:0:64}" + elif [[ ! "$MNIC" =~ ^[A-Za-z0-9._-]*$ ]]; then + error_msg="Invalid MNIC: ${MNIC:0:64}" + elif [[ ! "$INTERFACE" =~ ^[A-Za-z0-9._-]*$ ]]; then + error_msg="Invalid INTERFACE: ${INTERFACE:0:64}" + elif [[ ! "$LSHOSTNAME" =~ ^[A-Za-z0-9._-]*$ ]]; then + error_msg="Invalid LSHOSTNAME: ${LSHOSTNAME:0:64}" + elif [[ ! "$ES_HEAP_SIZE" =~ ^([0-9]+[kKmMgG]?)?$ ]]; then + error_msg="Invalid ES_HEAP_SIZE: ${ES_HEAP_SIZE:0:64}" + elif [[ ! "$LSHEAP" =~ ^([0-9]+[kKmMgG]?)?$ ]]; then + error_msg="Invalid LSHEAP: ${LSHEAP:0:64}" + elif [[ ! "$CORECOUNT" =~ ^[0-9]*$ ]]; then + error_msg="Invalid CORECOUNT: ${CORECOUNT:0:64}" + elif [[ ! "$CPUCORES" =~ ^[0-9]*$ ]]; then + error_msg="Invalid CPUCORES: ${CPUCORES:0:64}" + elif [[ ! "$IDH_MGTRESTRICT" =~ ^(True|False)?$ ]]; then + error_msg="Invalid IDH_MGTRESTRICT: ${IDH_MGTRESTRICT:0:64}" + fi + fi + + if [[ -n "$error_msg" ]]; then + log "ERROR" "$error_msg" + echo "$error_msg" + return 1 + fi + + # Free text; removing braces is enough to prevent any Jinja delimiter + NODE_DESCRIPTION=${NODE_DESCRIPTION//[\{\}[:cntrl:]]/} +} + function setupMinionFiles() { log "INFO" "Setting up minion files for $MINION_ID (pillar: $PILLARFILE)" @@ -1061,6 +1134,8 @@ function setupMinionFiles() { return 1 fi + validate_minion_vars || return 1 + # Create the base minion files create_minion_files || return 1 From a9f7ffc3fead4607cfded29f781e98467da57f35 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 15 Sep 2026 13:10:33 -0400 Subject: [PATCH 13/18] FIX: validate MAINIP without so-common during setup setup runs so-minion -o=setup before /usr/sbin/so-common is installed, so valid_ip4 was undefined, every MAINIP was rejected, and no minion pillar was written. Pillar compile then failed for the new manager and setup gave up waiting for the salt master. Use an inline IPv4 regex instead. --- salt/manager/tools/sbin/so-minion | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 34bb6f8b1..cbae05652 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -1078,6 +1078,9 @@ function updateMineAndApplyStates() { # Values end up in a Jinja-rendered pillar and in bash, and may come from the minion function validate_minion_vars() { local error_msg="" + # Inline rather than valid_ip4: so-common is not installed yet when setup runs -o=setup + local octet='(25[0-5]|2[0-4][0-9]|1?[0-9]?[0-9])' + local ip4_re="^($octet\.){3}$octet$" case "$NODETYPE" in EVAL|STANDALONE|MANAGER|MANAGERSEARCH|MANAGERHYPE|IMPORT) @@ -1092,7 +1095,7 @@ function validate_minion_vars() { esac if [[ -z "$error_msg" ]]; then - if ! valid_ip4 "$MAINIP"; then + if [[ ! "$MAINIP" =~ $ip4_re ]]; then error_msg="Invalid MAINIP: ${MAINIP:0:64}" elif [[ ! "$MNIC" =~ ^[A-Za-z0-9._-]*$ ]]; then error_msg="Invalid MNIC: ${MNIC:0:64}" From 1e86be11b23ee2a520bfa5c127eb0e960bd7e0a3 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 16 Sep 2026 10:34:20 -0400 Subject: [PATCH 14/18] FIX: install /usr/sbin management scripts as root Salt installed the so-* scripts into /usr/sbin owned by unprivileged service UIDs (939/socore, plus 930-960 per service) at mode 755, while root executes those same files from cron, systemd and state cmd.run. Any file-write primitive as one of those UIDs was therefore root. Two mechanisms behind this are not visible in the diff: file.recurse also manages the destination directory, so /usr/sbin itself was chowned to whichever service UID ran last. A directory's owner may always chmod it, so that UID could replace even the scripts already declared user: root -- so-config-backup, so-suricata-eve-clean, so-nsm-mount-nvme. usr_sbin_perms now pins the directory to root:root 555, the mode the filesystem RPM ships. Omitting user:/group: is a no-op on files that already exist, because check_perms only chowns when a user is named. Explicit user: root is what lets upgraded grids self-heal on the next highstate, and what makes a revert chown back rather than silently do nothing. --- salt/common/init.sls | 21 +++++++++++++++++---- salt/elastalert/config.sls | 4 ++-- salt/elasticagent/config.sls | 4 ++-- salt/elasticfleet/config.sls | 12 ++++++------ salt/elasticfleet/install_agent_grid.sls | 4 ++-- salt/elasticsearch/config.sls | 20 ++++++++++---------- salt/hypervisor/init.sls | 4 ++++ salt/idh/config.sls | 4 ++-- salt/influxdb/config.sls | 4 ++-- salt/kafka/config.sls | 8 ++++---- salt/kibana/config.sls | 8 ++++---- salt/libvirt/64962/init.sls | 2 ++ salt/logstash/config.sls | 4 ++-- salt/manager/init.sls | 8 ++++---- salt/nginx/config.sls | 4 ++-- salt/redis/config.sls | 8 ++++---- salt/salt/init.sls | 6 ++++-- salt/sensor/init.sls | 2 ++ salt/sensoroni/config.sls | 4 ++-- salt/soc/config.sls | 4 ++-- salt/strelka/config.sls | 4 ++-- salt/suricata/config.sls | 8 ++++---- salt/telegraf/config.sls | 4 ++-- salt/zeek/config.sls | 4 ++-- 24 files changed, 89 insertions(+), 66 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index bafd12fd6..34b84f09d 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -117,14 +117,25 @@ elastic_curl_config: {% endif %} +# A non-root owner here can chmod the directory and replace any script in it, including +# the root-owned ones. 555 is the mode the filesystem RPM ships; root ignores it anyway. +usr_sbin_perms: + file.directory: + - name: /usr/sbin + - user: root + - group: root + - mode: 555 + common_sbin: file.recurse: - name: /usr/sbin - source: salt://common/tools/sbin - - user: 939 - - group: 939 + - user: root + - group: root - file_mode: 755 - show_changes: False + - require: + - file: usr_sbin_perms {% if GLOBALS.role == 'so-heavynode' %} - exclude_pat: - so-pcap-import @@ -159,8 +170,8 @@ common_sbin_jinja: file.recurse: - name: /usr/sbin - source: salt://common/tools/sbin_jinja - - user: 939 - - group: 939 + - user: root + - group: root - file_mode: 755 - template: jinja - show_changes: False @@ -173,6 +184,8 @@ so-status_script: file.managed: - name: /usr/sbin/so-status - source: salt://common/tools/sbin/so-status + - user: root + - group: root - mode: 755 {% if GLOBALS.is_sensor %} diff --git a/salt/elastalert/config.sls b/salt/elastalert/config.sls index 25d5bf5f8..147666e6e 100644 --- a/salt/elastalert/config.sls +++ b/salt/elastalert/config.sls @@ -33,8 +33,8 @@ elastalert_sbin: file.recurse: - name: /usr/sbin - source: salt://elastalert/tools/sbin - - user: 933 - - group: 939 + - user: root + - group: root - file_mode: 755 #elastalert_sbin_jinja: diff --git a/salt/elasticagent/config.sls b/salt/elasticagent/config.sls index b54186fab..63992b199 100644 --- a/salt/elasticagent/config.sls +++ b/salt/elasticagent/config.sls @@ -39,8 +39,8 @@ elasticagent_sbin_jinja: file.recurse: - name: /usr/sbin - source: salt://elasticagent/tools/sbin_jinja - - user: 949 - - group: 939 + - user: root + - group: root - file_mode: 755 - template: jinja diff --git a/salt/elasticfleet/config.sls b/salt/elasticfleet/config.sls index 6a8919605..59f052b7f 100644 --- a/salt/elasticfleet/config.sls +++ b/salt/elasticfleet/config.sls @@ -31,8 +31,8 @@ elasticfleet_sbin: file.recurse: - name: /usr/sbin - source: salt://elasticfleet/tools/sbin - - user: 947 - - group: 939 + - user: root + - group: root - file_mode: 755 - show_changes: False @@ -40,8 +40,8 @@ elasticfleet_sbin_jinja: file.recurse: - name: /usr/sbin - source: salt://elasticfleet/tools/sbin_jinja - - user: 947 - - group: 939 + - user: root + - group: root - file_mode: 755 - template: jinja - exclude_pat: @@ -81,8 +81,8 @@ eapackageupgrade: file.managed: - name: /usr/sbin/so-elastic-fleet-package-upgrade - source: salt://elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade - - user: 947 - - group: 939 + - user: root + - group: root - mode: 755 - template: jinja diff --git a/salt/elasticfleet/install_agent_grid.sls b/salt/elasticfleet/install_agent_grid.sls index 7ed727b48..ca36b06de 100644 --- a/salt/elasticfleet/install_agent_grid.sls +++ b/salt/elasticfleet/install_agent_grid.sls @@ -14,8 +14,8 @@ so-elastic-agent-install: file.managed: - name: /usr/sbin/so-elastic-agent-install - source: salt://elasticfleet/tools/sbin/so-elastic-agent-install - - user: 947 - - group: 939 + - user: root + - group: root - mode: 755 - show_changes: False diff --git a/salt/elasticsearch/config.sls b/salt/elasticsearch/config.sls index cbfdca406..5c1645ca6 100644 --- a/salt/elasticsearch/config.sls +++ b/salt/elasticsearch/config.sls @@ -37,8 +37,8 @@ elasticsearch_sbin: file.recurse: - name: /usr/sbin - source: salt://elasticsearch/tools/sbin - - user: 930 - - group: 939 + - user: root + - group: root - file_mode: 755 - exclude_pat: - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state @@ -49,8 +49,8 @@ so-elasticsearch-system-indices-patch-script: file.managed: - name: /usr/sbin/so-elasticsearch-system-indices-patch - source: salt://elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch - - user: 930 - - group: 939 + - user: root + - group: root - mode: 755 - show_changes: False @@ -58,8 +58,8 @@ elasticsearch_sbin_jinja: file.recurse: - name: /usr/sbin - source: salt://elasticsearch/tools/sbin_jinja - - user: 939 - - group: 939 + - user: root + - group: root - file_mode: 755 - template: jinja - exclude_pat: @@ -72,8 +72,8 @@ so-elasticsearch-ilm-policy-load-script: file.managed: - name: /usr/sbin/so-elasticsearch-ilm-policy-load - source: salt://elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load - - user: 930 - - group: 939 + - user: root + - group: root - mode: 754 - template: jinja - defaults: @@ -84,8 +84,8 @@ so-elasticsearch-pipelines-script: file.managed: - name: /usr/sbin/so-elasticsearch-pipelines - source: salt://elasticsearch/tools/sbin/so-elasticsearch-pipelines - - user: 930 - - group: 939 + - user: root + - group: root - mode: 754 - show_changes: False diff --git a/salt/hypervisor/init.sls b/salt/hypervisor/init.sls index 4a2edad75..106d366b7 100644 --- a/salt/hypervisor/init.sls +++ b/salt/hypervisor/init.sls @@ -21,12 +21,16 @@ hypervisor_sbin: file.recurse: - name: /usr/sbin - source: salt://hypervisor/tools/sbin + - user: root + - group: root - file_mode: 744 hypervisor_sbin_jinja: file.recurse: - name: /usr/sbin - source: salt://hypervisor/tools/sbin_jinja + - user: root + - group: root - template: jinja - file_mode: 744 diff --git a/salt/idh/config.sls b/salt/idh/config.sls index 2e6315007..e17823604 100644 --- a/salt/idh/config.sls +++ b/salt/idh/config.sls @@ -86,8 +86,8 @@ idh_sbin: file.recurse: - name: /usr/sbin - source: salt://idh/tools/sbin - - user: 939 - - group: 939 + - user: root + - group: root - file_mode: 755 #idh_sbin_jinja: diff --git a/salt/influxdb/config.sls b/salt/influxdb/config.sls index bf8b67b78..819d78b98 100644 --- a/salt/influxdb/config.sls +++ b/salt/influxdb/config.sls @@ -41,8 +41,8 @@ influxdb_sbin: file.recurse: - name: /usr/sbin - source: salt://influxdb/tools/sbin - - user: 939 - - group: 939 + - user: root + - group: root - file_mode: 755 #influxdb_sbin_jinja: diff --git a/salt/kafka/config.sls b/salt/kafka/config.sls index 1bedf96a0..6a7c30c94 100644 --- a/salt/kafka/config.sls +++ b/salt/kafka/config.sls @@ -30,16 +30,16 @@ kafka_sbin_tools: file.recurse: - name: /usr/sbin - source: salt://kafka/tools/sbin - - user: 960 - - group: 960 + - user: root + - group: root - file_mode: 755 kafka_sbin_jinja_tools: file.recurse: - name: /usr/sbin - source: salt://kafka/tools/sbin_jinja - - user: 960 - - group: 960 + - user: root + - group: root - file_mode: 755 - template: jinja - defaults: diff --git a/salt/kibana/config.sls b/salt/kibana/config.sls index 433fc9601..bc4e5f431 100644 --- a/salt/kibana/config.sls +++ b/salt/kibana/config.sls @@ -36,16 +36,16 @@ kibana_sbin: file.recurse: - name: /usr/sbin - source: salt://kibana/tools/sbin - - user: 932 - - group: 939 + - user: root + - group: root - file_mode: 755 kibana_sbin_jinja: file.recurse: - name: /usr/sbin - source: salt://kibana/tools/sbin_jinja - - user: 932 - - group: 939 + - user: root + - group: root - file_mode: 755 - template: jinja - defaults: diff --git a/salt/libvirt/64962/init.sls b/salt/libvirt/64962/init.sls index e20f24f2d..973da0e97 100644 --- a/salt/libvirt/64962/init.sls +++ b/salt/libvirt/64962/init.sls @@ -6,6 +6,8 @@ so-fix-salt-ldap_script: file.managed: - name: /usr/sbin/so-fix-salt-ldap.py - source: salt://libvirt/64962/scripts/so-fix-salt-ldap.py + - user: root + - group: root - mode: 744 fix-salt-ldap: diff --git a/salt/logstash/config.sls b/salt/logstash/config.sls index 80fbe44b2..7a09349fc 100644 --- a/salt/logstash/config.sls +++ b/salt/logstash/config.sls @@ -40,8 +40,8 @@ logstash_sbin: file.recurse: - name: /usr/sbin - source: salt://logstash/tools/sbin - - user: 931 - - group: 939 + - user: root + - group: root - file_mode: 755 #logstash_sbin_jinja: diff --git a/salt/manager/init.sls b/salt/manager/init.sls index a74d16ff2..85df680d4 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -113,8 +113,8 @@ manager_sbin: file.recurse: - name: /usr/sbin - source: salt://manager/tools/sbin - - user: 939 - - group: 939 + - user: root + - group: root - file_mode: 755 - exclude_pat: - "*_test.py" @@ -124,8 +124,8 @@ manager_sbin_jinja: file.recurse: - name: /usr/sbin/ - source: salt://manager/tools/sbin_jinja/ - - user: socore - - group: socore + - user: root + - group: root - file_mode: 755 - template: jinja - show_changes: False diff --git a/salt/nginx/config.sls b/salt/nginx/config.sls index ead3d9986..9b0e72a68 100644 --- a/salt/nginx/config.sls +++ b/salt/nginx/config.sls @@ -57,8 +57,8 @@ nginx_sbin: file.recurse: - name: /usr/sbin - source: salt://nginx/tools/sbin - - user: 939 - - group: 939 + - user: root + - group: root - file_mode: 755 #nginx_sbin_jinja: diff --git a/salt/redis/config.sls b/salt/redis/config.sls index 11aaa1f0e..13ac7e056 100644 --- a/salt/redis/config.sls +++ b/salt/redis/config.sls @@ -50,16 +50,16 @@ redis_sbin: file.recurse: - name: /usr/sbin - source: salt://redis/tools/sbin - - user: 939 - - group: 939 + - user: root + - group: root - file_mode: 755 redis_sbin_jinja: file.recurse: - name: /usr/sbin - source: salt://redis/tools/sbin_jinja - - user: 939 - - group: 939 + - user: root + - group: root - file_mode: 755 - template: jinja diff --git a/salt/salt/init.sls b/salt/salt/init.sls index d39c984eb..64a0a8794 100644 --- a/salt/salt/init.sls +++ b/salt/salt/init.sls @@ -3,6 +3,8 @@ salt_bootstrap: file.managed: - name: /usr/sbin/bootstrap-salt.sh - source: salt://salt/scripts/bootstrap-salt.sh + - user: root + - group: root - mode: 755 - show_changes: False @@ -10,6 +12,6 @@ salt_sbin: file.recurse: - name: /usr/sbin - source: salt://salt/tools/sbin - - user: 939 - - group: 939 + - user: root + - group: root - file_mode: 755 diff --git a/salt/sensor/init.sls b/salt/sensor/init.sls index 7d1714c2c..0b302e186 100644 --- a/salt/sensor/init.sls +++ b/salt/sensor/init.sls @@ -35,6 +35,8 @@ combine_bond_script: file.managed: - name: /usr/sbin/so-combine-bond - source: salt://sensor/tools/sbin_jinja/so-combine-bond + - user: root + - group: root - mode: 755 - template: jinja - defaults: diff --git a/salt/sensoroni/config.sls b/salt/sensoroni/config.sls index 225d0ddb4..75c92b971 100644 --- a/salt/sensoroni/config.sls +++ b/salt/sensoroni/config.sls @@ -64,8 +64,8 @@ sensoroni_sbin: file.recurse: - name: /usr/sbin - source: salt://sensoroni/tools/sbin - - user: 939 - - group: 939 + - user: root + - group: root - file_mode: 755 #sensoroni_sbin_jinja: diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 3609b7024..677c63e5d 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -171,8 +171,8 @@ soc_sbin: file.recurse: - name: /usr/sbin - source: salt://soc/tools/sbin - - user: 939 - - group: 939 + - user: root + - group: root - file_mode: 755 #soc_sbin_jinja: diff --git a/salt/strelka/config.sls b/salt/strelka/config.sls index f03afa61b..2c5fe741d 100644 --- a/salt/strelka/config.sls +++ b/salt/strelka/config.sls @@ -51,8 +51,8 @@ strelka_sbin: file.recurse: - name: /usr/sbin - source: salt://strelka/tools/sbin - - user: 939 - - group: 939 + - user: root + - group: root - file_mode: 755 {% else %} diff --git a/salt/suricata/config.sls b/salt/suricata/config.sls index da75580e3..dd228ef31 100644 --- a/salt/suricata/config.sls +++ b/salt/suricata/config.sls @@ -76,16 +76,16 @@ suricata_sbin: file.recurse: - name: /usr/sbin - source: salt://suricata/tools/sbin - - user: 939 - - group: 939 + - user: root + - group: root - file_mode: 755 suricata_sbin_jinja: file.recurse: - name: /usr/sbin - source: salt://suricata/tools/sbin_jinja - - user: 939 - - group: 939 + - user: root + - group: root - file_mode: 755 - template: jinja diff --git a/salt/telegraf/config.sls b/salt/telegraf/config.sls index 3873d1da0..18ac51ddd 100644 --- a/salt/telegraf/config.sls +++ b/salt/telegraf/config.sls @@ -65,8 +65,8 @@ telegraf_sbin: file.recurse: - name: /usr/sbin - source: salt://telegraf/tools/sbin - - user: 939 - - group: 939 + - user: root + - group: root - file_mode: 755 #telegraf_sbin_jinja: diff --git a/salt/zeek/config.sls b/salt/zeek/config.sls index 3677b1ba1..17a495010 100644 --- a/salt/zeek/config.sls +++ b/salt/zeek/config.sls @@ -101,8 +101,8 @@ zeek_sbin: file.recurse: - name: /usr/sbin - source: salt://zeek/tools/sbin - - user: 939 - - group: 939 + - user: root + - group: root - file_mode: 755 #zeek_sbin_jinja: From 1f1d3ded416f84c92eee34e20f99152fcd2c8a2c Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 16 Sep 2026 10:34:21 -0400 Subject: [PATCH 15/18] FIX: root-own the Salt default tree so a SOC file-write cannot reach root code /opt/so/saltstack/default holds the source for every root-executed script -- /usr/sbin, the reactors, _runners/_modules/_beacons, the master engines and salt-relay.sh -- plus every state the root master renders. SOC mounts /opt/so/saltstack rw as uid 939, so root-owning /usr/sbin alone was not enough: the next highstate would copy attacker-controlled bytes out of the tree into the root-owned destination and run them. SOC never writes under default/, it only reads it. Every SOC write targets local/, which stays socore-owned, as does /opt/so/state. No mode is enforced on default/ -- SOC reads that tree, and 750/640 would break its config load. Also stops copy_new_files(), so-saltstack-update and setup from chowning the tree back to socore, and replaces preserve: True in soup_scripts.sls, which carried uid/gid in from the /tmp staging tree and would have undone the ownership before the first post-soup highstate. --- salt/common/soup_scripts.sls | 56 +++++++++++++++------ salt/common/tools/sbin/so-common | 3 +- salt/manager/init.sls | 10 ++-- salt/manager/tools/sbin/so-saltstack-update | 4 +- salt/soc/dyanno/hypervisor/init.sls | 4 +- setup/so-functions | 2 + 6 files changed, 57 insertions(+), 22 deletions(-) diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls index 6b2f1551d..50be19bb4 100644 --- a/salt/common/soup_scripts.sls +++ b/salt/common/soup_scripts.sls @@ -18,47 +18,61 @@ copy_so-common_common_tools_sbin: - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-common - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-common - force: True - - preserve: True + - user: root + - group: root + - mode: 755 copy_so-image-common_common_tools_sbin: file.copy: - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-image-common - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-image-common - force: True - - preserve: True + - user: root + - group: root + - mode: 755 copy_soup_manager_tools_sbin: file.copy: - name: /opt/so/saltstack/default/salt/manager/tools/sbin/soup - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/soup - force: True - - preserve: True + - user: root + - group: root + - mode: 755 copy_so-firewall_manager_tools_sbin: file.copy: - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-firewall - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall - force: True - - preserve: True + - user: root + - group: root + - mode: 755 copy_so-yaml_manager_tools_sbin: file.copy: - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-yaml.py - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-yaml.py - force: True - - preserve: True + - user: root + - group: root + - mode: 755 copy_so-repo-sync_manager_tools_sbin: file.copy: - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-repo-sync - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync - - preserve: True + - user: root + - group: root + - mode: 755 copy_bootstrap-salt_manager_tools_sbin: file.copy: - name: /opt/so/saltstack/default/salt/salt/scripts/bootstrap-salt.sh - source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh - - preserve: True + - user: root + - group: root + - mode: 644 # This section is used to put the new script in place so that it can be called during soup. # It is faster than calling the states that normally manage them to put them in place. @@ -67,46 +81,60 @@ copy_so-common_sbin: - name: /usr/sbin/so-common - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-common - force: True - - preserve: True + - user: root + - group: root + - mode: 755 copy_so-image-common_sbin: file.copy: - name: /usr/sbin/so-image-common - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-image-common - force: True - - preserve: True + - user: root + - group: root + - mode: 755 copy_soup_sbin: file.copy: - name: /usr/sbin/soup - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/soup - force: True - - preserve: True + - user: root + - group: root + - mode: 755 copy_so-firewall_sbin: file.copy: - name: /usr/sbin/so-firewall - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall - force: True - - preserve: True + - user: root + - group: root + - mode: 755 copy_so-yaml_sbin: file.copy: - name: /usr/sbin/so-yaml.py - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-yaml.py - force: True - - preserve: True + - user: root + - group: root + - mode: 755 copy_so-repo-sync_sbin: file.copy: - name: /usr/sbin/so-repo-sync - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync - force: True - - preserve: True + - user: root + - group: root + - mode: 755 copy_bootstrap-salt_sbin: file.copy: - name: /usr/sbin/bootstrap-salt.sh - source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh - force: True - - preserve: True + - user: root + - group: root + - mode: 755 diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index fbcd37bb9..c8ebae084 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -240,7 +240,8 @@ copy_new_files() { cd $UPDATE_DIR rsync -a salt $DEFAULT_SALT_DIR/ --delete "${EXCLUDE_ARGS[@]}" rsync -a pillar $DEFAULT_SALT_DIR/ --delete "${EXCLUDE_ARGS[@]}" - chown -R socore:socore $DEFAULT_SALT_DIR/ + # Root-executed code; SOC only needs to read it. Local dirs stay socore-owned. + chown -R root:root $DEFAULT_SALT_DIR/ cd /tmp } diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 85df680d4..87669fad4 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -190,11 +190,15 @@ so_fleetagent_monitor: - month: '*' - dayweek: '*' -socore_own_saltstack_default: +# This tree is the source of every root-executed script (/usr/sbin, reactors, _runners, +# engines, salt-relay.sh). SOC mounts /opt/so/saltstack rw as uid 939 but only writes +# under local/. Do not add dir_mode/file_mode here -- SOC reads default/ and 750/640 +# would break its config load. +root_own_saltstack_default: file.directory: - name: /opt/so/saltstack/default - - user: socore - - group: socore + - user: root + - group: root - recurse: - user - group diff --git a/salt/manager/tools/sbin/so-saltstack-update b/salt/manager/tools/sbin/so-saltstack-update index c8723beb0..4ff4b0519 100755 --- a/salt/manager/tools/sbin/so-saltstack-update +++ b/salt/manager/tools/sbin/so-saltstack-update @@ -124,8 +124,8 @@ copy_new_files() { rsync -a salt $default_salt_dir/ rsync -a pillar $default_salt_dir/ - chown -R socore:socore $default_salt_dir/salt - chown -R socore:socore $default_salt_dir/pillar + chown -R root:root $default_salt_dir/salt + chown -R root:root $default_salt_dir/pillar chmod 755 $default_salt_dir/pillar/firewall/addfirewall.sh rm -rf /tmp/sogh diff --git a/salt/soc/dyanno/hypervisor/init.sls b/salt/soc/dyanno/hypervisor/init.sls index 1f17552e7..41b7a2ebf 100644 --- a/salt/soc/dyanno/hypervisor/init.sls +++ b/salt/soc/dyanno/hypervisor/init.sls @@ -18,8 +18,8 @@ hypervisor_annotation: - name: /opt/so/saltstack/default/salt/hypervisor/soc_hypervisor.yaml - source: salt://soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja - template: jinja - - user: socore - - group: socore + - user: root + - group: root - defaults: HYPERVISORS: {{ HYPERVISORS }} baseDomainStatus: {{ salt['pillar.get']('baseDomain:status', 'Initialized') }} diff --git a/setup/so-functions b/setup/so-functions index 1269b4e9d..64e809be5 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2105,6 +2105,8 @@ setup_salt_master_dirs() { info "Chown the salt dirs on the manager for socore" logCmd "chown -R socore:socore /opt/so" + # The default tree is root-executed code; SOC reads it but never writes it. + logCmd "chown -R root:root $default_salt_dir" } set_progress_str() { From 1fc5bb7afab5454208c4a93e8e836ec5d64c804d Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Thu, 17 Sep 2026 07:56:42 -0400 Subject: [PATCH 16/18] Refactor for ESQL --- salt/soc/defaults.yaml | 1 + salt/soc/files/soc/sigma_so_pipeline.yaml | 25 +++++++++++++++++++++++ salt/soc/soc_soc.yaml | 5 +++++ 3 files changed, 31 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index c6e833d32..f51a938bc 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1465,6 +1465,7 @@ soc: - core - emerging_threats_addon useEsql: false + caseInsensitive: true elastic: hostUrl: remoteHostUrls: [] diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index 64b691d8f..ae867be49 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -1,6 +1,31 @@ name: Security Onion Baseline Pipeline priority: 90 transformations: + # ES|QL scalar == returns null on multivalued fields; the + # backend reads this key and emits MV_INTERSECTS instead. + - id: declare_multivalue_fields + type: set_state + key: multivalue_fields + val: + - event.type + - event.action + - event.category + - tags + - process.args + - related.ip + - dns.resolved_ip + - id: esql_default_index + type: set_state + key: index + val: .ds-logs-* + - id: esql_source_metadata + type: set_state + key: metadata + val: "_id, _index, _source" + - id: esql_source_keep + type: set_state + key: keep + val: "_id, _index, _source" - id: baseline_field_name_mapping type: field_name_mapping mapping: diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index b329b736b..e49ad6ce7 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -396,6 +396,11 @@ soc: global: True advanced: True forcedType: bool + caseInsensitive: + description: "Match string values case-insensitively. Applies to ES|QL only." + global: True + advanced: True + forcedType: bool elastic: index: description: Comma-separated list of indices or index patterns (wildcard "*" supported) that SOC will search for records. From b3567405f9192375d1bcf85e4fa334d8b526fd3d Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 17 Sep 2026 07:57:44 -0400 Subject: [PATCH 17/18] FIX: suricata.fileinfo maps boolean gaps into long file.bytes.missing --- salt/elasticsearch/files/ingest/suricata.fileinfo | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/suricata.fileinfo b/salt/elasticsearch/files/ingest/suricata.fileinfo index 4f6182139..d59aa2401 100644 --- a/salt/elasticsearch/files/ingest/suricata.fileinfo +++ b/salt/elasticsearch/files/ingest/suricata.fileinfo @@ -5,7 +5,8 @@ { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } }, { "rename": { "field": "message2.app_proto", "target_field": "network.protocol", "ignore_missing": true } }, { "rename": { "field": "message2.fileinfo.filename", "target_field": "file.name", "ignore_missing": true } }, - { "rename": { "field": "message2.fileinfo.gaps", "target_field": "file.bytes.missing", "ignore_missing": true } }, + { "rename": { "field": "message2.fileinfo.gaps", "target_field": "suricata.fileinfo.gaps", "ignore_missing": true } }, + { "set": { "if": "ctx.suricata?.fileinfo?.gaps == false", "field": "file.bytes.missing", "value": 0 } }, { "rename": { "field": "message2.fileinfo.magic", "target_field": "file.mime_type", "ignore_missing": true } }, { "rename": { "field": "message2.fileinfo.md5", "target_field": "hash.md5", "ignore_missing": true } }, { "rename": { "field": "message2.fileinfo.sha1", "target_field": "hash.sha1", "ignore_missing": true } }, From f3aa39c5a4a485838bf51dcc2b3c0475b54872e2 Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Thu, 17 Sep 2026 09:43:44 -0400 Subject: [PATCH 18/18] Tweak name --- salt/soc/defaults.yaml | 2 +- salt/soc/soc_soc.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index f51a938bc..5035ec39e 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1465,7 +1465,7 @@ soc: - core - emerging_threats_addon useEsql: false - caseInsensitive: true + esqlCaseInsensitive: true elastic: hostUrl: remoteHostUrls: [] diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index e49ad6ce7..eed72f04c 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -396,8 +396,8 @@ soc: global: True advanced: True forcedType: bool - caseInsensitive: - description: "Match string values case-insensitively. Applies to ES|QL only." + esqlCaseInsensitive: + description: "Match string values case-insensitively when converting Sigma rules. Applies to ES|QL only" global: True advanced: True forcedType: bool