FIX: validate MAINIP without so-common during setup

setup runs so-minion -o=setup before /usr/sbin/so-common is installed, so
valid_ip4 was undefined, every MAINIP was rejected, and no minion pillar
was written. Pillar compile then failed for the new manager and setup gave
up waiting for the salt master. Use an inline IPv4 regex instead.

.github/DISCUSSION_TEMPLATE/3-0.yml

@@ -13,6 +13,7 @@ body:
         - 3.1.0
         - 3.2.0
         - 3.3.0
+        - 3.4.0
         - Other (please provide detail below)
     validations:
       required: true

HOTFIX

@@ -1 +1 @@
-20260911
+

VERSION

@@ -1 +1 @@
-3.3.0
+3.4.0

salt/common/tools/sbin/so-sensor-clean

@@ -8,21 +8,37 @@
 # Elastic License 2.0.
 
 
-SENSOR_DIR='/nsm'
+SENSOR_DIR="${SENSOR_DIR:-/nsm}"
 CRIT_DISK_USAGE=90
-CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
-LOG="/opt/so/log/sensor_clean.log"
-TODAY=$(date -u "+%Y-%m-%d")
+LOG="${LOG:-/opt/so/log/sensor_clean.log}"
+LOCK="${LOCK:-/var/tmp/so-sensor-clean.lock}"
+MAX_PASSES=100
 
+ZEEK_LOGS="$SENSOR_DIR/zeek/logs"
+STRELKA_FILES="$SENSOR_DIR/strelka/processed"
+SURICATA_LOGS="$SENSOR_DIR/suricata"
+PCAPS="$SENSOR_DIR/pcapout"
+
+log() {
+	echo "$(date) - $*" >>"$LOG"
+}
+
+disk_usage() {
+	df -P "$SENSOR_DIR" | tail -1 | awk '{print $5}' | tr -d %
+}
+
+disk_avail() {
+	df -P "$SENSOR_DIR" | tail -1 | awk '{print $4}'
+}
+
+# sets REMOVED=1 if anything was actually deleted
 clean() {
 	## find the oldest Zeek logs directory
-	OLDEST_DIR=$(ls /nsm/zeek/logs/ | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
-	if [ -z "$OLDEST_DIR" -o "$OLDEST_DIR" == ".." -o "$OLDEST_DIR" == "." ]; then
-		echo "$(date) - No old Zeek logs available to clean up in /nsm/zeek/logs/" >>$LOG
-		#exit 0
-	else
-		echo "$(date) - Removing directory: /nsm/zeek/logs/$OLDEST_DIR" >>$LOG
-		rm -rf /nsm/zeek/logs/"$OLDEST_DIR"
+	OLDEST_DIR=$(ls "$ZEEK_LOGS" 2>/dev/null | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
+	if [ -n "$OLDEST_DIR" ]; then
+		log "Removing directory: $ZEEK_LOGS/$OLDEST_DIR"
+		rm -rf "$ZEEK_LOGS/$OLDEST_DIR"
+		REMOVED=1
 	fi
 
 	## Remarking for now, as we are moving extracted files to /nsm/strelka/processed
@@ -43,58 +59,73 @@ clean() {
 	#fi
 
 	## Clean up Zeek extracted files processed by Strelka
-	STRELKA_FILES='/nsm/strelka/processed'
-	OLDEST_STRELKA=$(find $STRELKA_FILES -type f -printf '%T+ %p\n' | sort -n | head -n 1)
-	if [ -z "$OLDEST_STRELKA" -o "$OLDEST_STRELKA" == ".." -o "$OLDEST_STRELKA" == "." ]; then
-		echo "$(date) - No old files available to clean up in $STRELKA_FILES" >>$LOG
-	else
+	OLDEST_STRELKA=$(find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
+	if [ -n "$OLDEST_STRELKA" ]; then
 		OLDEST_STRELKA_DATE=$(echo $OLDEST_STRELKA | awk '{print $1}' | cut -d+ -f1)
-		OLDEST_STRELKA_FILE=$(echo $OLDEST_STRELKA | awk '{print $2}')
-		echo "$(date) - Removing extracted files for $OLDEST_STRELKA_DATE" >>$LOG
-		find $STRELKA_FILES -type f -printf '%T+ %p\n' | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
-			echo "$(date) - Removing file: $FILE" >>$LOG
+		log "Removing extracted files for $OLDEST_STRELKA_DATE"
+		REMOVED=1
+		find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
+			log "Removing file: $FILE"
 			rm -f "$FILE"
 		done
 	fi
 
 	## Clean up Suricata log files
-	SURICATA_LOGS='/nsm/suricata'
-	OLDEST_SURICATA=$(find $SURICATA_LOGS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
-	if [[ -z "$OLDEST_SURICATA" ]] || [[ "$OLDEST_SURICATA" == ".." ]] || [[ "$OLDEST_SURICATA" == "." ]]; then
-		echo "$(date) - No old files available to clean up in $SURICATA_LOGS" >>$LOG
-	else
+	OLDEST_SURICATA=$(find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
+	if [ -n "$OLDEST_SURICATA" ]; then
 		OLDEST_SURICATA_DATE=$(echo $OLDEST_SURICATA | awk '{print $1}' | cut -d+ -f1)
-		OLDEST_SURICATA_FILE=$(echo $OLDEST_SURICATA | awk '{print $2}')
-		echo "$(date) - Removing logs for $OLDEST_SURICATA_DATE" >>$LOG
-		find $SURICATA_LOGS -type f -printf '%T+ %p\n' | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
-			echo "$(date) - Removing file: $FILE" >>$LOG
+		log "Removing logs for $OLDEST_SURICATA_DATE"
+		REMOVED=1
+		find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
+			log "Removing file: $FILE"
 			rm -f "$FILE"
 		done
 	fi
 
 	## Clean up extracted pcaps
-	PCAPS='/nsm/pcapout'
-	OLDEST_PCAP=$(find $PCAPS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
-	if [ -z "$OLDEST_PCAP" -o "$OLDEST_PCAP" == ".." -o "$OLDEST_PCAP" == "." ]; then
-		echo "$(date) - No old files available to clean up in $PCAPS" >>$LOG
-	else
+	OLDEST_PCAP=$(find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
+	if [ -n "$OLDEST_PCAP" ]; then
 		OLDEST_PCAP_DATE=$(echo $OLDEST_PCAP | awk '{print $1}' | cut -d+ -f1)
-		OLDEST_PCAP_FILE=$(echo $OLDEST_PCAP | awk '{print $2}')
-		echo "$(date) - Removing extracted files for $OLDEST_PCAP_DATE" >>$LOG
-		find $PCAPS -type f -printf '%T+ %p\n' | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
-			echo "$(date) - Removing file: $FILE" >>$LOG
+		log "Removing extracted files for $OLDEST_PCAP_DATE"
+		REMOVED=1
+		find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
+			log "Removing file: $FILE"
 			rm -f "$FILE"
 		done
 	fi
 }
 
-# Check to see if we are already running
-NUM_RUNNING=$(pgrep -cf "/bin/bash /usr/sbin/so-sensor-clean")
-[ "$NUM_RUNNING" -gt 1 ] && echo "$(date) - $NUM_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0
-
-if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then
-	while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
-		clean
-		CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
-	done
+# Only one instance at a time; the lock is the fd, so it releases on any exit
+exec 9>"$LOCK" || exit 1
+if ! flock -n 9; then
+	log "another so-sensor-clean is already running (lock $LOCK held); exiting"
+	exit 0
 fi
+
+CUR_USAGE=$(disk_usage)
+[ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ] || exit 0
+
+log "$SENSOR_DIR at ${CUR_USAGE}% (threshold ${CRIT_DISK_USAGE}%); starting cleanup"
+
+PASS=0
+while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
+	PASS=$((PASS + 1))
+	if [ "$PASS" -gt "$MAX_PASSES" ]; then
+		log "stopping after $MAX_PASSES passes; $SENSOR_DIR still at ${CUR_USAGE}%"
+		break
+	fi
+
+	REMOVED=0
+	BEFORE=$(disk_avail)
+	clean
+	CUR_USAGE=$(disk_usage)
+
+	if [ "$REMOVED" -eq 0 ]; then
+		log "nothing left to remove in $ZEEK_LOGS, $STRELKA_FILES, $SURICATA_LOGS, $PCAPS; $SENSOR_DIR still at ${CUR_USAGE}% - space is consumed outside of NSM cleanup scope"
+		break
+	fi
+	if [ "$(disk_avail)" -le "$BEFORE" ]; then
+		log "pass $PASS freed no space; $SENSOR_DIR still at ${CUR_USAGE}% - stopping until next run"
+		break
+	fi
+done

salt/manager/tools/sbin/so-minion

@@ -121,8 +121,14 @@ for i in "$@"; do
 	esac
 done
 
-PILLARFILE=/opt/so/saltstack/local/pillar/minions/$MINION_ID.sls
-ADVPILLARFILE=/opt/so/saltstack/local/pillar/minions/adv_$MINION_ID.sls
+if [[ -n "$MINION_ID" && ! "$MINION_ID" =~ ^[A-Za-z0-9._-]{1,253}$ ]]; then
+	echo "Invalid minion id: $MINION_ID"
+	log "ERROR" "Invalid minion id: $MINION_ID"
+	exit 1
+fi
+
+readonly PILLARFILE=/opt/so/saltstack/local/pillar/minions/$MINION_ID.sls
+readonly ADVPILLARFILE=/opt/so/saltstack/local/pillar/minions/adv_$MINION_ID.sls
 
 function getinstallinfo() {
 	log "INFO" "Getting install info for minion $MINION_ID"
@@ -133,10 +139,23 @@ function getinstallinfo() {
 		return 1
 	fi
 
-	while read -r var; do export "$var"; done <<< "$INSTALLVARS"
-	if [ $? -ne 0 ]; then
-		log "ERROR" "Failed to source install variables"
-		return 1
+	# install.txt is controlled by the minion; only accept known keys and never eval or export them
+	local line key
+	while IFS= read -r line; do
+		[[ "$line" == *=* ]] || continue
+		key=${line%%=*}
+		case "$key" in
+			MAINIP|MNIC|NODE_DESCRIPTION|ES_HEAP_SIZE|PATCHSCHEDULENAME|INTERFACE|NODETYPE|CORECOUNT|LSHOSTNAME|LSHEAP|CPUCORES|IDH_MGTRESTRICT|IDH_SERVICES)
+				printf -v "$key" '%s' "${line#*=}"
+				;;
+			*)
+				log "WARN" "Ignoring unexpected install var from $MINION_ID: ${key:0:64}"
+				;;
+		esac
+	done <<< "$INSTALLVARS"
+
+	if [[ "$NODE_DESCRIPTION" == \'*\' ]]; then
+		NODE_DESCRIPTION=${NODE_DESCRIPTION:1:-1}
 	fi
 
 	log "INFO" "Fetched install info for $MINION_ID (node type: ${NODETYPE:-unset})"
@@ -176,6 +195,12 @@ function pcapspace() {
 		fi
 	fi
 
+	# Must be checked before arithmetic expansion, which evaluates array subscripts
+	if [[ ! "$SPACESIZE" =~ ^[0-9]+$ ]]; then
+		log "ERROR" "Invalid disk size for $MINION_ID: ${SPACESIZE:0:64}"
+		return 1
+	fi
+
 	local s=$(( $SPACESIZE / 1000000 ))
 	local s1=$(( $s / 4 * $PCAP_PERCENTAGE ))
 
@@ -1050,6 +1075,57 @@ function updateMineAndApplyStates() {
 	fi
 }
 
+# Values end up in a Jinja-rendered pillar and in bash, and may come from the minion
+function validate_minion_vars() {
+	local error_msg=""
+	# Inline rather than valid_ip4: so-common is not installed yet when setup runs -o=setup
+	local octet='(25[0-5]|2[0-4][0-9]|1?[0-9]?[0-9])'
+	local ip4_re="^($octet\.){3}$octet$"
+
+	case "$NODETYPE" in
+		EVAL|STANDALONE|MANAGER|MANAGERSEARCH|MANAGERHYPE|IMPORT)
+			# Manager pillars also rewrite the CA pillar, so never accept them from a remote node
+			[[ "$OPERATION" == "setup" ]] || error_msg="Node type $NODETYPE can only be configured during setup"
+			;;
+		FLEET|IDH|HEAVYNODE|SENSOR|SEARCHNODE|RECEIVER|HYPERVISOR|DESKTOP)
+			;;
+		*)
+			error_msg="Invalid node type: ${NODETYPE:0:64}"
+			;;
+	esac
+
+	if [[ -z "$error_msg" ]]; then
+		if [[ ! "$MAINIP" =~ $ip4_re ]]; then
+			error_msg="Invalid MAINIP: ${MAINIP:0:64}"
+		elif [[ ! "$MNIC" =~ ^[A-Za-z0-9._-]*$ ]]; then
+			error_msg="Invalid MNIC: ${MNIC:0:64}"
+		elif [[ ! "$INTERFACE" =~ ^[A-Za-z0-9._-]*$ ]]; then
+			error_msg="Invalid INTERFACE: ${INTERFACE:0:64}"
+		elif [[ ! "$LSHOSTNAME" =~ ^[A-Za-z0-9._-]*$ ]]; then
+			error_msg="Invalid LSHOSTNAME: ${LSHOSTNAME:0:64}"
+		elif [[ ! "$ES_HEAP_SIZE" =~ ^([0-9]+[kKmMgG]?)?$ ]]; then
+			error_msg="Invalid ES_HEAP_SIZE: ${ES_HEAP_SIZE:0:64}"
+		elif [[ ! "$LSHEAP" =~ ^([0-9]+[kKmMgG]?)?$ ]]; then
+			error_msg="Invalid LSHEAP: ${LSHEAP:0:64}"
+		elif [[ ! "$CORECOUNT" =~ ^[0-9]*$ ]]; then
+			error_msg="Invalid CORECOUNT: ${CORECOUNT:0:64}"
+		elif [[ ! "$CPUCORES" =~ ^[0-9]*$ ]]; then
+			error_msg="Invalid CPUCORES: ${CPUCORES:0:64}"
+		elif [[ ! "$IDH_MGTRESTRICT" =~ ^(True|False)?$ ]]; then
+			error_msg="Invalid IDH_MGTRESTRICT: ${IDH_MGTRESTRICT:0:64}"
+		fi
+	fi
+
+	if [[ -n "$error_msg" ]]; then
+		log "ERROR" "$error_msg"
+		echo "$error_msg"
+		return 1
+	fi
+
+	# Free text; removing braces is enough to prevent any Jinja delimiter
+	NODE_DESCRIPTION=${NODE_DESCRIPTION//[\{\}[:cntrl:]]/}
+}
+
 function setupMinionFiles() {
 	log "INFO" "Setting up minion files for $MINION_ID (pillar: $PILLARFILE)"
 	
@@ -1061,6 +1137,8 @@ function setupMinionFiles() {
 		return 1
 	fi
 
+	validate_minion_vars || return 1
+
 	# Create the base minion files
 	create_minion_files || return 1
 

salt/nginx/etc/nginx.conf

@@ -183,7 +183,7 @@ http {
 		ssl_prefer_server_ciphers on;
 		ssl_protocols TLSv1.2 TLSv1.3;
 
-		location ~* (^/login/.*|^/js/.*|^/css/.*|^/images/.*|^/pages/.*|^/docs/.*) {
+		location ~* (^/login|^/login/.*|^/js/.*|^/css/.*|^/images/.*|^/pages/.*|^/docs/.*) {
 			proxy_pass            http://{{ GLOBALS.manager }}:9822;
 			proxy_read_timeout    90;
 			proxy_connect_timeout 90;

salt/soc/defaults.yaml

@@ -1556,6 +1556,69 @@ soc:
           reconcilePersona: ""
           toolUseTurnAttempts: 12
           toolUseTurnDelayMs: 175
+          tools:
+            filterEventFields:
+              - "@timestamp"
+              - "client.name"
+              - "destination.ip"
+              - "destination.port"
+              - "destination.geo.country_name"
+              - "dns.query.name"
+              - "dns.query_name"
+              - "event.action"
+              - "event.category"
+              - "event.module"
+              - "event.dataset"
+              - "event.outcome"
+              - "event.severity"
+              - "event.severity_label"
+              - "event.type"
+              - "event_data.agent.name"
+              - "event_data.host.os.name"
+              - "file.mime_type"
+              - "file.name"
+              - "hash.md5"
+              - "hash.sha1"
+              - "host.mac"
+              - "host.name"
+              - "host.os.name"
+              - "http.method"
+              - "http.useragent"
+              - "http.virtual_host"
+              - "log.id.uid"
+              - "network.community_id"
+              - "network.protocol"
+              - "network.transport"
+              - "notice.message"
+              - "observer.name"
+              - "process.name"
+              - "process.executable"
+              - "process.entity_id"
+              - "process.command_line"
+              - "process.Ext.ancestry"
+              - "process.parent.entity_id"
+              - "process.parent.command_line"
+              - "rule.category"
+              - "rule.name"
+              - "rule.uuid"
+              - "software.name"
+              - "software.type"
+              - "software.version.unparsed"
+              - "source.ip"
+              - "source.port"
+              - "source.geo.country_name"
+              - "ssh.cypher_algorithm"
+              - "ssh.client"
+              - "ssh.server"
+              - "ssl.cipher"
+              - "ssl.server_name"
+              - "ssl.version"
+              - "system.auth.sudo.command"
+              - "user.name"
+              - "user.domain"
+              - "user.effective.name"
+              - "weird.name"
+              - "tags"
         onionconfig:
           saltstackDir: /opt/so/saltstack
           bypassEnabled: false

salt/soc/soc_soc.yaml

@@ -916,6 +916,11 @@ soc:
             description: The number of times to retry extracting memories from a session if errors occur.
             global: True
             advanced: True
+          tools:
+            filterEventFields:
+              description: A whitelist of fields to return when OnionAI uses the query_events tool. All other fields are removed. One field per line.
+              global: True
+              multiline: True
       client:
         assistant:
           enabled:

salt/zeek/defaults.yaml

@@ -18,6 +18,7 @@ zeek:
       StatsLogEnable: 0
       StatsLogExpireInterval: 0
       StatusCmdShowAll: 0
+      StopWait: 1
       CrashExpireInterval: 0
       SitePolicyScripts: local.zeek
       LogDir: /nsm/zeek/logs

salt/zeek/disabled.sls

@@ -9,9 +9,19 @@
 include:
   - zeek.sostatus
   
+# Stop first so the entrypoint's SIGTERM trap can archive the final logs; docker_container.absent
+# with force is a 'docker rm -f', which never delivers SIGTERM. force stays so the state still
+# converges if the stop overruns.
+so-zeek_stopped:
+  docker_container.stopped:
+    - name: so-zeek
+    - error_on_absent: False
+
 so-zeek:
   docker_container.absent:
     - force: True
+    - require:
+      - docker_container: so-zeek_stopped
 
 so-zeek_so-status.disabled:
   file.comment:

salt/zeek/enabled.sls

@@ -19,6 +19,10 @@ so-zeek:
     - restart_policy: unless-stopped
     - start: True
     - privileged: True
+    # Docker's default 10s grace is not enough for the entrypoint's SIGTERM trap to run
+    # 'zeekctl stop' and let StopWait archive the final logs. Overrunning it means SIGKILL,
+    # which strands those logs in spool/tmp and marks every node crashed on the next start.
+    - stop_timeout: 180
     {% if DOCKERMERGED.containers['so-zeek'].ulimits %}
     - ulimits:
     {%   for ULIMIT in DOCKERMERGED.containers['so-zeek'].ulimits %}

salt/zeek/soc_zeek.yaml

@@ -99,6 +99,18 @@ zeek:
         regexFailureMessage: You must enter a whole number of days, or 0 to keep crash directories forever.
         helpLink: zeek
         advanced: True
+      StopWait:
+        description: >-
+          Set to 1 to make "zeekctl stop" wait for the final logs to be archived instead of
+          letting that finish in the background. Security Onion stops Zeek by stopping its
+          container, so anything still running in the background is killed when the container
+          exits - without this, the last logs of each run are stranded unarchived in
+          /nsm/zeek/spool/tmp and never reach Elasticsearch. It is read only for that reason.
+        regex: ^[01]$
+        regexFailureMessage: You must enter 0 or 1.
+        helpLink: zeek
+        advanced: True
+        readonly: True
       MinDiskSpace:
         description: >-
           Percentage of free disk space below which ZeekControl reports a warning, or 0 to disable the check