FIX: prevent so-sensor-clean runaway loop and concurrent instances

The cleanup loop had no progress check or pass limit, so once /nsm was over
threshold with nothing left to reclaim it spun at full speed, writing 1.1 GB /
12.5M lines to sensor_clean.log in five hours. Stop when a pass removes
nothing, when a pass frees no space, or at MAX_PASSES, and drop the per-pass
"no old files" logging in favor of one actionable line.

Replace the pgrep guard with flock -n. A find|while read subshell inherits the
parent's argv, so pgrep -cf counted one instance as hundreds; it was also
check-then-act, which let cron stack up overlapping runs.

Paths now derive from SENSOR_DIR with env-overridable LOG/LOCK so the
over-threshold path can be tested against a scratch filesystem.

.github/DISCUSSION_TEMPLATE/3-0.yml

@@ -13,6 +13,7 @@ body:
         - 3.1.0
         - 3.2.0
         - 3.3.0
+        - 3.4.0
         - Other (please provide detail below)
     validations:
       required: true

HOTFIX

@@ -1 +1 @@
-3.3.0-20260911
+

VERSION

@@ -1 +1 @@
-3.3.0
+3.4.0

salt/common/tools/sbin/so-sensor-clean

@@ -8,21 +8,37 @@
 # Elastic License 2.0.
 
 
-SENSOR_DIR='/nsm'
+SENSOR_DIR="${SENSOR_DIR:-/nsm}"
 CRIT_DISK_USAGE=90
-CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
-LOG="/opt/so/log/sensor_clean.log"
-TODAY=$(date -u "+%Y-%m-%d")
+LOG="${LOG:-/opt/so/log/sensor_clean.log}"
+LOCK="${LOCK:-/var/tmp/so-sensor-clean.lock}"
+MAX_PASSES=100
 
+ZEEK_LOGS="$SENSOR_DIR/zeek/logs"
+STRELKA_FILES="$SENSOR_DIR/strelka/processed"
+SURICATA_LOGS="$SENSOR_DIR/suricata"
+PCAPS="$SENSOR_DIR/pcapout"
+
+log() {
+	echo "$(date) - $*" >>"$LOG"
+}
+
+disk_usage() {
+	df -P "$SENSOR_DIR" | tail -1 | awk '{print $5}' | tr -d %
+}
+
+disk_avail() {
+	df -P "$SENSOR_DIR" | tail -1 | awk '{print $4}'
+}
+
+# sets REMOVED=1 if anything was actually deleted
 clean() {
 	## find the oldest Zeek logs directory
-	OLDEST_DIR=$(ls /nsm/zeek/logs/ | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
-	if [ -z "$OLDEST_DIR" -o "$OLDEST_DIR" == ".." -o "$OLDEST_DIR" == "." ]; then
-		echo "$(date) - No old Zeek logs available to clean up in /nsm/zeek/logs/" >>$LOG
-		#exit 0
-	else
-		echo "$(date) - Removing directory: /nsm/zeek/logs/$OLDEST_DIR" >>$LOG
-		rm -rf /nsm/zeek/logs/"$OLDEST_DIR"
+	OLDEST_DIR=$(ls "$ZEEK_LOGS" 2>/dev/null | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
+	if [ -n "$OLDEST_DIR" ]; then
+		log "Removing directory: $ZEEK_LOGS/$OLDEST_DIR"
+		rm -rf "$ZEEK_LOGS/$OLDEST_DIR"
+		REMOVED=1
 	fi
 
 	## Remarking for now, as we are moving extracted files to /nsm/strelka/processed
@@ -43,58 +59,73 @@ clean() {
 	#fi
 
 	## Clean up Zeek extracted files processed by Strelka
-	STRELKA_FILES='/nsm/strelka/processed'
-	OLDEST_STRELKA=$(find $STRELKA_FILES -type f -printf '%T+ %p\n' | sort -n | head -n 1)
-	if [ -z "$OLDEST_STRELKA" -o "$OLDEST_STRELKA" == ".." -o "$OLDEST_STRELKA" == "." ]; then
-		echo "$(date) - No old files available to clean up in $STRELKA_FILES" >>$LOG
-	else
+	OLDEST_STRELKA=$(find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
+	if [ -n "$OLDEST_STRELKA" ]; then
 		OLDEST_STRELKA_DATE=$(echo $OLDEST_STRELKA | awk '{print $1}' | cut -d+ -f1)
-		OLDEST_STRELKA_FILE=$(echo $OLDEST_STRELKA | awk '{print $2}')
-		echo "$(date) - Removing extracted files for $OLDEST_STRELKA_DATE" >>$LOG
-		find $STRELKA_FILES -type f -printf '%T+ %p\n' | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
-			echo "$(date) - Removing file: $FILE" >>$LOG
+		log "Removing extracted files for $OLDEST_STRELKA_DATE"
+		REMOVED=1
+		find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
+			log "Removing file: $FILE"
 			rm -f "$FILE"
 		done
 	fi
 
 	## Clean up Suricata log files
-	SURICATA_LOGS='/nsm/suricata'
-	OLDEST_SURICATA=$(find $SURICATA_LOGS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
-	if [[ -z "$OLDEST_SURICATA" ]] || [[ "$OLDEST_SURICATA" == ".." ]] || [[ "$OLDEST_SURICATA" == "." ]]; then
-		echo "$(date) - No old files available to clean up in $SURICATA_LOGS" >>$LOG
-	else
+	OLDEST_SURICATA=$(find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
+	if [ -n "$OLDEST_SURICATA" ]; then
 		OLDEST_SURICATA_DATE=$(echo $OLDEST_SURICATA | awk '{print $1}' | cut -d+ -f1)
-		OLDEST_SURICATA_FILE=$(echo $OLDEST_SURICATA | awk '{print $2}')
-		echo "$(date) - Removing logs for $OLDEST_SURICATA_DATE" >>$LOG
-		find $SURICATA_LOGS -type f -printf '%T+ %p\n' | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
-			echo "$(date) - Removing file: $FILE" >>$LOG
+		log "Removing logs for $OLDEST_SURICATA_DATE"
+		REMOVED=1
+		find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
+			log "Removing file: $FILE"
 			rm -f "$FILE"
 		done
 	fi
 
 	## Clean up extracted pcaps
-	PCAPS='/nsm/pcapout'
-	OLDEST_PCAP=$(find $PCAPS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
-	if [ -z "$OLDEST_PCAP" -o "$OLDEST_PCAP" == ".." -o "$OLDEST_PCAP" == "." ]; then
-		echo "$(date) - No old files available to clean up in $PCAPS" >>$LOG
-	else
+	OLDEST_PCAP=$(find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
+	if [ -n "$OLDEST_PCAP" ]; then
 		OLDEST_PCAP_DATE=$(echo $OLDEST_PCAP | awk '{print $1}' | cut -d+ -f1)
-		OLDEST_PCAP_FILE=$(echo $OLDEST_PCAP | awk '{print $2}')
-		echo "$(date) - Removing extracted files for $OLDEST_PCAP_DATE" >>$LOG
-		find $PCAPS -type f -printf '%T+ %p\n' | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
-			echo "$(date) - Removing file: $FILE" >>$LOG
+		log "Removing extracted files for $OLDEST_PCAP_DATE"
+		REMOVED=1
+		find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
+			log "Removing file: $FILE"
 			rm -f "$FILE"
 		done
 	fi
 }
 
-# Check to see if we are already running
-NUM_RUNNING=$(pgrep -cf "/bin/bash /usr/sbin/so-sensor-clean")
-[ "$NUM_RUNNING" -gt 1 ] && echo "$(date) - $NUM_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0
-
-if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then
-	while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
-		clean
-		CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
-	done
+# Only one instance at a time; the lock is the fd, so it releases on any exit
+exec 9>"$LOCK" || exit 1
+if ! flock -n 9; then
+	log "another so-sensor-clean is already running (lock $LOCK held); exiting"
+	exit 0
 fi
+
+CUR_USAGE=$(disk_usage)
+[ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ] || exit 0
+
+log "$SENSOR_DIR at ${CUR_USAGE}% (threshold ${CRIT_DISK_USAGE}%); starting cleanup"
+
+PASS=0
+while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
+	PASS=$((PASS + 1))
+	if [ "$PASS" -gt "$MAX_PASSES" ]; then
+		log "stopping after $MAX_PASSES passes; $SENSOR_DIR still at ${CUR_USAGE}%"
+		break
+	fi
+
+	REMOVED=0
+	BEFORE=$(disk_avail)
+	clean
+	CUR_USAGE=$(disk_usage)
+
+	if [ "$REMOVED" -eq 0 ]; then
+		log "nothing left to remove in $ZEEK_LOGS, $STRELKA_FILES, $SURICATA_LOGS, $PCAPS; $SENSOR_DIR still at ${CUR_USAGE}% - space is consumed outside of NSM cleanup scope"
+		break
+	fi
+	if [ "$(disk_avail)" -le "$BEFORE" ]; then
+		log "pass $PASS freed no space; $SENSOR_DIR still at ${CUR_USAGE}% - stopping until next run"
+		break
+	fi
+done

salt/nginx/etc/nginx.conf

@@ -183,7 +183,7 @@ http {
 		ssl_prefer_server_ciphers on;
 		ssl_protocols TLSv1.2 TLSv1.3;
 
-		location ~* (^/login/.*|^/js/.*|^/css/.*|^/images/.*|^/pages/.*|^/docs/.*) {
+		location ~* (^/login|^/login/.*|^/js/.*|^/css/.*|^/images/.*|^/pages/.*|^/docs/.*) {
 			proxy_pass            http://{{ GLOBALS.manager }}:9822;
 			proxy_read_timeout    90;
 			proxy_connect_timeout 90;

salt/soc/defaults.yaml

@@ -1537,7 +1537,7 @@ soc:
             Orchestrator: sonnet@SOAI
             Investigator: gemma@SOAI
             DetectionEngineer: gemma@SOAI
-          useMemory: false
+          useMemory: true
           useMemoryScanner: false
           dontScanBefore: ""
           memoryScanIntervalSeconds: 300
@@ -1556,6 +1556,69 @@ soc:
           reconcilePersona: ""
           toolUseTurnAttempts: 12
           toolUseTurnDelayMs: 175
+          tools:
+            filterEventFields:
+              - "@timestamp"
+              - "client.name"
+              - "destination.ip"
+              - "destination.port"
+              - "destination.geo.country_name"
+              - "dns.query.name"
+              - "dns.query_name"
+              - "event.action"
+              - "event.category"
+              - "event.module"
+              - "event.dataset"
+              - "event.outcome"
+              - "event.severity"
+              - "event.severity_label"
+              - "event.type"
+              - "event_data.agent.name"
+              - "event_data.host.os.name"
+              - "file.mime_type"
+              - "file.name"
+              - "hash.md5"
+              - "hash.sha1"
+              - "host.mac"
+              - "host.name"
+              - "host.os.name"
+              - "http.method"
+              - "http.useragent"
+              - "http.virtual_host"
+              - "log.id.uid"
+              - "network.community_id"
+              - "network.protocol"
+              - "network.transport"
+              - "notice.message"
+              - "observer.name"
+              - "process.name"
+              - "process.executable"
+              - "process.entity_id"
+              - "process.command_line"
+              - "process.Ext.ancestry"
+              - "process.parent.entity_id"
+              - "process.parent.command_line"
+              - "rule.category"
+              - "rule.name"
+              - "rule.uuid"
+              - "software.name"
+              - "software.type"
+              - "software.version.unparsed"
+              - "source.ip"
+              - "source.port"
+              - "source.geo.country_name"
+              - "ssh.cypher_algorithm"
+              - "ssh.client"
+              - "ssh.server"
+              - "ssl.cipher"
+              - "ssl.server_name"
+              - "ssl.version"
+              - "system.auth.sudo.command"
+              - "user.name"
+              - "user.domain"
+              - "user.effective.name"
+              - "weird.name"
+              - "tags"
         onionconfig:
           saltstackDir: /opt/so/saltstack
           bypassEnabled: false

salt/soc/soc_soc.yaml

@@ -916,6 +916,11 @@ soc:
             description: The number of times to retry extracting memories from a session if errors occur.
             global: True
             advanced: True
+          tools:
+            filterEventFields:
+              description: A whitelist of fields to return when OnionAI uses the query_events tool. All other fields are removed. One field per line.
+              global: True
+              multiline: True
       client:
         assistant:
           enabled: