Update HOTFIX version to 3.3.0-20260911

Gives users a chance to reconfigure embed model before messages they send to the OnionAI get sent to SOAI by default.

.github/DISCUSSION_TEMPLATE/3-0.yml

@@ -13,7 +13,6 @@ body:
         - 3.1.0
         - 3.2.0
         - 3.3.0
-        - 3.4.0
         - Other (please provide detail below)
     validations:
       required: true

HOTFIX

@@ -1 +1 @@
-
+3.3.0-20260911

VERSION

@@ -1 +1 @@
-3.4.0
+3.3.0

salt/common/tools/sbin/so-sensor-clean

@@ -8,37 +8,21 @@
 # Elastic License 2.0.
 
 
-SENSOR_DIR="${SENSOR_DIR:-/nsm}"
+SENSOR_DIR='/nsm'
 CRIT_DISK_USAGE=90
-LOG="${LOG:-/opt/so/log/sensor_clean.log}"
+CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
-LOCK="${LOCK:-/var/tmp/so-sensor-clean.lock}"
+LOG="/opt/so/log/sensor_clean.log"
-MAX_PASSES=100
+TODAY=$(date -u "+%Y-%m-%d")
 
-ZEEK_LOGS="$SENSOR_DIR/zeek/logs"
-STRELKA_FILES="$SENSOR_DIR/strelka/processed"
-SURICATA_LOGS="$SENSOR_DIR/suricata"
-PCAPS="$SENSOR_DIR/pcapout"
-
-log() {
-	echo "$(date) - $*" >>"$LOG"
-}
-
-disk_usage() {
-	df -P "$SENSOR_DIR" | tail -1 | awk '{print $5}' | tr -d %
-}
-
-disk_avail() {
-	df -P "$SENSOR_DIR" | tail -1 | awk '{print $4}'
-}
-
-# sets REMOVED=1 if anything was actually deleted
 clean() {
 	## find the oldest Zeek logs directory
-	OLDEST_DIR=$(ls "$ZEEK_LOGS" 2>/dev/null | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
+	OLDEST_DIR=$(ls /nsm/zeek/logs/ | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
-	if [ -n "$OLDEST_DIR" ]; then
+	if [ -z "$OLDEST_DIR" -o "$OLDEST_DIR" == ".." -o "$OLDEST_DIR" == "." ]; then
-		log "Removing directory: $ZEEK_LOGS/$OLDEST_DIR"
+		echo "$(date) - No old Zeek logs available to clean up in /nsm/zeek/logs/" >>$LOG
-		rm -rf "$ZEEK_LOGS/$OLDEST_DIR"
+		#exit 0
-		REMOVED=1
+	else
+		echo "$(date) - Removing directory: /nsm/zeek/logs/$OLDEST_DIR" >>$LOG
+		rm -rf /nsm/zeek/logs/"$OLDEST_DIR"
 	fi
 
 	## Remarking for now, as we are moving extracted files to /nsm/strelka/processed
@@ -59,73 +43,58 @@ clean() {
 	#fi
 
 	## Clean up Zeek extracted files processed by Strelka
-	OLDEST_STRELKA=$(find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
+	STRELKA_FILES='/nsm/strelka/processed'
-	if [ -n "$OLDEST_STRELKA" ]; then
+	OLDEST_STRELKA=$(find $STRELKA_FILES -type f -printf '%T+ %p\n' | sort -n | head -n 1)
+	if [ -z "$OLDEST_STRELKA" -o "$OLDEST_STRELKA" == ".." -o "$OLDEST_STRELKA" == "." ]; then
+		echo "$(date) - No old files available to clean up in $STRELKA_FILES" >>$LOG
+	else
 		OLDEST_STRELKA_DATE=$(echo $OLDEST_STRELKA | awk '{print $1}' | cut -d+ -f1)
-		log "Removing extracted files for $OLDEST_STRELKA_DATE"
+		OLDEST_STRELKA_FILE=$(echo $OLDEST_STRELKA | awk '{print $2}')
-		REMOVED=1
+		echo "$(date) - Removing extracted files for $OLDEST_STRELKA_DATE" >>$LOG
-		find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
+		find $STRELKA_FILES -type f -printf '%T+ %p\n' | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
-			log "Removing file: $FILE"
+			echo "$(date) - Removing file: $FILE" >>$LOG
 			rm -f "$FILE"
 		done
 	fi
 
 	## Clean up Suricata log files
-	OLDEST_SURICATA=$(find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
+	SURICATA_LOGS='/nsm/suricata'
-	if [ -n "$OLDEST_SURICATA" ]; then
+	OLDEST_SURICATA=$(find $SURICATA_LOGS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
+	if [[ -z "$OLDEST_SURICATA" ]] || [[ "$OLDEST_SURICATA" == ".." ]] || [[ "$OLDEST_SURICATA" == "." ]]; then
+		echo "$(date) - No old files available to clean up in $SURICATA_LOGS" >>$LOG
+	else
 		OLDEST_SURICATA_DATE=$(echo $OLDEST_SURICATA | awk '{print $1}' | cut -d+ -f1)
-		log "Removing logs for $OLDEST_SURICATA_DATE"
+		OLDEST_SURICATA_FILE=$(echo $OLDEST_SURICATA | awk '{print $2}')
-		REMOVED=1
+		echo "$(date) - Removing logs for $OLDEST_SURICATA_DATE" >>$LOG
-		find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
+		find $SURICATA_LOGS -type f -printf '%T+ %p\n' | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
-			log "Removing file: $FILE"
+			echo "$(date) - Removing file: $FILE" >>$LOG
 			rm -f "$FILE"
 		done
 	fi
 
 	## Clean up extracted pcaps
-	OLDEST_PCAP=$(find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
+	PCAPS='/nsm/pcapout'
-	if [ -n "$OLDEST_PCAP" ]; then
+	OLDEST_PCAP=$(find $PCAPS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
+	if [ -z "$OLDEST_PCAP" -o "$OLDEST_PCAP" == ".." -o "$OLDEST_PCAP" == "." ]; then
+		echo "$(date) - No old files available to clean up in $PCAPS" >>$LOG
+	else
 		OLDEST_PCAP_DATE=$(echo $OLDEST_PCAP | awk '{print $1}' | cut -d+ -f1)
-		log "Removing extracted files for $OLDEST_PCAP_DATE"
+		OLDEST_PCAP_FILE=$(echo $OLDEST_PCAP | awk '{print $2}')
-		REMOVED=1
+		echo "$(date) - Removing extracted files for $OLDEST_PCAP_DATE" >>$LOG
-		find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
+		find $PCAPS -type f -printf '%T+ %p\n' | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
-			log "Removing file: $FILE"
+			echo "$(date) - Removing file: $FILE" >>$LOG
 			rm -f "$FILE"
 		done
 	fi
 }
 
-# Only one instance at a time; the lock is the fd, so it releases on any exit
+# Check to see if we are already running
-exec 9>"$LOCK" || exit 1
+NUM_RUNNING=$(pgrep -cf "/bin/bash /usr/sbin/so-sensor-clean")
-if ! flock -n 9; then
+[ "$NUM_RUNNING" -gt 1 ] && echo "$(date) - $NUM_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0
-	log "another so-sensor-clean is already running (lock $LOCK held); exiting"
+
-	exit 0
+if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then
+	while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
+		clean
+		CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
+	done
 fi
-
-CUR_USAGE=$(disk_usage)
-[ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ] || exit 0
-
-log "$SENSOR_DIR at ${CUR_USAGE}% (threshold ${CRIT_DISK_USAGE}%); starting cleanup"
-
-PASS=0
-while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
-	PASS=$((PASS + 1))
-	if [ "$PASS" -gt "$MAX_PASSES" ]; then
-		log "stopping after $MAX_PASSES passes; $SENSOR_DIR still at ${CUR_USAGE}%"
-		break
-	fi
-
-	REMOVED=0
-	BEFORE=$(disk_avail)
-	clean
-	CUR_USAGE=$(disk_usage)
-
-	if [ "$REMOVED" -eq 0 ]; then
-		log "nothing left to remove in $ZEEK_LOGS, $STRELKA_FILES, $SURICATA_LOGS, $PCAPS; $SENSOR_DIR still at ${CUR_USAGE}% - space is consumed outside of NSM cleanup scope"
-		break
-	fi
-	if [ "$(disk_avail)" -le "$BEFORE" ]; then
-		log "pass $PASS freed no space; $SENSOR_DIR still at ${CUR_USAGE}% - stopping until next run"
-		break
-	fi
-done

salt/nginx/etc/nginx.conf

@@ -183,7 +183,7 @@ http {
 		ssl_prefer_server_ciphers on;
 		ssl_protocols TLSv1.2 TLSv1.3;
 
-		location ~* (^/login|^/login/.*|^/js/.*|^/css/.*|^/images/.*|^/pages/.*|^/docs/.*) {
+		location ~* (^/login/.*|^/js/.*|^/css/.*|^/images/.*|^/pages/.*|^/docs/.*) {
 			proxy_pass            http://{{ GLOBALS.manager }}:9822;
 			proxy_read_timeout    90;
 			proxy_connect_timeout 90;

salt/soc/defaults.yaml

@@ -1537,7 +1537,7 @@ soc:
             Orchestrator: sonnet@SOAI
             Investigator: gemma@SOAI
             DetectionEngineer: gemma@SOAI
-          useMemory: true
+          useMemory: false
           useMemoryScanner: false
           dontScanBefore: ""
           memoryScanIntervalSeconds: 300
@@ -1556,69 +1556,6 @@ soc:
           reconcilePersona: ""
           toolUseTurnAttempts: 12
           toolUseTurnDelayMs: 175
-          tools:
-            filterEventFields:
-              - "@timestamp"
-              - "client.name"
-              - "destination.ip"
-              - "destination.port"
-              - "destination.geo.country_name"
-              - "dns.query.name"
-              - "dns.query_name"
-              - "event.action"
-              - "event.category"
-              - "event.module"
-              - "event.dataset"
-              - "event.outcome"
-              - "event.severity"
-              - "event.severity_label"
-              - "event.type"
-              - "event_data.agent.name"
-              - "event_data.host.os.name"
-              - "file.mime_type"
-              - "file.name"
-              - "hash.md5"
-              - "hash.sha1"
-              - "host.mac"
-              - "host.name"
-              - "host.os.name"
-              - "http.method"
-              - "http.useragent"
-              - "http.virtual_host"
-              - "log.id.uid"
-              - "network.community_id"
-              - "network.protocol"
-              - "network.transport"
-              - "notice.message"
-              - "observer.name"
-              - "process.name"
-              - "process.executable"
-              - "process.entity_id"
-              - "process.command_line"
-              - "process.Ext.ancestry"
-              - "process.parent.entity_id"
-              - "process.parent.command_line"
-              - "rule.category"
-              - "rule.name"
-              - "rule.uuid"
-              - "software.name"
-              - "software.type"
-              - "software.version.unparsed"
-              - "source.ip"
-              - "source.port"
-              - "source.geo.country_name"
-              - "ssh.cypher_algorithm"
-              - "ssh.client"
-              - "ssh.server"
-              - "ssl.cipher"
-              - "ssl.server_name"
-              - "ssl.version"
-              - "system.auth.sudo.command"
-              - "user.name"
-              - "user.domain"
-              - "user.effective.name"
-              - "weird.name"
-              - "tags"
         onionconfig:
           saltstackDir: /opt/so/saltstack
           bypassEnabled: false

salt/soc/soc_soc.yaml

@@ -916,11 +916,6 @@ soc:
             description: The number of times to retry extracting memories from a session if errors occur.
             global: True
             advanced: True
-          tools:
-            filterEventFields:
-              description: A whitelist of fields to return when OnionAI uses the query_events tool. All other fields are removed. One field per line.
-              global: True
-              multiline: True
       client:
         assistant:
           enabled: