Merge pull request #16238 from Security-Onion-Solutions/hotfix/3.3.0

Hotfix/3.3.0

.github/DISCUSSION_TEMPLATE/3-0.yml

@@ -13,7 +13,6 @@ body:
         - 3.1.0
         - 3.2.0
         - 3.3.0
-        - 3.4.0
         - Other (please provide detail below)
     validations:
       required: true

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 3.3.0-20260908 ISO image released on 2026/09/08
+### 3.3.0-20260911 ISO image released on 2026/09/11
 
 
 ### Download and Verify
 
-3.3.0-20260908 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-3.3.0-20260908.iso
+3.3.0-20260911 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-3.3.0-20260911.iso
  
-MD5: 5A2C42D0083F2D7B4DC2178C30EBC05F  
-SHA1: 505220A8A3315AFEE601C13772996018425CCD29  
-SHA256: 6EB8401296A1D051FEC558C520D2D4AB1912A72D351A2426FBF8C87FEE2BA844  
+MD5: 12B18433D3A2198A185892FF79CF638F  
+SHA1: 2B3C2E1FA7A78ED1F956E7EDCC12E32593C14EEE  
+SHA256: 0938C73B76CE30EC9E4394D312C79EA7CAC721B6818541697279A6221F7D870D  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.3.0-20260908.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.3.0-20260911.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.3.0-20260908.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.3.0-20260911.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-3.3.0-20260908.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-3.3.0-20260911.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-3.3.0-20260908.iso.sig securityonion-3.3.0-20260908.iso
+gpg --verify securityonion-3.3.0-20260911.iso.sig securityonion-3.3.0-20260911.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Tue 08 Sep 2026 10:07:12 AM EDT using RSA key ID FE507013
+gpg: Signature made Fri 11 Sep 2026 11:23:56 AM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

HOTFIX

@@ -1 +1 @@
-
+20260911

VERSION

@@ -1 +1 @@
-3.4.0
+3.3.0

salt/common/tools/sbin/so-sensor-clean

@@ -8,37 +8,21 @@
 # Elastic License 2.0.
 
 
-SENSOR_DIR="${SENSOR_DIR:-/nsm}"
+SENSOR_DIR='/nsm'
 CRIT_DISK_USAGE=90
-LOG="${LOG:-/opt/so/log/sensor_clean.log}"
-LOCK="${LOCK:-/var/tmp/so-sensor-clean.lock}"
-MAX_PASSES=100
+CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
+LOG="/opt/so/log/sensor_clean.log"
+TODAY=$(date -u "+%Y-%m-%d")
 
-ZEEK_LOGS="$SENSOR_DIR/zeek/logs"
-STRELKA_FILES="$SENSOR_DIR/strelka/processed"
-SURICATA_LOGS="$SENSOR_DIR/suricata"
-PCAPS="$SENSOR_DIR/pcapout"
-
-log() {
-	echo "$(date) - $*" >>"$LOG"
-}
-
-disk_usage() {
-	df -P "$SENSOR_DIR" | tail -1 | awk '{print $5}' | tr -d %
-}
-
-disk_avail() {
-	df -P "$SENSOR_DIR" | tail -1 | awk '{print $4}'
-}
-
-# sets REMOVED=1 if anything was actually deleted
 clean() {
 	## find the oldest Zeek logs directory
-	OLDEST_DIR=$(ls "$ZEEK_LOGS" 2>/dev/null | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
-	if [ -n "$OLDEST_DIR" ]; then
-		log "Removing directory: $ZEEK_LOGS/$OLDEST_DIR"
-		rm -rf "$ZEEK_LOGS/$OLDEST_DIR"
-		REMOVED=1
+	OLDEST_DIR=$(ls /nsm/zeek/logs/ | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
+	if [ -z "$OLDEST_DIR" -o "$OLDEST_DIR" == ".." -o "$OLDEST_DIR" == "." ]; then
+		echo "$(date) - No old Zeek logs available to clean up in /nsm/zeek/logs/" >>$LOG
+		#exit 0
+	else
+		echo "$(date) - Removing directory: /nsm/zeek/logs/$OLDEST_DIR" >>$LOG
+		rm -rf /nsm/zeek/logs/"$OLDEST_DIR"
 	fi
 
 	## Remarking for now, as we are moving extracted files to /nsm/strelka/processed
@@ -59,73 +43,58 @@ clean() {
 	#fi
 
 	## Clean up Zeek extracted files processed by Strelka
-	OLDEST_STRELKA=$(find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
-	if [ -n "$OLDEST_STRELKA" ]; then
+	STRELKA_FILES='/nsm/strelka/processed'
+	OLDEST_STRELKA=$(find $STRELKA_FILES -type f -printf '%T+ %p\n' | sort -n | head -n 1)
+	if [ -z "$OLDEST_STRELKA" -o "$OLDEST_STRELKA" == ".." -o "$OLDEST_STRELKA" == "." ]; then
+		echo "$(date) - No old files available to clean up in $STRELKA_FILES" >>$LOG
+	else
 		OLDEST_STRELKA_DATE=$(echo $OLDEST_STRELKA | awk '{print $1}' | cut -d+ -f1)
-		log "Removing extracted files for $OLDEST_STRELKA_DATE"
-		REMOVED=1
-		find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
-			log "Removing file: $FILE"
+		OLDEST_STRELKA_FILE=$(echo $OLDEST_STRELKA | awk '{print $2}')
+		echo "$(date) - Removing extracted files for $OLDEST_STRELKA_DATE" >>$LOG
+		find $STRELKA_FILES -type f -printf '%T+ %p\n' | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
+			echo "$(date) - Removing file: $FILE" >>$LOG
 			rm -f "$FILE"
 		done
 	fi
 
 	## Clean up Suricata log files
-	OLDEST_SURICATA=$(find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
-	if [ -n "$OLDEST_SURICATA" ]; then
+	SURICATA_LOGS='/nsm/suricata'
+	OLDEST_SURICATA=$(find $SURICATA_LOGS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
+	if [[ -z "$OLDEST_SURICATA" ]] || [[ "$OLDEST_SURICATA" == ".." ]] || [[ "$OLDEST_SURICATA" == "." ]]; then
+		echo "$(date) - No old files available to clean up in $SURICATA_LOGS" >>$LOG
+	else
 		OLDEST_SURICATA_DATE=$(echo $OLDEST_SURICATA | awk '{print $1}' | cut -d+ -f1)
-		log "Removing logs for $OLDEST_SURICATA_DATE"
-		REMOVED=1
-		find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
-			log "Removing file: $FILE"
+		OLDEST_SURICATA_FILE=$(echo $OLDEST_SURICATA | awk '{print $2}')
+		echo "$(date) - Removing logs for $OLDEST_SURICATA_DATE" >>$LOG
+		find $SURICATA_LOGS -type f -printf '%T+ %p\n' | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
+			echo "$(date) - Removing file: $FILE" >>$LOG
 			rm -f "$FILE"
 		done
 	fi
 
 	## Clean up extracted pcaps
-	OLDEST_PCAP=$(find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
-	if [ -n "$OLDEST_PCAP" ]; then
+	PCAPS='/nsm/pcapout'
+	OLDEST_PCAP=$(find $PCAPS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
+	if [ -z "$OLDEST_PCAP" -o "$OLDEST_PCAP" == ".." -o "$OLDEST_PCAP" == "." ]; then
+		echo "$(date) - No old files available to clean up in $PCAPS" >>$LOG
+	else
 		OLDEST_PCAP_DATE=$(echo $OLDEST_PCAP | awk '{print $1}' | cut -d+ -f1)
-		log "Removing extracted files for $OLDEST_PCAP_DATE"
-		REMOVED=1
-		find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
-			log "Removing file: $FILE"
+		OLDEST_PCAP_FILE=$(echo $OLDEST_PCAP | awk '{print $2}')
+		echo "$(date) - Removing extracted files for $OLDEST_PCAP_DATE" >>$LOG
+		find $PCAPS -type f -printf '%T+ %p\n' | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
+			echo "$(date) - Removing file: $FILE" >>$LOG
 			rm -f "$FILE"
 		done
 	fi
 }
 
-# Only one instance at a time; the lock is the fd, so it releases on any exit
-exec 9>"$LOCK" || exit 1
-if ! flock -n 9; then
-	log "another so-sensor-clean is already running (lock $LOCK held); exiting"
-	exit 0
+# Check to see if we are already running
+NUM_RUNNING=$(pgrep -cf "/bin/bash /usr/sbin/so-sensor-clean")
+[ "$NUM_RUNNING" -gt 1 ] && echo "$(date) - $NUM_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0
+
+if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then
+	while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
+		clean
+		CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
+	done
 fi
-
-CUR_USAGE=$(disk_usage)
-[ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ] || exit 0
-
-log "$SENSOR_DIR at ${CUR_USAGE}% (threshold ${CRIT_DISK_USAGE}%); starting cleanup"
-
-PASS=0
-while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
-	PASS=$((PASS + 1))
-	if [ "$PASS" -gt "$MAX_PASSES" ]; then
-		log "stopping after $MAX_PASSES passes; $SENSOR_DIR still at ${CUR_USAGE}%"
-		break
-	fi
-
-	REMOVED=0
-	BEFORE=$(disk_avail)
-	clean
-	CUR_USAGE=$(disk_usage)
-
-	if [ "$REMOVED" -eq 0 ]; then
-		log "nothing left to remove in $ZEEK_LOGS, $STRELKA_FILES, $SURICATA_LOGS, $PCAPS; $SENSOR_DIR still at ${CUR_USAGE}% - space is consumed outside of NSM cleanup scope"
-		break
-	fi
-	if [ "$(disk_avail)" -le "$BEFORE" ]; then
-		log "pass $PASS freed no space; $SENSOR_DIR still at ${CUR_USAGE}% - stopping until next run"
-		break
-	fi
-done

salt/manager/tools/sbin/soup

@@ -120,6 +120,9 @@ check_err() {
       161)
         echo 'Required intermediate Elasticsearch upgrade not complete'
       ;;
+      162)
+        echo 'One or more Elastic Agent nodes do not support the x86-64-v3 CPU instruction set'
+      ;;
       170)
         echo "Intermediate upgrade completed successfully to $next_step_so_version, but next soup to Security Onion $originally_requested_so_version could not be started automatically."
         echo "Start soup again manually to continue the upgrade to Security Onion $originally_requested_so_version."
@@ -347,6 +350,83 @@ check_cluster_health() {
   exit 0
 }
 
+no_soup_for_you() {
+  echo ""
+  echo "No soup for you!"
+  exit 162
+}
+
+check_cpu_compatibility() {
+  # Roles running a container built from the so-elastic-agent image; mirrors the
+  # elasticagent and elasticfleet entries in salt/reactor/pillar_push_map.yaml.
+  local cpu_target='G@role:so-heavynode or G@role:so-eval or G@role:so-fleet or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone'
+  local expected_nodes cpu_results node result confirm
+  local -a unsupported=() offline=()
+
+  echo "Checking that Elastic Agent nodes support the x86-64-v3 CPU instruction set now required by Elastic."
+
+  if [[ "$SKIP_CPU_CHECK" == "true" ]]; then
+    printf "\nSkipping the x86-64-v3 CPU check because --skip-cpu-check was specified.\n\n"
+    return
+  fi
+
+  # Nodes that never answer are absent from the results, so diff against who should have.
+  expected_nodes=$(salt -C "$cpu_target" --preview-target --out=json 2>/dev/null | jq -r '.[]?') || true
+  if [[ -z "$expected_nodes" ]]; then
+    printf "\nCould not determine which nodes run the Elastic Agent, so the x86-64-v3 CPU check cannot run.\n"
+    no_soup_for_you
+  fi
+
+  cpu_results=$(salt -t 30 -C "$cpu_target" cmd.run "/lib64/ld-linux-x86-64.so.2 --help | grep x86-64-v3" --out=json 2>/dev/null) || true
+
+  while IFS= read -r node; do
+    [[ -z "$node" ]] && continue
+    result=$(jq -r --arg node "$node" '.[$node] // empty' <<< "$cpu_results" 2>/dev/null)
+    if [[ -z "$result" || "$result" == *"did not return"* ]]; then
+      offline+=("$node")
+    elif [[ "$result" != *"x86-64-v3 (supported"* ]]; then
+      # glibc appends "(supported, searched)" only when supported; the open paren keeps
+      # this from matching a future "(unsupported".
+      unsupported+=("$node")
+    fi
+  done <<< "$expected_nodes"
+
+  if [[ ${#unsupported[@]} -eq 0 && ${#offline[@]} -eq 0 ]]; then
+    printf "\nAll Elastic Agent nodes support x86-64-v3. We can proceed with SOUP.\n\n"
+    return
+  fi
+
+  echo ""
+  if [[ ${#unsupported[@]} -gt 0 ]]; then
+    echo "The following node(s) do NOT support the x86-64-v3 CPU instruction set:"
+    printf '  %s\n' "${unsupported[@]}"
+    echo ""
+    echo "Upstream Elastic now builds its binaries for x86-64-v3, so these nodes can no"
+    echo "longer run Elastic. Upgrading them WILL BREAK them."
+    echo ""
+  fi
+  if [[ ${#offline[@]} -gt 0 ]]; then
+    echo "The following node(s) did not respond and could not be checked:"
+    printf '  %s\n' "${offline[@]}"
+    echo ""
+    echo "These nodes are offline, so we cannot confirm they support x86-64-v3, which"
+    echo "upstream Elastic now requires."
+    echo ""
+  fi
+
+  if [[ -n $UNATTENDED ]]; then
+    echo "Unattended mode cannot prompt for an override. Re-run soup interactively, or pass --skip-cpu-check to bypass this check."
+    no_soup_for_you
+  fi
+
+  read -rp "Type 'override' to continue anyway, or press Enter to exit: " confirm
+  if [[ "${confirm,,}" == "override" ]]; then
+    printf "\nOverride accepted. Continuing at your own risk.\n\n"
+  else
+    no_soup_for_you
+  fi
+}
+
 check_fleet_server() {
   echo "Checking that Elastic Fleet Server is responding."
   # Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls,
@@ -1977,6 +2057,9 @@ main() {
 
   echo "Let's see if we need to update Security Onion."
   upgrade_check
+
+  check_cpu_compatibility
+
   upgrade_space
 
   echo "Verifying Elasticsearch version compatibility across the grid before upgrading."
@@ -2255,6 +2338,17 @@ fi
   echo "### soup has been served at $(date) ###"
 }
 
+SKIP_CPU_CHECK=false
+declare -a SOUP_ARGS=()
+for arg in "$@"; do
+  if [[ "$arg" == "--skip-cpu-check" ]]; then
+    SKIP_CPU_CHECK=true
+  else
+    SOUP_ARGS+=("$arg")
+  fi
+done
+set -- "${SOUP_ARGS[@]}"
+
 while getopts ":b:f:y" opt; do
   case ${opt} in
     b )
@@ -2278,7 +2372,7 @@ while getopts ":b:f:y" opt; do
       ISOLOC="$OPTARG"
     ;;
     \? )
-      echo "Usage: soup [-b] [-y] [-f <iso location>]"
+      echo "Usage: soup [-b] [-y] [-f <iso location>] [--skip-cpu-check]"
       exit 1
     ;;
     : )

salt/nginx/etc/nginx.conf

@@ -183,7 +183,7 @@ http {
 		ssl_prefer_server_ciphers on;
 		ssl_protocols TLSv1.2 TLSv1.3;
 
-		location ~* (^/login|^/login/.*|^/js/.*|^/css/.*|^/images/.*|^/pages/.*|^/docs/.*) {
+		location ~* (^/login/.*|^/js/.*|^/css/.*|^/images/.*|^/pages/.*|^/docs/.*) {
 			proxy_pass            http://{{ GLOBALS.manager }}:9822;
 			proxy_read_timeout    90;
 			proxy_connect_timeout 90;

salt/soc/defaults.yaml

@@ -1537,7 +1537,7 @@ soc:
             Orchestrator: sonnet@SOAI
             Investigator: gemma@SOAI
             DetectionEngineer: gemma@SOAI
-          useMemory: true
+          useMemory: false
           useMemoryScanner: false
           dontScanBefore: ""
           memoryScanIntervalSeconds: 300
@@ -1556,69 +1556,6 @@ soc:
           reconcilePersona: ""
           toolUseTurnAttempts: 12
           toolUseTurnDelayMs: 175
-          tools:
-            filterEventFields:
-              - "@timestamp"
-              - "client.name"
-              - "destination.ip"
-              - "destination.port"
-              - "destination.geo.country_name"
-              - "dns.query.name"
-              - "dns.query_name"
-              - "event.action"
-              - "event.category"
-              - "event.module"
-              - "event.dataset"
-              - "event.outcome"
-              - "event.severity"
-              - "event.severity_label"
-              - "event.type"
-              - "event_data.agent.name"
-              - "event_data.host.os.name"
-              - "file.mime_type"
-              - "file.name"
-              - "hash.md5"
-              - "hash.sha1"
-              - "host.mac"
-              - "host.name"
-              - "host.os.name"
-              - "http.method"
-              - "http.useragent"
-              - "http.virtual_host"
-              - "log.id.uid"
-              - "network.community_id"
-              - "network.protocol"
-              - "network.transport"
-              - "notice.message"
-              - "observer.name"
-              - "process.name"
-              - "process.executable"
-              - "process.entity_id"
-              - "process.command_line"
-              - "process.Ext.ancestry"
-              - "process.parent.entity_id"
-              - "process.parent.command_line"
-              - "rule.category"
-              - "rule.name"
-              - "rule.uuid"
-              - "software.name"
-              - "software.type"
-              - "software.version.unparsed"
-              - "source.ip"
-              - "source.port"
-              - "source.geo.country_name"
-              - "ssh.cypher_algorithm"
-              - "ssh.client"
-              - "ssh.server"
-              - "ssl.cipher"
-              - "ssl.server_name"
-              - "ssl.version"
-              - "system.auth.sudo.command"
-              - "user.name"
-              - "user.domain"
-              - "user.effective.name"
-              - "weird.name"
-              - "tags"
         onionconfig:
           saltstackDir: /opt/so/saltstack
           bypassEnabled: false

salt/soc/soc_soc.yaml

@@ -916,11 +916,6 @@ soc:
             description: The number of times to retry extracting memories from a session if errors occur.
             global: True
             advanced: True
-          tools:
-            filterEventFields:
-              description: A whitelist of fields to return when OnionAI uses the query_events tool. All other fields are removed. One field per line.
-              global: True
-              multiline: True
       client:
         assistant:
           enabled: