oracle 10 testing

Merge pull request #15937 from Security-Onion-Solutions/hotfix/3.1.0
Hotfix/3.1.0
2026-06-12 05:15:26 +02:00 · 2026-06-03 14:11:30 -04:00 · 2026-05-28 10:20:53 -04:00 · 2026-05-28 09:53:00 -04:00 · 2026-05-28 09:34:17 -04:00 · 2026-05-27 13:37:41 -05:00
73 changed files with 2356 additions and 1107 deletions
@@ -1,17 +1,17 @@
-### 3.0.0-20260331 ISO image released on 2026/03/31
+### 3.1.0-20260528 ISO image released on 2026/05/28


 ### Download and Verify

-3.0.0-20260331 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
+3.1.0-20260528 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-3.1.0-20260528.iso
 
-MD5: ECD318A1662A6FDE0EF213F5A9BD4B07  
-SHA1: E55BE314440CCF3392DC0B06BC5E270B43176D9C  
-SHA256: 7FC47405E335CBE5C2B6C51FE7AC60248F35CBE504907B8B5A33822B23F8F4D5  
+MD5: 9D6FF58DEEE24089D722C73169765B3E  
+SHA1: 2B8B816B6CEC3B7F96B3C5E040EBF502DD2C412F  
+SHA256: 62FAB57E247C843D6A04F0796D8162C732B65D82FC3E4A59D087135B9FD32912  

 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.1.0-20260528.iso.sig

 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/

 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.1.0-20260528.iso.sig
 ```

 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-3.1.0-20260528.iso
 ```

 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-3.0.0-20260331.iso.sig securityonion-3.0.0-20260331.iso
+gpg --verify securityonion-3.1.0-20260528.iso.sig securityonion-3.1.0-20260528.iso
 ```

 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Mon 30 Mar 2026 06:22:14 PM EDT using RSA key ID FE507013
+gpg: Signature made Wed 27 May 2026 03:03:59 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.
@@ -0,0 +1 @@
+20260528
@@ -26,14 +26,33 @@ commonpkgs:
      - net-tools
      - nmap-ncat
      - procps-ng
+{# OL10 test path: python3-docker / python3-m2crypto are not packaged in EPEL 10 and are not
+   referenced by SO code (salt uses its bundled docker module from salt/python_modules.sls).
+   python3-rich is also unavailable on EL10 (its pygments dep is not packaged), so it is
+   installed via pip below. Gate on the grain because GLOBALS/pillars are not available this
+   early (see header note). #}
+{% if grains['osmajorrelease']|int < 10 %}
      - python3-docker
      - python3-m2crypto
+      - python3-rich
+{% else %}
+      - python3-pip
+{% endif %}
      - python3-packaging
      - python3-pyyaml
-      - python3-rich
      - rsync
      - sqlite
      - tcpdump
      - unzip
      - wget
      - yum-utils
+
+{% if grains['osmajorrelease']|int >= 10 %}
+# OL10 test path: rich is not packaged for EL10; install it into the system python3 for so-status.
+commonpkgs_pip_rich:
+  cmd.run:
+    - name: python3 -m pip install rich
+    - unless: python3 -c "import rich"
+    - require:
+      - pkg: commonpkgs
+{% endif %}
@@ -48,13 +48,6 @@ copy_so-yaml_manager_tools_sbin:
    - force: True
    - preserve: True

-copy_so-config_manager_tools_sbin:
-  file.copy:
-    - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-config.py
-    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-config.py
-    - force: True
-    - preserve: True
-
 copy_so-repo-sync_manager_tools_sbin:
  file.copy:
    - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-repo-sync
@@ -104,13 +97,6 @@ copy_so-yaml_sbin:
    - force: True
    - preserve: True

-copy_so-config_sbin:
-  file.copy:
-    - name: /usr/sbin/so-config.py
-    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-config.py
-    - force: True
-    - preserve: True
-
 copy_so-repo-sync_sbin:
  file.copy:
    - name: /usr/sbin/so-repo-sync
@@ -354,7 +354,12 @@ gpg_rpm_import() {
 	else
 		local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/$OS/keys"
 	fi
-	RPMKEYS=('RPM-GPG-KEY-oracle' 'RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub')
+	if [[ "$OSVER" == "10" ]]; then
+		# OL10 test path uses public repos; the public oracle-epel-release and docker repos provide their own keys
+		RPMKEYS=('RPM-GPG-KEY-oracle' 'SALT-PROJECT-GPG-PUBKEY-2023.pub')
+	else
+		RPMKEYS=('RPM-GPG-KEY-oracle' 'RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub')
+	fi
 	for RPMKEY in "${RPMKEYS[@]}"; do
 		rpm --import $RPMKEYSLOC/$RPMKEY
 		echo "Imported $RPMKEY"
@@ -626,9 +631,9 @@ salt_minion_count() {
 }

 set_os() {
-	if [ -f /etc/redhat-release ] && grep -q "Red Hat Enterprise Linux release 9" /etc/redhat-release && [ -f /etc/oracle-release ]; then
+	if [ -f /etc/oracle-release ] && grep -qE "release (9|10)\b" /etc/oracle-release; then
 		OS=oracle
-		OSVER=9
+		OSVER=$(grep -oE "release [0-9]+" /etc/oracle-release | grep -oE "[0-9]+")
 		is_oracle=true
 		is_rpm=true
 	fi
@@ -112,8 +112,23 @@ update_docker_containers() {
  # does not include so-elastic-fleet since that container uses so-elastic-agent image
  local IMAGES_USING_ES_VERSION=("so-elasticsearch")

-  rm -rf $SIGNPATH >> "$LOG_FILE" 2>&1 
-  mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1 
+  rm -rf $SIGNPATH >> "$LOG_FILE" 2>&1
+  mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1
+
+  # OL10 test path: GnuPG 2.4 enables the keybox daemon (keyboxd) by default, which deadlocks
+  # under the rapid sequential gpg --verify calls below ("waiting for lock ... keydb_search
+  # failed: Connection timed out ... No public key"). Editing the default homedir's common.conf
+  # is unreliable (gpg re-adds use-keyboxd when it re-initializes the homedir), so run all the
+  # image-signature gpg ops in a dedicated homedir whose pre-written common.conf leaves keyboxd
+  # off, forcing the classic keybox. Isolated from the system keyring and deterministic.
+  if [ "$OSVER" = "10" ]; then
+    export GNUPGHOME="$SIGNPATH/gnupg"
+    rm -rf "$GNUPGHOME" >> "$LOG_FILE" 2>&1
+    mkdir -p "$GNUPGHOME" >> "$LOG_FILE" 2>&1
+    chmod 700 "$GNUPGHOME"
+    echo "# keyboxd disabled for SO image signature verification on EL10" > "$GNUPGHOME/common.conf"
+    gpgconf --kill keyboxd gpg-agent >> "$LOG_FILE" 2>&1 || true
+  fi

  # Let's make sure we have the public key
  run_check_net_err \
@@ -164,8 +179,8 @@ update_docker_containers() {
    # Pull down the trusted docker image
    run_check_net_err \
    "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" \
-    "Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 
-    
+    "Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1
+
    # Get signature
    run_check_net_err \
    "curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' $sig_url --output $SIGNPATH/$image.sig" \
@@ -189,11 +204,24 @@ update_docker_containers() {
          HOSTNAME=$(hostname)
        fi
        docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$image $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 || {
-          echo "Unable to tag $image" >> "$LOG_FILE" 2>&1 
+          echo "Unable to tag $image" >> "$LOG_FILE" 2>&1
          exit 1
        }
-        docker push $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 || {
-          echo "Unable to push $image" >> "$LOG_FILE" 2>&1 
+        # Push to the embedded registry via a registry-to-registry copy. Avoids
+        # `docker push`, which on Docker 29.x with the containerd image store
+        # represents freshly-pulled images as an index whose layer content
+        # isn't reachable through the push path. The local `docker tag` above
+        # is preserved so so-image-pull's `:5000` existence check still works.
+        # Pin to the digest already gpg-verified above so we copy exactly the
+        # bytes we approved.
+        local VERIFIED_REF
+        VERIFIED_REF=$(echo "$DOCKERINSPECT" | jq -r ".[0].RepoDigests[] | select(. | contains(\"$CONTAINER_REGISTRY\"))" | head -n 1)
+        if [ -z "$VERIFIED_REF" ] || [ "$VERIFIED_REF" = "null" ]; then
+          echo "Unable to determine verified digest for $image" >> "$LOG_FILE" 2>&1
+          exit 1
+        fi
+        docker buildx imagetools create --tag $HOSTNAME:5000/$IMAGEREPO/$image "$VERIFIED_REF" >> "$LOG_FILE" 2>&1 || {
+          echo "Unable to copy $image to embedded registry" >> "$LOG_FILE" 2>&1
          exit 1
        }
      fi
@@ -165,6 +165,8 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading component template"  # false positive (elasticsearch index or template names contain 'error')
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading composable template" # false positive (elasticsearch composable template names contain 'error')
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error while parsing document for index \[.ds-logs-kratos-so-.*object mapping for \[file\]" # false positive (mapping error occuring BEFORE kratos index has rolled over in 2.4.210)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|No such container"            # false positive (telegraf trying to run stats on an old container)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|passwords do not match"       # false positive (automated hydra test)
 fi

 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
@@ -227,7 +229,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|from NIC checksum offloading" # zeek reporter.log
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal"           # docker container getting recycled
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-(tychon|aws_billing|microsoft_defender_endpoint|armis|o365_metrics|microsoft_sentinel|snyk).*user so_kibana lacks the required permissions \[(logs|metrics)-\1" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user. (installed as so_elastic / so_kibana)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-(tychon|aws_billing|microsoft_defender_endpoint|armis|o365_metrics|microsoft_sentinel|snyk|cyera|island_browser).*user so_kibana lacks the required permissions \[(logs|metrics)-\1" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user. This error should not be seen on fresh ES 9.3.3 installs or after SO 3.1.0 with soups addition of check_transform_health_and_reauthorize()
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown"             # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1
 fi

@@ -18,10 +18,18 @@ dockergroup:
 dockerheldpackages:
  pkg.installed:
    - pkgs:
+{% if GLOBALS.os_version|int >= 10 %}
+      # OL10 test path: install latest Docker CE from the public repo (no .el9 builds available)
+      - containerd.io
+      - docker-ce
+      - docker-ce-cli
+      - docker-ce-rootless-extras
+{% else %}
      - containerd.io: 2.2.1-1.el9
      - docker-ce: 3:29.2.1-1.el9
      - docker-ce-cli: 1:29.2.1-1.el9
      - docker-ce-rootless-extras: 29.2.1-1.el9
+{% endif %}
    - hold: True
    - update_holds: True

@@ -51,6 +51,16 @@ so-elastic-fleet-package-registry:
      - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
    {%   endfor %}
    {% endif %}
+
+wait_for_so-elastic-fleet-package-registry:
+  http.wait_for_successful_query:
+    - name: "http://localhost:8080/health"
+    - status: 200
+    - wait_for: 300
+    - request_interval: 15
+    - require:
+      - docker_container: so-elastic-fleet-package-registry
+
 delete_so-elastic-fleet-package-registry_so-status.disabled:
  file.uncomment:
    - name: /opt/so/conf/so-status/so-status.conf
@@ -26,7 +26,9 @@ include:
 wait_for_elasticsearch_elasticfleet:
  cmd.run:
    - name: so-elasticsearch-wait
+{% endif %}

+{% if GLOBALS.role == "so-fleet" %}
 # Sync Elastic Agent artifacts to Fleet Node
 elasticagent_syncartifacts:
  file.recurse:
@@ -18,17 +18,6 @@ so-elastic-fleet-auto-configure-logstash-outputs:
    - retry:
        attempts: 4
        interval: 30
-
-{# Separate from above in order to catch elasticfleet-logstash.crt changes and force update to fleet output policy #}
-so-elastic-fleet-auto-configure-logstash-outputs-force:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-outputs-update --certs
-    - retry:
-        attempts: 4
-        interval: 30
-    - onchanges:
-        - x509: etc_elasticfleet_logstash_crt
-        - x509: elasticfleet_kafka_crt
 {% endif %}

 # If enabled, automatically update Fleet Server URLs & ES Connection
@@ -240,7 +240,7 @@ elastic_fleet_policy_create() {
        --arg DESC "$DESC" \
        --arg TIMEOUT $TIMEOUT \
        --arg FLEETSERVER "$FLEETSERVER" \
-            '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER}'
+            '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER,"advanced_settings":{"agent_logging_level": "warning"}}'
        )
    # Create Fleet Policy
    if ! fleet_api "agent_policies" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
@@ -235,6 +235,16 @@ function update_kafka_outputs() {

 {% endif %}

+# Compare the current Elastic Fleet certificate against what is on disk
+POLICY_CERT_SHA=$(jq -r '.item.ssl.certificate' <<< $RAW_JSON | openssl x509 -noout -sha256 -fingerprint)
+DISK_CERT_SHA=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt -noout -sha256 -fingerprint)
+
+if [[ "$POLICY_CERT_SHA" != "$DISK_CERT_SHA" ]]; then
+    printf "Certificate on disk doesn't match certificate in policy - forcing update\n"
+    UPDATE_CERTS=true
+    FORCE_UPDATE=true
+fi
+
 # Sort & hash the new list of Logstash Outputs
 NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}")
 NEW_HASH=$(sha256sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
@@ -232,7 +232,6 @@ printf '%s\n'\
    "      grid_enrollment_general: '$GRIDNODESENROLLMENTOKENGENERAL'"\
    "      grid_enrollment_heavy: '$GRIDNODESENROLLMENTOKENHEAVY'"\
    "" >> "$pillar_file"
-/usr/sbin/so-config.py import-file "$pillar_file" --note "so-elastic-fleet-setup"

 #Store Grid Nodes Enrollment token in Global pillar
 global_pillar_file=/opt/so/saltstack/local/pillar/global/soc_global.sls
@@ -240,7 +239,6 @@ printf '%s\n'\
    "  fleet_grid_enrollment_token_general: '$GRIDNODESENROLLMENTOKENGENERAL'"\
    "  fleet_grid_enrollment_token_heavy: '$GRIDNODESENROLLMENTOKENHEAVY'"\
    "" >> "$global_pillar_file"
-/usr/sbin/so-config.py import-file "$global_pillar_file" --note "so-elastic-fleet-setup"

 # Call Elastic-Fleet Salt State
 printf "\nApplying elasticfleet state"
@@ -3958,10 +3958,13 @@ elasticsearch:
        - vulnerability-mappings
        - common-settings
        - common-dynamic-mappings
+        - logs-redis.log@package
+        - logs-redis.log@custom
        data_stream:
          allow_custom_routing: false
          hidden: false
-        ignore_missing_component_templates: []
+        ignore_missing_component_templates:
+        - logs-redis.log@custom
        index_patterns:
        - logs-redis.log*
        priority: 501
@@ -63,7 +63,8 @@
    { "set":             { "if": "ctx.event?.dataset != null && !ctx.event.dataset.contains('.')", "field": "event.dataset", "value": "{{event.module}}.{{event.dataset}}" } },
    { "split":           { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "dataset_tag_temp" } },
    { "append":          { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}"  } },
-    { "grok":            { "if": "ctx.http?.response?.status_code != null", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long} %{GREEDYDATA}"]} },
+    { "grok":            { "if": "ctx.http?.response?.status_code instanceof String", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long}(?:\\s+%{GREEDYDATA})?"], "ignore_failure": true } },
+    { "convert":         { "if": "ctx.http?.response?.status_code != null && !(ctx.http.response.status_code instanceof Number)", "field": "http.response.status_code", "type": "long", "ignore_failure": true } },
    { "set":             { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
    { "remove":          { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } },
    { "pipeline": { "name": "global@custom", "ignore_missing_pipeline": true, "description": "[Fleet] Global pipeline for all data streams" } }
@@ -177,12 +177,84 @@
                "description": "Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"
            }
        },
+        {
+            "script": {
+                "description": "Snapshot event.ingested into _tmp.event_ingested_pre_fleet before .fleet_final_pipeline-1 overwrites it with ES ingest time",
+                "lang": "painless",
+                "if": "ctx.event?.ingested != null && ctx.event?.created == null",
+                "ignore_failure": true,
+                "source": "ctx.putIfAbsent('_tmp', [:]); ctx._tmp.event_ingested_pre_fleet = ctx.event.ingested;"
+            }
+        },
        {
            "pipeline": {
                "name": ".fleet_final_pipeline-1",
                "ignore_missing_pipeline": true
            }
        },
+        {
+            "script": {
+                "description": "Calculate time from Elastic Agent to Logstash.",
+                "lang": "painless",
+                "if": "ctx._tmp?.logstash_from_agent != null",
+                "ignore_failure": true,
+                "source": "ZonedDateTime start = ctx._tmp.event_ingested_pre_fleet != null ? ZonedDateTime.parse(ctx._tmp.event_ingested_pre_fleet) : ZonedDateTime.parse(ctx['@timestamp']); ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_elasticagent_to_logstash = ChronoUnit.SECONDS.between(start, ZonedDateTime.parse(ctx._tmp.logstash_from_agent));"
+            }
+        },
+        {
+            "script": {
+                "description": "Calculate time from Logstash to Redis",
+                "lang": "painless",
+                "if": "ctx._tmp?.logstash_from_agent != null && ctx._tmp?.logstash_to_redis != null",
+                "ignore_failure": true,
+                "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_logstash_to_redis = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_from_agent), ZonedDateTime.parse(ctx._tmp.logstash_to_redis));"
+            }
+        },
+        {
+            "script": {
+                "description": "Calculate time message spends in redis queue (logstash delay in pulling event).",
+                "lang": "painless",
+                "if": "ctx._tmp?.logstash_to_redis != null && ctx._tmp?.logstash_from_redis != null",
+                "ignore_failure": true,
+                "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_redis_to_logstash = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_to_redis), ZonedDateTime.parse(ctx._tmp.logstash_from_redis));"
+            }
+        },
+        {
+            "script": {
+                "description": "Calculate time from Logstash to Elasticsearch (after read from Redis).",
+                "lang": "painless",
+                "if": "ctx._tmp?.logstash_from_redis != null",
+                "ignore_failure": true,
+                "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_logstash_to_elasticsearch = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_from_redis), metadata().now);"
+            }
+        },
+        {
+            "script": {
+                "description": "Calculate time from Elastic Agent to Kafka.",
+                "lang": "painless",
+                "if": "ctx._tmp?.logstash_from_kafka != null && ctx._tmp?.logstash_from_agent == null",
+                "ignore_failure": true,
+                "source": "ZonedDateTime start = ctx._tmp.event_ingested_pre_fleet != null ? ZonedDateTime.parse(ctx._tmp.event_ingested_pre_fleet) : ZonedDateTime.parse(ctx['@timestamp']); ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_elasticagent_to_kafka = ChronoUnit.SECONDS.between(start, ZonedDateTime.parse(ctx._tmp.logstash_from_kafka));"
+            }
+        },
+        {
+            "script": {
+                "description": "Calculate time message spends in Kafka queue (logstash delay in pulling event).",
+                "lang": "painless",
+                "if": "ctx._tmp?.logstash_from_kafka != null && ctx.metadata?.kafka?.timestamp != null && ctx._tmp?.logstash_from_agent == null",
+                "ignore_failure": true,
+                "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_kafka_queue = ChronoUnit.SECONDS.between(ZonedDateTime.ofInstant(Instant.ofEpochMilli(Long.parseLong(ctx.metadata.kafka.timestamp.toString())), ZoneId.of('UTC')), ZonedDateTime.parse(ctx._tmp.logstash_from_kafka));"
+            }
+        },
+        {
+            "script": {
+                "description": "Calculate time from Logstash to Elasticsearch (after read from Kafka).",
+                "lang": "painless",
+                "if": "ctx._tmp?.logstash_from_kafka != null && ctx._tmp?.logstash_from_agent == null",
+                "ignore_failure": true,
+                "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_kafka_to_elasticsearch = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_from_kafka), metadata().now);"
+            }
+        },
        {
            "remove": {
                "field": "event.agent_id_status",
@@ -202,11 +274,12 @@
                    "event.dataset_temp",
                    "dataset_tag_temp",
                    "module_temp",
-                    "datastream_dataset_temp"
+                    "datastream_dataset_temp",
+                    "_tmp"
                ],
                "ignore_missing": true,
                "ignore_failure": true
            }
        }
    ]
-}
+}
@@ -0,0 +1,71 @@
+{
+    "description": "zeek.ja4d",
+    "processors": [
+        {
+            "set": {
+                "field": "event.dataset",
+                "value": "ja4d"
+            }
+        },
+        {
+            "remove": {
+                "field": [
+                    "host"
+                ],
+                "ignore_failure": true
+            }
+        },
+        {
+            "json": {
+                "field": "message",
+                "target_field": "message2",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.ja4d",
+                "target_field": "hash.ja4d",
+                "ignore_missing": true,
+                "if": "ctx?.message2?.ja4d != null && ctx.message2.ja4d.length() > 0"
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.client_mac",
+                "target_field": "host.mac",
+                "ignore_missing": true,
+                "if": "ctx?.message2?.client_mac != null && ctx.message2.client_mac.length() > 0"
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.hostname",
+                "target_field": "host.hostname",
+                "ignore_missing": true,
+                "if": "ctx?.message2?.hostname != null && ctx.message2.hostname.length() > 0"
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.requested_ip",
+                "target_field": "dhcp.requested_address",
+                "ignore_missing": true,
+                "if": "ctx?.message2?.requested_ip != null && ctx.message2.requested_ip.length() > 0"
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.vendor_class_id",
+                "target_field": "zeek.ja4d.vendor_class_id",
+                "ignore_missing": true,
+                "if": "ctx?.message2?.vendor_class_id != null && ctx.message2.vendor_class_id.length() > 0"
+            }
+        },
+        {
+            "pipeline": {
+                "name": "zeek.common"
+            }
+        }
+    ]
+}
@@ -398,6 +398,7 @@ firewall:
                - elasticsearch_rest
                - docker_registry
                - influxdb
+                - postgres
                - sensoroni
                - yum
                - beats_5044
@@ -410,6 +411,7 @@ firewall:
              portgroups:
                - docker_registry
                - influxdb
+                - postgres
                - sensoroni
                - yum
                - beats_5044
@@ -427,6 +429,7 @@ firewall:
                - yum
                - docker_registry
                - influxdb
+                - postgres
                - sensoroni
            searchnode:
              portgroups:
@@ -437,6 +440,7 @@ firewall:
                - yum
                - docker_registry
                - influxdb
+                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -450,6 +454,7 @@ firewall:
                - yum
                - docker_registry
                - influxdb
+                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -459,6 +464,7 @@ firewall:
                - yum
                - docker_registry
                - influxdb
+                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -492,6 +498,7 @@ firewall:
              portgroups:
                - docker_registry
                - influxdb
+                - postgres
                - sensoroni
                - yum
                - elastic_agent_control
@@ -502,6 +509,7 @@ firewall:
                - yum
                - docker_registry
                - influxdb
+                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -610,6 +618,7 @@ firewall:
                - elasticsearch_rest
                - docker_registry
                - influxdb
+                - postgres
                - sensoroni
                - yum
                - beats_5044
@@ -622,6 +631,7 @@ firewall:
              portgroups:
                - docker_registry
                - influxdb
+                - postgres
                - sensoroni
                - yum
                - beats_5044
@@ -639,6 +649,7 @@ firewall:
                - yum
                - docker_registry
                - influxdb
+                - postgres
                - sensoroni
            searchnode:
              portgroups:
@@ -649,6 +660,7 @@ firewall:
                - yum
                - docker_registry
                - influxdb
+                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -662,6 +674,7 @@ firewall:
                - yum
                - docker_registry
                - influxdb
+                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -671,6 +684,7 @@ firewall:
                - yum
                - docker_registry
                - influxdb
+                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -702,6 +716,7 @@ firewall:
              portgroups:
                - docker_registry
                - influxdb
+                - postgres
                - sensoroni
                - yum
                - elastic_agent_control
@@ -712,6 +727,7 @@ firewall:
                - yum
                - docker_registry
                - influxdb
+                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -820,6 +836,7 @@ firewall:
                - elasticsearch_rest
                - docker_registry
                - influxdb
+                - postgres
                - sensoroni
                - yum
                - beats_5044
@@ -832,6 +849,7 @@ firewall:
              portgroups:
                - docker_registry
                - influxdb
+                - postgres
                - sensoroni
                - yum
                - beats_5044
@@ -849,6 +867,7 @@ firewall:
                - yum
                - docker_registry
                - influxdb
+                - postgres
                - sensoroni
            searchnode:
              portgroups:
@@ -858,6 +877,7 @@ firewall:
                - yum
                - docker_registry
                - influxdb
+                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -870,6 +890,7 @@ firewall:
                - yum
                - docker_registry
                - influxdb
+                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -879,6 +900,7 @@ firewall:
                - yum
                - docker_registry
                - influxdb
+                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -912,6 +934,7 @@ firewall:
              portgroups:
                - docker_registry
                - influxdb
+                - postgres
                - sensoroni
                - yum
                - elastic_agent_control
@@ -922,6 +945,7 @@ firewall:
                - yum
                - docker_registry
                - influxdb
+                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -1040,6 +1064,7 @@ firewall:
                - elasticsearch_rest
                - docker_registry
                - influxdb
+                - postgres
                - sensoroni
                - yum
                - beats_5044
@@ -1052,6 +1077,7 @@ firewall:
              portgroups:
                - docker_registry
                - influxdb
+                - postgres
                - sensoroni
                - yum
                - beats_5044
@@ -1063,6 +1089,7 @@ firewall:
              portgroups:
                - docker_registry
                - influxdb
+                - postgres
                - sensoroni
                - yum
                - beats_5044
@@ -1074,6 +1101,7 @@ firewall:
              portgroups:
                - docker_registry
                - influxdb
+                - postgres
                - sensoroni
                - yum
                - redis
@@ -1083,6 +1111,7 @@ firewall:
              portgroups:
                - docker_registry
                - influxdb
+                - postgres
                - sensoroni
                - yum
                - redis
@@ -1093,6 +1122,7 @@ firewall:
                - yum
                - docker_registry
                - influxdb
+                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -1129,6 +1159,7 @@ firewall:
              portgroups:
                - docker_registry
                - influxdb
+                - postgres
                - sensoroni
                - yum
                - elastic_agent_control
@@ -1139,6 +1170,7 @@ firewall:
                - yum
                - docker_registry
                - influxdb
+                - postgres
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
@@ -1482,6 +1514,7 @@ firewall:
                - kibana
                - redis
                - influxdb
+                - postgres
                - elasticsearch_rest
                - elasticsearch_node
                - elastic_agent_control
@@ -1,6 +1,5 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% from 'docker/docker.map.jinja' import DOCKERMERGED %}
-{% from 'telegraf/map.jinja' import TELEGRAFMERGED %}
 {% import_yaml 'firewall/defaults.yaml' as FIREWALL_DEFAULT %}

 {# add our ip to self #}
@@ -56,16 +55,4 @@

 {% endif %}

-{# Open Postgres (5432) to minion hostgroups when Telegraf is configured to write to Postgres #}
-{% set TG_OUT = TELEGRAFMERGED.output | upper %}
-{% if TG_OUT in ['POSTGRES', 'BOTH'] %}
-{%   if role.startswith('manager') or role == 'standalone' or role == 'eval' %}
-{%     for r in ['sensor', 'searchnode', 'heavynode', 'receiver', 'fleet', 'idh', 'desktop', 'import'] %}
-{%       if FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r] is defined %}
-{%         do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r].portgroups.append('postgres') %}
-{%       endif %}
-{%     endfor %}
-{%   endif %}
-{% endif %}
-
 {% set FIREWALL_MERGED = salt['pillar.get']('firewall', FIREWALL_DEFAULT.firewall, merge=True) %}
@@ -20,11 +20,8 @@ so-kafka_so-status.disabled:
 ensure_default_pipeline:
  cmd.run:
    - name: |
-        set -e
-        /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled False
-        /usr/sbin/so-config.py sync-yaml-mutation /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls replace kafka.enabled False --note "kafka.disabled"
+        /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled False;
        /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/global/soc_global.sls global.pipeline REDIS
-        /usr/sbin/so-config.py sync-yaml-mutation /opt/so/saltstack/local/pillar/global/soc_global.sls replace global.pipeline REDIS --note "kafka.disabled"
 {% endif %}

 {# If Kafka has never been manually enabled, the 'Kafka' user does not exist. In this case certs for Kafka should not exist since they'll be owned by uid 960 #}
@@ -34,4 +31,4 @@ check_kafka_cert_{{cert}}:
    - name: /etc/pki/{{cert}}
    - onlyif: stat -c %U /etc/pki/{{cert}} | grep -q UNKNOWN
    - show_changes: False
-{% endfor %}
+{% endfor %}
@@ -26,12 +26,12 @@ logstash:
    manager:
      - so/0011_input_endgame.conf
      - so/0012_input_elastic_agent.conf.jinja
-      - so/0013_input_lumberjack_fleet.conf
+      - so/0013_input_lumberjack_fleet.conf.jinja
      - so/9999_output_redis.conf.jinja
    receiver:
      - so/0011_input_endgame.conf
      - so/0012_input_elastic_agent.conf.jinja
-      - so/0013_input_lumberjack_fleet.conf
+      - so/0013_input_lumberjack_fleet.conf.jinja
      - so/9999_output_redis.conf.jinja
    search:
      - so/0900_input_redis.conf.jinja
@@ -69,4 +69,5 @@ logstash:
    pipeline_x_batch_x_size: 125
    pipeline_x_ecs_compatibility: disabled
  dmz_nodes: []
+  latency_metrics: False

@@ -1,3 +1,4 @@
+{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
 input {
  elastic_agent {
    port => 5055
@@ -11,10 +12,15 @@ input {
  }
 }
 filter {
-if ![metadata] {
-  mutate {
-    rename => {"@metadata" => "metadata"}
+  {% if LOGSTASH_MERGED.get('latency_metrics', False) %}
+  ruby {
+    code => "event.set('[_tmp][logstash_from_agent]', Time.now().utc.iso8601(3));"
+  }
+  {% endif %}
+  if ![metadata] {
+    mutate {
+      rename => {"@metadata" => "metadata"}
+    }
  }
 }
-}

@@ -1,23 +0,0 @@
-input {
-  elastic_agent {
-    port => 5056
-    tags => [ "elastic-agent", "fleet-lumberjack-input" ]
-    ssl_enabled => true
-    ssl_certificate => "/usr/share/logstash/elasticfleet-lumberjack.crt"
-    ssl_key => "/usr/share/logstash/elasticfleet-lumberjack.key"
-    ecs_compatibility => v8
-    id => "fleet-lumberjack-in"  
-    codec => "json"
-  }
-}
-
-
-filter {
-if ![metadata] {
-  mutate {
-    rename => {"@metadata" => "metadata"}
-  }
-}
-}
-
-
@@ -0,0 +1,26 @@
+{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
+input {
+  elastic_agent {
+    port => 5056
+    tags => [ "elastic-agent", "fleet-lumberjack-input" ]
+    ssl_enabled => true
+    ssl_certificate => "/usr/share/logstash/elasticfleet-lumberjack.crt"
+    ssl_key => "/usr/share/logstash/elasticfleet-lumberjack.key"
+    ecs_compatibility => v8
+    id => "fleet-lumberjack-in"
+    codec => "json"
+  }
+}
+
+filter {
+  {% if LOGSTASH_MERGED.get('latency_metrics', False) %}
+  ruby {
+    code => "event.set('[_tmp][logstash_from_fleet]', Time.now().utc.iso8601(3));"
+  }
+  {% endif %}
+  if ![metadata] {
+    mutate {
+      rename => {"@metadata" => "metadata"}
+    }
+  }
+}
@@ -1,3 +1,4 @@
+{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
 {%- set kafka_password = salt['pillar.get']('kafka:config:password') %}
 {%- set kafka_trustpass = salt['pillar.get']('kafka:config:trustpass') %}
 {%- set kafka_brokers = salt['pillar.get']('kafka:nodes', {}) %}
@@ -30,6 +31,11 @@ input {
    }
 }
 filter {
+  {% if LOGSTASH_MERGED.get('latency_metrics', False) %}
+  ruby {
+    code => "event.set('[_tmp][logstash_from_kafka]', Time.now().utc.iso8601(3));"
+  }
+  {% endif %}
  if ![metadata] {
    mutate {
      rename => { "@metadata" => "metadata" }
@@ -1,4 +1,4 @@
-{%- from 'logstash/map.jinja' import LOGSTASH_REDIS_NODES with context %}
+{%- from 'logstash/map.jinja' import LOGSTASH_REDIS_NODES, LOGSTASH_MERGED %}
 {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %}

 {%- for index in range(LOGSTASH_REDIS_NODES|length) %}
@@ -18,3 +18,10 @@ input {
 }
 {%   endfor %}
 {% endfor -%}
+filter {
+  {% if LOGSTASH_MERGED.get('latency_metrics', False) %}
+  ruby {
+    code => "event.set('[_tmp][logstash_from_redis]', Time.now().utc.iso8601(3));"
+  }
+  {% endif %}
+}
@@ -1,3 +1,11 @@
+{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
+{% if LOGSTASH_MERGED.get('latency_metrics', False) %}
+filter {
+  ruby {
+    code => "event.set('[_tmp][logstash_to_elasticsearch]', Time.now().utc.iso8601(3));"
+  }
+}
+{% endif %}
 output {
  if "elastic-agent" in [tags] and "so-ip-mappings" in [tags] {
    elasticsearch {
@@ -13,13 +13,20 @@ filter {
                    add_tag => "fleet-lumberjack-{{ GLOBALS.hostname }}"
          }
  }
-
-output { 
-    lumberjack { 
-        codec => json 
+{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
+{% if LOGSTASH_MERGED.get('latency_metrics', False) %}
+filter {
+  ruby {
+    code => "event.set('[_tmp][fleet_to_logstash]', Time.now().utc.iso8601(3));"
+  }
+}
+{% endif %}
+output {
+    lumberjack {
+        codec => json
        hosts => {{  FAILOVER_LOGSTASH_NODES }}
        ssl_certificate => "/usr/share/filebeat/ca.crt"
-        port => 5056 
+        port => 5056
        id => "fleet-lumberjack-{{ GLOBALS.hostname }}"
-        } 
+        }
    }
@@ -1,10 +1,17 @@
+{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
 {%- if grains.role in ['so-heavynode', 'so-receiver'] %}
  {%- set HOST = GLOBALS.hostname %}
 {%- else %}
  {%- set HOST = GLOBALS.manager %}
 {%- endif %}
 {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %}
-
+{% if LOGSTASH_MERGED.get('latency_metrics', False) %}
+filter {
+  ruby {
+    code => "event.set('[_tmp][logstash_to_redis]', Time.now().utc.iso8601(3));"
+  }
+}
+{% endif %}
 output {
 	redis {
 		host => '{{ HOST }}'
@@ -86,3 +86,8 @@ logstash:
    multiline: True
    advanced: True
    forcedType: "[]string"
+  latency_metrics:
+    description: Enable latency metrics within events processed by logstash. Useful for pinpointing log ingest delay.
+    forcedType: bool
+    global: False
+    advanced: True
@@ -1,448 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-"""
-so-config.py writes SOC/onionconfig settings to Postgres.
-
-so-yaml.py remains a YAML file editor. Call this tool when a pillar-backed
-setting also needs to be reflected in the onionconfig database.
-"""
-
-import argparse
-import json
-import os
-from pathlib import Path
-import subprocess
-import sys
-
-import yaml
-
-
-PILLAR_ROOT = Path(os.environ.get("SO_CONFIG_PILLAR_ROOT", "/opt/so/saltstack/local/pillar"))
-DOCKER_CONTAINER = os.environ.get("SO_CONFIG_PG_CONTAINER", "so-postgres")
-PG_DATABASE = os.environ.get("SO_CONFIG_PG_DATABASE", "securityonion")
-PG_USER = os.environ.get("SO_CONFIG_PG_USER", "postgres")
-DEFAULT_USER_ID = os.environ.get("SO_CONFIG_USER_ID", "so-config")
-
-EXCLUDE_BASENAMES = {
-    "secrets.sls",
-    "auth.sls",
-    "top.sls",
-}
-EXCLUDE_PATH_FRAGMENTS = (
-    "/elasticsearch/nodes.sls",
-    "/redis/nodes.sls",
-    "/kafka/nodes.sls",
-    "/hypervisor/nodes.sls",
-    "/logstash/nodes.sls",
-    "/node_data/ips.sls",
-    "/postgres/auth.sls",
-    "/elasticsearch/auth.sls",
-    "/kibana/secrets.sls",
-)
-
-
-class SkipPath(Exception):
-    pass
-
-
-def pg_str(value):
-    if value is None:
-        return "NULL"
-    return "'" + str(value).replace("'", "''") + "'"
-
-
-def pg_jsonb(value):
-    return pg_str(json.dumps(value)) + "::jsonb"
-
-
-def docker_psql(sql):
-    proc = subprocess.run(
-        ["docker", "exec", "-i", DOCKER_CONTAINER,
-         "psql", "-U", PG_USER, "-d", PG_DATABASE,
-         "-tA", "-q", "-v", "ON_ERROR_STOP=1"],
-        input=sql.encode(),
-        capture_output=True,
-        check=False,
-        timeout=60,
-    )
-    if proc.returncode != 0:
-        sys.stderr.write(proc.stderr.decode(errors="replace"))
-        raise RuntimeError(f"docker exec psql failed with rc={proc.returncode}")
-    return proc.stdout.decode(errors="replace")
-
-
-def schema_ready():
-    sql = """
-SELECT to_regclass('public.settings') IS NOT NULL
-   AND to_regclass('public.audit_settings') IS NOT NULL;
-"""
-    return docker_psql(sql).strip() == "t"
-
-
-def cmd_wait_schema(args):
-    import time
-
-    deadline = time.time() + args.timeout
-    while time.time() <= deadline:
-        if schema_ready():
-            return 0
-        time.sleep(args.interval)
-    print("so-config: onionconfig schema is not ready", file=sys.stderr)
-    return 1
-
-
-def upsert_setting(setting_id, value, *, node_id="", duplicated_from_id=None,
-                   user_id=DEFAULT_USER_ID, note=None):
-    note = note or "so-config upsert"
-    sql = f"""
-BEGIN;
-WITH old_row AS (
-    SELECT value
-      FROM settings
-     WHERE setting_id = {pg_str(setting_id)}
-       AND node_id = {pg_str(node_id)}
-     FOR UPDATE
-),
-upserted AS (
-    INSERT INTO settings (setting_id, value, duplicated_from_id, node_id)
-    VALUES ({pg_str(setting_id)}, {pg_jsonb(value)}, {pg_str(duplicated_from_id)}, {pg_str(node_id)})
-    ON CONFLICT (setting_id, node_id) DO UPDATE
-       SET value = EXCLUDED.value,
-           duplicated_from_id = EXCLUDED.duplicated_from_id
-    RETURNING value
-)
-INSERT INTO audit_settings (setting_id, node_id, user_id, old_value, new_value, note)
-SELECT {pg_str(setting_id)},
-       {pg_str(node_id)},
-       {pg_str(user_id)},
-       (SELECT value FROM old_row),
-       (SELECT value FROM upserted),
-       {pg_str(note)}
- WHERE NOT EXISTS (SELECT 1 FROM old_row)
-    OR (SELECT value FROM old_row) IS DISTINCT FROM (SELECT value FROM upserted);
-COMMIT;
-"""
-    docker_psql(sql)
-
-
-def delete_setting(setting_id, *, node_id="", user_id=DEFAULT_USER_ID, note=None):
-    note = note or "so-config delete"
-    sql = f"""
-BEGIN;
-WITH deleted AS (
-    DELETE FROM settings
-     WHERE setting_id = {pg_str(setting_id)}
-       AND node_id = {pg_str(node_id)}
-     RETURNING value
-)
-INSERT INTO audit_settings (setting_id, node_id, user_id, old_value, new_value, note)
-SELECT {pg_str(setting_id)}, {pg_str(node_id)}, {pg_str(user_id)}, value, NULL::jsonb, {pg_str(note)}
-  FROM deleted;
-COMMIT;
-"""
-    docker_psql(sql)
-
-
-def delete_setting_prefix(setting_id, *, node_id="", user_id=DEFAULT_USER_ID, note=None):
-    if not setting_id:
-        raise ValueError("setting_id prefix cannot be empty")
-    note = note or "so-config delete-prefix"
-    sql = f"""
-BEGIN;
-WITH deleted AS (
-    DELETE FROM settings
-     WHERE node_id = {pg_str(node_id)}
-       AND (
-             setting_id = {pg_str(setting_id)}
-          OR substring(setting_id from 1 for char_length({pg_str(setting_id)}) + 1) = {pg_str(setting_id + ".")}
-       )
-     RETURNING setting_id, value
-)
-INSERT INTO audit_settings (setting_id, node_id, user_id, old_value, new_value, note)
-SELECT setting_id, {pg_str(node_id)}, {pg_str(user_id)}, value, NULL::jsonb, {pg_str(note)}
-  FROM deleted;
-COMMIT;
-"""
-    docker_psql(sql)
-
-
-def purge_node(node_id, *, user_id=DEFAULT_USER_ID, note=None):
-    note = note or "so-config purge-node"
-    sql = f"""
-BEGIN;
-WITH deleted AS (
-    DELETE FROM settings
-     WHERE node_id = {pg_str(node_id)}
-     RETURNING setting_id, value
-)
-INSERT INTO audit_settings (setting_id, node_id, user_id, old_value, new_value, note)
-SELECT setting_id, {pg_str(node_id)}, {pg_str(user_id)}, value, NULL::jsonb, {pg_str(note)}
-  FROM deleted;
-COMMIT;
-"""
-    docker_psql(sql)
-
-
-def parse_value(value, value_file=None):
-    if value_file:
-        with open(value_file, "r") as fh:
-            value = fh.read()
-    parsed = yaml.safe_load(value)
-    if parsed is None and value == "":
-        return ""
-    return parsed
-
-
-def parse_yaml_file(path):
-    with open(path, "rb") as fh:
-        raw = fh.read()
-    if b"{%" in raw or b"{{" in raw:
-        raise SkipPath(f"{path}: Jinja-templated files stay disk-only")
-    if not raw.strip():
-        return {}
-    parsed = yaml.safe_load(raw)
-    return parsed if parsed is not None else {}
-
-
-def flatten(prefix, value):
-    if isinstance(value, dict):
-        for key, child in value.items():
-            child_id = f"{prefix}.{key}" if prefix else str(key)
-            yield from flatten(child_id, child)
-    else:
-        yield prefix, value
-
-
-def classify_pillar_path(path):
-    norm = Path(path).resolve()
-    norm_str = str(norm)
-
-    if norm.name in EXCLUDE_BASENAMES:
-        raise SkipPath(f"{path}: excluded basename")
-    for fragment in EXCLUDE_PATH_FRAGMENTS:
-        if fragment in norm_str:
-            raise SkipPath(f"{path}: excluded path fragment {fragment}")
-    if norm.suffix != ".sls":
-        raise SkipPath(f"{path}: not an .sls file")
-
-    parent = norm.parent.name
-    stem = norm.stem
-
-    if parent == "minions":
-        if stem.startswith("adv_"):
-            return {"kind": "advanced", "setting_id": "advanced", "node_id": stem[4:]}
-        return {"kind": "normal", "node_id": stem}
-
-    section = parent
-    if stem == f"soc_{section}":
-        return {"kind": "normal", "node_id": ""}
-    if stem == f"adv_{section}":
-        return {"kind": "advanced", "setting_id": f"{section}.advanced", "node_id": ""}
-
-    raise SkipPath(f"{path}: not a SOC-managed pillar file")
-
-
-def import_pillar_file(path, *, user_id=DEFAULT_USER_ID, note=None):
-    meta = classify_pillar_path(path)
-    note = note or f"so-config import-file {path}"
-
-    if meta["kind"] == "advanced":
-        with open(path, "r") as fh:
-            upsert_setting(meta["setting_id"], fh.read(), node_id=meta["node_id"],
-                           user_id=user_id, note=note)
-        return 1
-
-    data = parse_yaml_file(path)
-    if not isinstance(data, dict):
-        raise SkipPath(f"{path}: top-level YAML is not a map")
-
-    count = 0
-    for setting_id, value in flatten("", data):
-        upsert_setting(setting_id, value, node_id=meta["node_id"],
-                       user_id=user_id, note=note)
-        count += 1
-    return count
-
-
-def iter_pillar_files(root):
-    root = Path(root)
-    if not root.is_dir():
-        return
-    for path in sorted(root.rglob("*.sls")):
-        if path.is_file():
-            yield path
-
-
-def cmd_set(args):
-    upsert_setting(args.setting_id, parse_value(args.value, args.value_file),
-                   node_id=args.node_id,
-                   duplicated_from_id=args.duplicated_from_id,
-                   user_id=args.user_id,
-                   note=args.note)
-    return 0
-
-
-def cmd_delete(args):
-    delete_setting(args.setting_id, node_id=args.node_id,
-                   user_id=args.user_id, note=args.note)
-    return 0
-
-
-def cmd_delete_prefix(args):
-    delete_setting_prefix(args.setting_id, node_id=args.node_id,
-                          user_id=args.user_id, note=args.note)
-    return 0
-
-
-def cmd_purge_node(args):
-    purge_node(args.node_id, user_id=args.user_id, note=args.note)
-    return 0
-
-
-def cmd_import_file(args):
-    count = import_pillar_file(args.path, user_id=args.user_id, note=args.note)
-    print(f"imported {count} settings from {args.path}")
-    return 0
-
-
-def cmd_import_minion(args):
-    count = 0
-    for name in (f"{args.node_id}.sls", f"adv_{args.node_id}.sls"):
-        path = PILLAR_ROOT / "minions" / name
-        if path.exists():
-            count += import_pillar_file(path, user_id=args.user_id, note=args.note)
-    print(f"imported {count} settings for node {args.node_id}")
-    return 0
-
-
-def cmd_import_all(args):
-    count = 0
-    skipped = 0
-    for path in iter_pillar_files(args.root):
-        try:
-            count += import_pillar_file(path, user_id=args.user_id, note=args.note)
-        except SkipPath as exc:
-            skipped += 1
-            if args.verbose:
-                print(f"skip: {exc}", file=sys.stderr)
-    print(f"imported {count} settings, skipped {skipped} files")
-    if args.state_file:
-        with open(args.state_file, "w") as fh:
-            fh.write("ok\n")
-    return 0
-
-
-def cmd_sync_yaml_mutation(args):
-    meta = classify_pillar_path(args.path)
-    note = args.note or f"so-config sync-yaml-mutation {args.operation} {args.path}"
-
-    if meta["kind"] == "advanced":
-        import_pillar_file(args.path, user_id=args.user_id, note=note)
-        return 0
-
-    if args.operation in ("add", "replace"):
-        upsert_setting(args.key, parse_value(args.value, args.value_file),
-                       node_id=meta["node_id"],
-                       user_id=args.user_id,
-                       note=note)
-    elif args.operation == "remove":
-        delete_setting_prefix(args.key, node_id=meta["node_id"],
-                              user_id=args.user_id, note=note)
-    else:
-        raise ValueError(f"unsupported operation: {args.operation}")
-    return 0
-
-
-def build_parser():
-    parser = argparse.ArgumentParser(description=__doc__)
-    sub = parser.add_subparsers(dest="command", required=True)
-
-    p = sub.add_parser("wait-schema", help="wait for SOC-created onionconfig tables")
-    p.add_argument("--timeout", type=int, default=120)
-    p.add_argument("--interval", type=int, default=2)
-    p.set_defaults(func=cmd_wait_schema)
-
-    p = sub.add_parser("set", help="upsert one setting")
-    p.add_argument("setting_id")
-    p.add_argument("value", nargs="?", default="")
-    p.add_argument("--value-file")
-    p.add_argument("--node-id", default="")
-    p.add_argument("--duplicated-from-id")
-    p.add_argument("--user-id", default=DEFAULT_USER_ID)
-    p.add_argument("--note")
-    p.set_defaults(func=cmd_set)
-
-    p = sub.add_parser("delete", help="delete one setting")
-    p.add_argument("setting_id")
-    p.add_argument("--node-id", default="")
-    p.add_argument("--user-id", default=DEFAULT_USER_ID)
-    p.add_argument("--note")
-    p.set_defaults(func=cmd_delete)
-
-    p = sub.add_parser("delete-prefix", help="delete one setting and all child settings")
-    p.add_argument("setting_id")
-    p.add_argument("--node-id", default="")
-    p.add_argument("--user-id", default=DEFAULT_USER_ID)
-    p.add_argument("--note")
-    p.set_defaults(func=cmd_delete_prefix)
-
-    p = sub.add_parser("purge-node", help="delete all settings for one node")
-    p.add_argument("node_id")
-    p.add_argument("--user-id", default=DEFAULT_USER_ID)
-    p.add_argument("--note")
-    p.set_defaults(func=cmd_purge_node)
-
-    p = sub.add_parser("import-file", help="import one SOC-managed pillar file")
-    p.add_argument("path")
-    p.add_argument("--user-id", default=DEFAULT_USER_ID)
-    p.add_argument("--note")
-    p.set_defaults(func=cmd_import_file)
-
-    p = sub.add_parser("import-minion", help="import one minion's pillar files")
-    p.add_argument("node_id")
-    p.add_argument("--user-id", default=DEFAULT_USER_ID)
-    p.add_argument("--note")
-    p.set_defaults(func=cmd_import_minion)
-
-    p = sub.add_parser("import-all", help="import all SOC-managed local pillar files")
-    p.add_argument("--root", default=str(PILLAR_ROOT))
-    p.add_argument("--state-file")
-    p.add_argument("--user-id", default=DEFAULT_USER_ID)
-    p.add_argument("--note", default="so-config initial import")
-    p.add_argument("--verbose", action="store_true")
-    p.set_defaults(func=cmd_import_all)
-
-    p = sub.add_parser("sync-yaml-mutation",
-                       help="mirror one so-yaml add/replace/remove mutation to onionconfig")
-    p.add_argument("path")
-    p.add_argument("operation", choices=("add", "replace", "remove"))
-    p.add_argument("key")
-    p.add_argument("value", nargs="?", default="")
-    p.add_argument("--value-file")
-    p.add_argument("--user-id", default=DEFAULT_USER_ID)
-    p.add_argument("--note")
-    p.set_defaults(func=cmd_sync_yaml_mutation)
-
-    return parser
-
-
-def main(argv):
-    parser = build_parser()
-    args = parser.parse_args(argv)
-    try:
-        return args.func(args)
-    except SkipPath as exc:
-        print(f"skip: {exc}", file=sys.stderr)
-        return 2
-    except Exception as exc:
-        print(f"so-config: {exc}", file=sys.stderr)
-        return 1
-
-
-if __name__ == "__main__":
-    sys.exit(main(sys.argv[1:]))
@@ -1,178 +0,0 @@
-import importlib
-import os
-import tempfile
-import unittest
-from unittest.mock import patch
-
-
-soconfig = importlib.import_module("so-config")
-
-
-class TestSoConfigPathMapping(unittest.TestCase):
-
-    def test_classify_global_soc(self):
-        meta = soconfig.classify_pillar_path(
-            "/opt/so/saltstack/local/pillar/soc/soc_soc.sls")
-        self.assertEqual(meta["kind"], "normal")
-        self.assertEqual(meta["node_id"], "")
-
-    def test_classify_global_advanced(self):
-        meta = soconfig.classify_pillar_path(
-            "/opt/so/saltstack/local/pillar/soc/adv_soc.sls")
-        self.assertEqual(meta["kind"], "advanced")
-        self.assertEqual(meta["setting_id"], "soc.advanced")
-        self.assertEqual(meta["node_id"], "")
-
-    def test_classify_minion(self):
-        meta = soconfig.classify_pillar_path(
-            "/opt/so/saltstack/local/pillar/minions/h1_sensor.sls")
-        self.assertEqual(meta["kind"], "normal")
-        self.assertEqual(meta["node_id"], "h1_sensor")
-
-    def test_classify_minion_advanced(self):
-        meta = soconfig.classify_pillar_path(
-            "/opt/so/saltstack/local/pillar/minions/adv_h1_sensor.sls")
-        self.assertEqual(meta["kind"], "advanced")
-        self.assertEqual(meta["setting_id"], "advanced")
-        self.assertEqual(meta["node_id"], "h1_sensor")
-
-    def test_classify_skips_bootstrap(self):
-        with self.assertRaises(soconfig.SkipPath):
-            soconfig.classify_pillar_path(
-                "/opt/so/saltstack/local/pillar/secrets.sls")
-
-
-class TestSoConfigImport(unittest.TestCase):
-
-    def test_flatten_keeps_lists_as_values(self):
-        flattened = dict(soconfig.flatten("", {
-            "host": {"mainip": "10.0.0.1"},
-            "suricata": {"pcap": {"enabled": True}},
-            "items": ["a", "b"],
-        }))
-        self.assertEqual(flattened["host.mainip"], "10.0.0.1")
-        self.assertEqual(flattened["suricata.pcap.enabled"], True)
-        self.assertEqual(flattened["items"], ["a", "b"])
-
-    def test_import_file_upserts_flattened_settings(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            path = os.path.join(tmp, "h1_sensor.sls")
-            minions = os.path.join(tmp, "minions")
-            os.mkdir(minions)
-            path = os.path.join(minions, "h1_sensor.sls")
-            with open(path, "w") as fh:
-                fh.write("host:\n  mainip: 10.0.0.1\nsuricata:\n  enabled: true\n")
-
-            calls = []
-            with patch.object(soconfig, "upsert_setting",
-                              side_effect=lambda *args, **kwargs: calls.append((args, kwargs))):
-                count = soconfig.import_pillar_file(path)
-
-        self.assertEqual(count, 2)
-        self.assertIn((("host.mainip", "10.0.0.1"), {"node_id": "h1_sensor", "user_id": "so-config", "note": f"so-config import-file {path}"}), calls)
-        self.assertIn((("suricata.enabled", True), {"node_id": "h1_sensor", "user_id": "so-config", "note": f"so-config import-file {path}"}), calls)
-
-    def test_import_advanced_file_upserts_raw_content(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            minions = os.path.join(tmp, "minions")
-            os.mkdir(minions)
-            path = os.path.join(minions, "adv_h1_sensor.sls")
-            with open(path, "w") as fh:
-                fh.write("custom:\n  raw: true\n")
-
-            calls = []
-            with patch.object(soconfig, "upsert_setting",
-                              side_effect=lambda *args, **kwargs: calls.append((args, kwargs))):
-                count = soconfig.import_pillar_file(path)
-
-        self.assertEqual(count, 1)
-        self.assertEqual(calls[0][0], ("advanced", "custom:\n  raw: true\n"))
-        self.assertEqual(calls[0][1]["node_id"], "h1_sensor")
-
-
-class TestSoConfigSql(unittest.TestCase):
-
-    def test_schema_ready_checks_soc_tables(self):
-        captured = {}
-        with patch.object(soconfig, "docker_psql",
-                          side_effect=lambda sql: captured.update({"sql": sql}) or "t\n"):
-            ready = soconfig.schema_ready()
-
-        self.assertTrue(ready)
-        self.assertIn("to_regclass('public.settings')", captured["sql"])
-        self.assertIn("to_regclass('public.audit_settings')", captured["sql"])
-
-    def test_set_writes_settings_and_audit(self):
-        captured = {}
-        with patch.object(soconfig, "docker_psql",
-                          side_effect=lambda sql: captured.setdefault("sql", sql)):
-            soconfig.upsert_setting("host.mainip", "10.0.0.1",
-                                    node_id="h1_sensor", user_id="tester", note="unit")
-
-        self.assertIn("INSERT INTO settings", captured["sql"])
-        self.assertIn("INSERT INTO audit_settings", captured["sql"])
-        self.assertIn("'host.mainip'", captured["sql"])
-        self.assertIn("'h1_sensor'", captured["sql"])
-        self.assertIn("'tester'", captured["sql"])
-
-    def test_purge_node_audits_deleted_rows(self):
-        captured = {}
-        with patch.object(soconfig, "docker_psql",
-                          side_effect=lambda sql: captured.setdefault("sql", sql)):
-            soconfig.purge_node("h1_sensor", user_id="tester", note="unit")
-
-        self.assertIn("DELETE FROM settings", captured["sql"])
-        self.assertIn("WHERE node_id = 'h1_sensor'", captured["sql"])
-        self.assertIn("INSERT INTO audit_settings", captured["sql"])
-
-    def test_delete_prefix_removes_children_and_audits(self):
-        captured = {}
-        with patch.object(soconfig, "docker_psql",
-                          side_effect=lambda sql: captured.setdefault("sql", sql)):
-            soconfig.delete_setting_prefix("elasticfleet", node_id="h1_sensor",
-                                           user_id="tester", note="unit")
-
-        self.assertIn("DELETE FROM settings", captured["sql"])
-        self.assertIn("setting_id = 'elasticfleet'", captured["sql"])
-        self.assertIn("'elasticfleet.'", captured["sql"])
-        self.assertIn("INSERT INTO audit_settings", captured["sql"])
-
-    def test_sync_yaml_replace_uses_path_node_id(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            minions = os.path.join(tmp, "minions")
-            os.mkdir(minions)
-            path = os.path.join(minions, "h1_sensor.sls")
-            open(path, "w").close()
-
-            calls = []
-            args = soconfig.build_parser().parse_args([
-                "sync-yaml-mutation", path, "replace", "suricata.enabled", "true"
-            ])
-            with patch.object(soconfig, "upsert_setting",
-                              side_effect=lambda *a, **kw: calls.append((a, kw))):
-                soconfig.cmd_sync_yaml_mutation(args)
-
-        self.assertEqual(calls[0][0], ("suricata.enabled", True))
-        self.assertEqual(calls[0][1]["node_id"], "h1_sensor")
-
-    def test_sync_yaml_remove_deletes_prefix(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            minions = os.path.join(tmp, "minions")
-            os.mkdir(minions)
-            path = os.path.join(minions, "h1_sensor.sls")
-            open(path, "w").close()
-
-            calls = []
-            args = soconfig.build_parser().parse_args([
-                "sync-yaml-mutation", path, "remove", "elasticfleet"
-            ])
-            with patch.object(soconfig, "delete_setting_prefix",
-                              side_effect=lambda *a, **kw: calls.append((a, kw))):
-                soconfig.cmd_sync_yaml_mutation(args)
-
-        self.assertEqual(calls[0][0], ("elasticfleet",))
-        self.assertEqual(calls[0][1]["node_id"], "h1_sensor")
-
-
-if __name__ == "__main__":
-    unittest.main()
@@ -0,0 +1,381 @@
+#!/usr/bin/env python3
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+# Imports detection overrides (e.g. from so-detections-backup) into the so-detection
+# index. Reads <publicId>.<ext> files (NDJSON, one override per line) from a source
+# directory, looks up the matching detection by publicId+engine, validates each
+# override against the same rules SOC enforces, dedupes against existing overrides
+# (operational fields only), and appends new ones.
+
+import argparse
+import ipaddress
+import json
+import os
+import re
+import sys
+from datetime import datetime
+
+import requests
+from requests.auth import HTTPBasicAuth
+import urllib3
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+DEFAULT_INDEX = "so-detection"
+AUTH_FILE = "/opt/so/conf/elasticsearch/curl.config"
+ES_URL = "https://localhost:9200"
+
+# Engines we know how to handle and the file extension the backup script writes.
+ENGINES = {
+    "suricata": "txt",
+}
+
+# Standard Suricata variables that ship with Security Onion. Anything else
+# referenced in an override is "custom" and the user needs to make sure it
+# exists in SOC Config before the override will function.
+BUILTIN_SURICATA_VARS = {
+    "$HOME_NET", "$EXTERNAL_NET",
+    "$HTTP_SERVERS", "$DNS_SERVERS", "$SQL_SERVERS", "$SMTP_SERVERS",
+    "$TELNET_SERVERS", "$AIM_SERVERS", "$DC_SERVERS", "$MODBUS_SERVER",
+    "$MODBUS_CLIENT", "$ENIP_CLIENT", "$ENIP_SERVER",
+    "$HTTP_PORTS", "$SHELLCODE_PORTS", "$ORACLE_PORTS", "$SSH_PORTS",
+    "$FTP_PORTS", "$FILE_DATA_PORTS",
+}
+
+VAR_PATTERN = re.compile(r"\$[A-Z_][A-Z0-9_]*")
+
+# Canonical valid values, per securityonion-soc/model/detection.go.
+SURICATA_OVERRIDE_TYPES = {"suppress", "threshold", "modify"}
+SUPPRESS_TRACKS = {"by_src", "by_dst", "by_either"}
+THRESHOLD_TRACKS = {"by_src", "by_dst", "by_both"}
+THRESHOLD_TYPES = {"limit", "threshold", "both"}
+
+STALE_WARNING = """\
+WARNING: so-detections-backup does not remove backup files when overrides are
+deleted via the Security Onion web UI. As a result, files in the source
+directory may represent overrides that were intentionally deleted and should
+NOT be re-imported.
+
+Before continuing, verify that the source directory reflects the overrides you
+actually want imported. Remove any files corresponding to overrides you previously deleted.
+"""
+
+
+def make_session(auth_file):
+    with open(auth_file, "r") as f:
+        for line in f:
+            if line.startswith("user ="):
+                creds = line.split("=", 1)[1].strip().replace('"', "")
+                user, _, password = creds.partition(":")
+                session = requests.Session()
+                session.auth = HTTPBasicAuth(user, password)
+                session.headers.update({"Content-Type": "application/json"})
+                session.verify = False
+                return session
+    raise RuntimeError(f"Could not find 'user =' line in {auth_file}")
+
+
+def find_detection(session, index, public_id, engine):
+    query = {
+        "query": {"bool": {"must": [
+            {"term": {"so_detection.publicId": public_id}},
+            {"term": {"so_detection.engine": engine}},
+        ]}},
+        "size": 2,
+    }
+    r = session.get(f"{ES_URL}/{index}/_search", json=query)
+    r.raise_for_status()
+    hits = r.json().get("hits", {}).get("hits", [])
+    if not hits:
+        return None, None, None
+    if len(hits) > 1:
+        # Shouldn't happen — publicId is unique per engine — but flag it.
+        print(f"  WARN: {len(hits)} detections matched publicId={public_id} engine={engine}; using first")
+    hit = hits[0]
+    existing = hit["_source"].get("so_detection", {}).get("overrides") or []
+    return hit["_id"], hit["_index"], existing
+
+
+def update_overrides(session, doc_index, doc_id, overrides):
+    body = {"doc": {"so_detection": {"overrides": overrides}}}
+    r = session.post(f"{ES_URL}/{doc_index}/_update/{doc_id}", json=body)
+    r.raise_for_status()
+    return r.json()
+
+
+def dedupe_key(override):
+    """Operational fields only, per Override.Equal() in detection.go.
+    Excludes timestamps and isEnabled so re-imports don't appear unique."""
+    t = override.get("type")
+    if t == "suppress":
+        return (t, override.get("track"), override.get("ip"))
+    if t == "threshold":
+        return (t, override.get("thresholdType"), override.get("track"),
+                override.get("count"), override.get("seconds"))
+    if t == "modify":
+        return (t, override.get("regex"), override.get("value"))
+
+
+def _validate_suricata_ip(ip):
+    if not ip:
+        return "ip cannot be empty"
+    if ip.startswith("$"):
+        return None
+    if ip.startswith("[") and ip.endswith("]"):
+        for part in ip[1:-1].split(","):
+            err = _validate_single_ip(part.strip())
+            if err:
+                return f"invalid IP in list: {err}"
+        return None
+    return _validate_single_ip(ip)
+
+
+def _validate_single_ip(ip):
+    try:
+        if "/" in ip:
+            ipaddress.ip_network(ip, strict=False)
+        else:
+            ipaddress.ip_address(ip)
+    except ValueError:
+        return f"invalid IP/CIDR {ip!r}"
+    return None
+
+
+def validate_override(override, engine):
+    """Mirror Override.Validate() from securityonion-soc/model/detection.go.
+    Returns None on success, an error string otherwise."""
+    t = override.get("type")
+    if not t:
+        return "override type is required"
+    if t not in SURICATA_OVERRIDE_TYPES:
+        return f"invalid type {t!r}: must be one of {sorted(SURICATA_OVERRIDE_TYPES)}"
+
+    has = {k: override.get(k) is not None for k in
+           ("regex", "value", "thresholdType", "track", "ip", "count", "seconds", "customFilter")}
+
+    if t == "suppress":
+        if not has["ip"] or not has["track"]:
+            return "suppress requires 'ip' and 'track'"
+        if any(has[k] for k in ("regex", "value", "thresholdType", "count", "seconds", "customFilter")):
+            return "suppress has unnecessary fields"
+        if override["track"] not in SUPPRESS_TRACKS:
+            return f"invalid track {override['track']!r}: must be one of {sorted(SUPPRESS_TRACKS)}"
+        return _validate_suricata_ip(override["ip"])
+
+    if t == "threshold":
+        if not all(has[k] for k in ("thresholdType", "track", "count", "seconds")):
+            return "threshold requires 'thresholdType', 'track', 'count', 'seconds'"
+        if any(has[k] for k in ("regex", "value", "customFilter")):
+            return "threshold has unnecessary fields"
+        if override["thresholdType"] not in THRESHOLD_TYPES:
+            return f"invalid thresholdType {override['thresholdType']!r}: must be one of {sorted(THRESHOLD_TYPES)}"
+        if override["track"] not in THRESHOLD_TRACKS:
+            return f"invalid track {override['track']!r}: must be one of {sorted(THRESHOLD_TRACKS)}"
+        if not isinstance(override["count"], int) or override["count"] <= 0:
+            return f"count must be a positive integer, got {override['count']!r}"
+        if not isinstance(override["seconds"], int) or override["seconds"] <= 0:
+            return f"seconds must be a positive integer, got {override['seconds']!r}"
+        return None
+
+    if t == "modify":
+        if not has["regex"] or not has["value"]:
+            return "modify requires 'regex' and 'value'"
+        if any(has[k] for k in ("thresholdType", "track", "count", "seconds", "customFilter")):
+            return "modify has unnecessary fields"
+        try:
+            re.compile(override["regex"])
+        except re.error as e:
+            return f"invalid regex: {e}"
+        return None
+
+
+def parse_overrides_file(path):
+    """Parse a file written by so-detections-backup.py: NDJSON, one override
+    per line. Returns a list of (override_dict, line_number)."""
+    overrides = []
+    with open(path, "r") as f:
+        for i, line in enumerate(f, start=1):
+            line = line.strip()
+            if not line:
+                continue
+            overrides.append((json.loads(line), i))
+    return overrides
+
+
+def describe(override):
+    """Human-readable summary of the operational fields for a given override type."""
+    t = override.get("type")
+    if t == "suppress":
+        return f"type=suppress track={override.get('track')} ip={override.get('ip')}"
+    if t == "threshold":
+        return (f"type=threshold track={override.get('track')} "
+                f"thresholdType={override.get('thresholdType')} "
+                f"count={override.get('count')} seconds={override.get('seconds')}")
+    if t == "modify":
+        return f"type=modify regex={override.get('regex')!r}"
+
+
+def collect_custom_vars(override):
+    found = set()
+    for value in override.values():
+        if isinstance(value, str):
+            for match in VAR_PATTERN.findall(value):
+                if match not in BUILTIN_SURICATA_VARS:
+                    found.add(match)
+    return found
+
+
+def parse_args():
+    p = argparse.ArgumentParser(
+        description="Import detection overrides into the so-detection index.",
+    )
+    p.add_argument("--source", "-s", required=True,
+                   help="Source directory containing <publicId>.<ext> override files.")
+    p.add_argument("--engine", "-e", default="suricata", choices=list(ENGINES.keys()),
+                   help="Detection engine (default: suricata).")
+    p.add_argument("--dry-run", "-n", action="store_true",
+                   help="Print what would happen without writing to Elasticsearch.")
+    p.add_argument("--no-import-note", action="store_true",
+                   help="Do not prepend '[Imported YYYY-MM-DD] ' to the override note.")
+    p.add_argument("--index", "-i", default=DEFAULT_INDEX,
+                   help=f"Elasticsearch index to update (default: {DEFAULT_INDEX}).")
+    return p.parse_args()
+
+
+def confirm_proceed(args):
+    """Show the stale-backup warning. Dry-run prints it and continues. Real
+    runs require the user typing 'yes' at the prompt."""
+    print(STALE_WARNING)
+    if args.dry_run:
+        print("(dry-run: no acknowledgement required)\n")
+        return True
+    answer = input("Type 'yes' to acknowledge and continue: ").strip().lower()
+    print()
+    return answer == "yes"
+
+
+def main():
+    args = parse_args()
+
+    if not os.path.isdir(args.source):
+        print(f"ERROR: source directory not found: {args.source}", file=sys.stderr)
+        sys.exit(1)
+
+    extension = ENGINES[args.engine]
+    files = sorted(f for f in os.listdir(args.source) if f.endswith(f".{extension}"))
+    if not files:
+        print(f"No *.{extension} files found in {args.source}")
+        sys.exit(0)
+
+    if not confirm_proceed(args):
+        print("Aborted.")
+        sys.exit(1)
+
+    session = make_session(AUTH_FILE)
+    today = datetime.now().strftime("%Y-%m-%d")
+    note_prefix = "" if args.no_import_note else f"[Imported {today}] "
+
+    counts = {"added": 0, "skipped_dedupe": 0, "skipped_not_found": 0, "invalid": 0, "error": 0}
+    custom_vars = set()
+
+    mode = "DRY-RUN" if args.dry_run else "IMPORT"
+    print(f"[{mode}] engine={args.engine} source={args.source} index={args.index}\n")
+
+    for filename in files:
+        public_id = os.path.splitext(filename)[0]
+        path = os.path.join(args.source, filename)
+        print(f"{public_id}:")
+
+        try:
+            new_overrides = parse_overrides_file(path)
+        except (json.JSONDecodeError, OSError) as e:
+            print(f"  ERROR: could not parse {filename}: {e}")
+            counts["error"] += 1
+            continue
+
+        if not new_overrides:
+            print("  SKIP: empty file")
+            continue
+
+        try:
+            doc_id, doc_index, existing = find_detection(session, args.index, public_id, args.engine)
+        except requests.HTTPError as e:
+            print(f"  ERROR: search failed: {e}")
+            counts["error"] += 1
+            continue
+
+        if doc_id is None:
+            print(f"  WARN: no detection found for publicId={public_id} engine={args.engine}; skipping")
+            counts["skipped_not_found"] += len(new_overrides)
+            continue
+
+        existing_keys = {dedupe_key(o) for o in existing}
+        merged = list(existing)
+        added_this_file = 0
+
+        for override, line_no in new_overrides:
+            err = validate_override(override, args.engine)
+            if err:
+                print(f"  INVALID (line {line_no}): {err}")
+                counts["invalid"] += 1
+                continue
+
+            custom_vars.update(collect_custom_vars(override))
+            key = dedupe_key(override)
+            if key in existing_keys:
+                print(f"  SKIP (line {line_no}): duplicate of existing override [{describe(override)}]")
+                counts["skipped_dedupe"] += 1
+                continue
+
+            if note_prefix:
+                override = dict(override)
+                override["note"] = note_prefix + (override.get("note") or "")
+
+            merged.append(override)
+            existing_keys.add(key)
+            added_this_file += 1
+            print(f"  ADD (line {line_no}): {describe(override)}")
+
+        if added_this_file == 0:
+            continue
+
+        if args.dry_run:
+            print(f"  DRY-RUN: would update {doc_index}/{doc_id} "
+                  f"({len(existing)} existing → {len(merged)} total)")
+            counts["added"] += added_this_file
+            continue
+
+        try:
+            update_overrides(session, doc_index, doc_id, merged)
+            print(f"  UPDATED {doc_index}/{doc_id} ({len(existing)} → {len(merged)})")
+            counts["added"] += added_this_file
+        except requests.HTTPError as e:
+            print(f"  ERROR: update failed: {e}")
+            counts["error"] += 1
+
+    print()
+    print("=" * 60)
+    print(f"Summary ({mode}):")
+    print(f"  Overrides added:           {counts['added']}")
+    print(f"  Skipped (already present): {counts['skipped_dedupe']}")
+    print(f"  Skipped (no detection):    {counts['skipped_not_found']}")
+    print(f"  Invalid (failed checks):   {counts['invalid']}")
+    print(f"  Errors:                    {counts['error']}")
+
+    if custom_vars:
+        print()
+        print("WARNING: detected custom Suricata variables in imported overrides:")
+        for v in sorted(custom_vars):
+            print(f"  {v}")
+        print("If any of these are not already defined in SOC Config (Suricata variables),")
+        print("you must add them manually before the rules will function correctly.")
+
+    sys.exit(0 if counts["error"] == 0 and counts["invalid"] == 0 else 1)
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,588 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+import importlib.util
+import json
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+from importlib.machinery import SourceFileLoader
+from io import StringIO
+from unittest.mock import MagicMock, patch
+
+import requests
+
+# The script has no .py extension; spec_from_file_location can't auto-detect a
+# loader, so we hand it a SourceFileLoader explicitly. (load_module() is
+# deprecated in 3.14 and slated for removal in 3.15.)
+HERE = os.path.dirname(os.path.abspath(__file__))
+SCRIPT = os.path.join(HERE, "so-detections-overrides-import")
+_loader = SourceFileLoader("so_overrides_import", SCRIPT)
+_spec = importlib.util.spec_from_loader("so_overrides_import", _loader)
+soi = importlib.util.module_from_spec(_spec)
+_loader.exec_module(soi)
+
+
+class TestValidateSuppress(unittest.TestCase):
+    def test_valid(self):
+        self.assertIsNone(soi.validate_override(
+            {"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}, "suricata"))
+
+    def test_valid_var(self):
+        self.assertIsNone(soi.validate_override(
+            {"type": "suppress", "track": "by_either", "ip": "$HOME_NET"}, "suricata"))
+
+    def test_valid_cidr(self):
+        self.assertIsNone(soi.validate_override(
+            {"type": "suppress", "track": "by_dst", "ip": "10.0.0.0/8"}, "suricata"))
+
+    def test_valid_bracket_list(self):
+        self.assertIsNone(soi.validate_override(
+            {"type": "suppress", "track": "by_src", "ip": "[1.2.3.4,10.0.0.0/8]"}, "suricata"))
+
+    def test_missing_ip(self):
+        err = soi.validate_override({"type": "suppress", "track": "by_src"}, "suricata")
+        self.assertIn("requires", err)
+
+    def test_missing_track(self):
+        err = soi.validate_override({"type": "suppress", "ip": "1.2.3.4"}, "suricata")
+        self.assertIn("requires", err)
+
+    def test_invalid_track(self):
+        err = soi.validate_override(
+            {"type": "suppress", "track": "by_both", "ip": "1.2.3.4"}, "suricata")
+        self.assertIn("invalid track", err)
+
+    def test_invalid_ip(self):
+        err = soi.validate_override(
+            {"type": "suppress", "track": "by_src", "ip": "not-an-ip"}, "suricata")
+        self.assertIn("invalid IP", err)
+
+    def test_unnecessary_field(self):
+        err = soi.validate_override(
+            {"type": "suppress", "track": "by_src", "ip": "1.2.3.4", "count": 5}, "suricata")
+        self.assertIn("unnecessary fields", err)
+
+
+class TestValidateThreshold(unittest.TestCase):
+    def test_valid(self):
+        self.assertIsNone(soi.validate_override({
+            "type": "threshold", "track": "by_src",
+            "thresholdType": "limit", "count": 10, "seconds": 60,
+        }, "suricata"))
+
+    def test_valid_by_both(self):
+        self.assertIsNone(soi.validate_override({
+            "type": "threshold", "track": "by_both",
+            "thresholdType": "both", "count": 1, "seconds": 1,
+        }, "suricata"))
+
+    def test_track_by_either_invalid(self):
+        err = soi.validate_override({
+            "type": "threshold", "track": "by_either",
+            "thresholdType": "limit", "count": 10, "seconds": 60,
+        }, "suricata")
+        self.assertIn("invalid track", err)
+
+    def test_invalid_threshold_type(self):
+        err = soi.validate_override({
+            "type": "threshold", "track": "by_src",
+            "thresholdType": "bogus", "count": 10, "seconds": 60,
+        }, "suricata")
+        self.assertIn("invalid thresholdType", err)
+
+    def test_zero_count(self):
+        err = soi.validate_override({
+            "type": "threshold", "track": "by_src",
+            "thresholdType": "limit", "count": 0, "seconds": 60,
+        }, "suricata")
+        self.assertIn("count", err)
+
+    def test_negative_seconds(self):
+        err = soi.validate_override({
+            "type": "threshold", "track": "by_src",
+            "thresholdType": "limit", "count": 10, "seconds": -1,
+        }, "suricata")
+        self.assertIn("seconds", err)
+
+    def test_missing_field(self):
+        err = soi.validate_override({
+            "type": "threshold", "track": "by_src",
+            "thresholdType": "limit", "count": 10,  # missing seconds
+        }, "suricata")
+        self.assertIn("requires", err)
+
+    def test_unnecessary_field(self):
+        err = soi.validate_override({
+            "type": "threshold", "track": "by_src",
+            "thresholdType": "limit", "count": 10, "seconds": 60,
+            "regex": "foo",
+        }, "suricata")
+        self.assertIn("unnecessary fields", err)
+
+
+class TestValidateModify(unittest.TestCase):
+    def test_valid(self):
+        self.assertIsNone(soi.validate_override(
+            {"type": "modify", "regex": r"content:\"foo\"", "value": "content:bar"}, "suricata"))
+
+    def test_invalid_regex(self):
+        err = soi.validate_override(
+            {"type": "modify", "regex": "(unbalanced", "value": "x"}, "suricata")
+        self.assertIn("invalid regex", err)
+
+    def test_missing_value(self):
+        err = soi.validate_override({"type": "modify", "regex": "x"}, "suricata")
+        self.assertIn("requires", err)
+
+    def test_unnecessary_field(self):
+        err = soi.validate_override(
+            {"type": "modify", "regex": "x", "value": "y", "track": "by_src"}, "suricata")
+        self.assertIn("unnecessary fields", err)
+
+
+class TestValidateMisc(unittest.TestCase):
+    def test_unknown_type(self):
+        err = soi.validate_override({"type": "suppresss", "track": "by_src", "ip": "1.2.3.4"}, "suricata")
+        self.assertIn("invalid type", err)
+
+    def test_missing_type(self):
+        err = soi.validate_override({"track": "by_src"}, "suricata")
+        self.assertIn("type is required", err)
+
+
+class TestValidateIP(unittest.TestCase):
+    def test_plain_ipv4(self):
+        self.assertIsNone(soi._validate_suricata_ip("1.2.3.4"))
+
+    def test_plain_ipv6(self):
+        self.assertIsNone(soi._validate_suricata_ip("::1"))
+
+    def test_cidr(self):
+        self.assertIsNone(soi._validate_suricata_ip("10.0.0.0/8"))
+
+    def test_var(self):
+        self.assertIsNone(soi._validate_suricata_ip("$CONCOURSEWORKERS"))
+
+    def test_bracket_list(self):
+        self.assertIsNone(soi._validate_suricata_ip("[1.2.3.4, 10.0.0.0/8]"))
+
+    def test_bracket_list_bad_member(self):
+        err = soi._validate_suricata_ip("[1.2.3.4,nope]")
+        self.assertIn("invalid IP in list", err)
+
+    def test_empty(self):
+        self.assertIn("empty", soi._validate_suricata_ip(""))
+
+    def test_invalid(self):
+        self.assertIn("invalid", soi._validate_suricata_ip("999.999.999.999"))
+
+
+class TestDedupeKey(unittest.TestCase):
+    def test_suppress(self):
+        a = {"type": "suppress", "track": "by_src", "ip": "1.2.3.4", "count": 99}
+        b = {"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}
+        # count is irrelevant for suppress dedupe
+        self.assertEqual(soi.dedupe_key(a), soi.dedupe_key(b))
+
+    def test_suppress_differs_on_ip(self):
+        a = {"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}
+        b = {"type": "suppress", "track": "by_src", "ip": "5.6.7.8"}
+        self.assertNotEqual(soi.dedupe_key(a), soi.dedupe_key(b))
+
+    def test_threshold(self):
+        a = {"type": "threshold", "track": "by_src", "thresholdType": "limit",
+             "count": 10, "seconds": 60, "ip": "ignored"}
+        b = {"type": "threshold", "track": "by_src", "thresholdType": "limit",
+             "count": 10, "seconds": 60}
+        self.assertEqual(soi.dedupe_key(a), soi.dedupe_key(b))
+
+    def test_threshold_differs_on_count(self):
+        a = {"type": "threshold", "track": "by_src", "thresholdType": "limit",
+             "count": 10, "seconds": 60}
+        b = {"type": "threshold", "track": "by_src", "thresholdType": "limit",
+             "count": 20, "seconds": 60}
+        self.assertNotEqual(soi.dedupe_key(a), soi.dedupe_key(b))
+
+    def test_modify(self):
+        a = {"type": "modify", "regex": "x", "value": "y"}
+        b = {"type": "modify", "regex": "x", "value": "y"}
+        self.assertEqual(soi.dedupe_key(a), soi.dedupe_key(b))
+
+
+class TestDescribe(unittest.TestCase):
+    def test_suppress(self):
+        s = soi.describe({"type": "suppress", "track": "by_src", "ip": "1.2.3.4"})
+        self.assertIn("suppress", s)
+        self.assertIn("by_src", s)
+        self.assertIn("1.2.3.4", s)
+
+    def test_threshold_includes_count(self):
+        s = soi.describe({"type": "threshold", "track": "by_src",
+                          "thresholdType": "limit", "count": 10, "seconds": 60})
+        self.assertIn("count=10", s)
+        self.assertIn("seconds=60", s)
+
+    def test_modify(self):
+        s = soi.describe({"type": "modify", "regex": "foo"})
+        self.assertIn("modify", s)
+        self.assertIn("foo", s)
+
+
+class TestParseOverridesFile(unittest.TestCase):
+    def _write(self, content):
+        fd, path = tempfile.mkstemp(suffix=".txt")
+        os.close(fd)
+        with open(path, "w") as f:
+            f.write(content)
+        self.addCleanup(os.unlink, path)
+        return path
+
+    def test_single_line(self):
+        path = self._write('{"type":"suppress","track":"by_src","ip":"1.2.3.4"}')
+        result = soi.parse_overrides_file(path)
+        self.assertEqual(len(result), 1)
+        self.assertEqual(result[0][0]["type"], "suppress")
+        self.assertEqual(result[0][1], 1)
+
+    def test_ndjson(self):
+        path = self._write(
+            '{"type":"suppress","track":"by_src","ip":"1.2.3.4"}\n'
+            '{"type":"suppress","track":"by_dst","ip":"5.6.7.8"}\n'
+        )
+        result = soi.parse_overrides_file(path)
+        self.assertEqual(len(result), 2)
+        self.assertEqual(result[1][1], 2)
+
+    def test_empty(self):
+        path = self._write("")
+        self.assertEqual(soi.parse_overrides_file(path), [])
+
+    def test_blank_lines_skipped(self):
+        path = self._write('\n{"type":"suppress","track":"by_src","ip":"1.2.3.4"}\n\n')
+        result = soi.parse_overrides_file(path)
+        self.assertEqual(len(result), 1)
+        self.assertEqual(result[0][1], 2)  # line number reflects original position
+
+    def test_invalid_raises(self):
+        path = self._write("not json")
+        with self.assertRaises(json.JSONDecodeError):
+            soi.parse_overrides_file(path)
+
+
+class TestCollectCustomVars(unittest.TestCase):
+    def test_finds_custom(self):
+        v = soi.collect_custom_vars({"ip": "$CONCOURSEWORKERS"})
+        self.assertEqual(v, {"$CONCOURSEWORKERS"})
+
+    def test_filters_builtins(self):
+        v = soi.collect_custom_vars({"ip": "$HOME_NET"})
+        self.assertEqual(v, set())
+
+    def test_mixed(self):
+        v = soi.collect_custom_vars({"ip": "[$HOME_NET,$MYNET]"})
+        self.assertEqual(v, {"$MYNET"})
+
+    def test_non_string_fields_ignored(self):
+        v = soi.collect_custom_vars({"count": 10, "isEnabled": True})
+        self.assertEqual(v, set())
+
+
+class TestMakeSession(unittest.TestCase):
+    def _write(self, content):
+        fd, path = tempfile.mkstemp()
+        os.close(fd)
+        with open(path, "w") as f:
+            f.write(content)
+        self.addCleanup(os.unlink, path)
+        return path
+
+    def test_valid_auth_file(self):
+        path = self._write('user = "admin:secret"\n')
+        session = soi.make_session(path)
+        self.assertEqual(session.auth.username, "admin")
+        self.assertEqual(session.auth.password, "secret")
+        self.assertFalse(session.verify)
+
+    def test_missing_user_line(self):
+        path = self._write("# no user line here\n")
+        with self.assertRaises(RuntimeError):
+            soi.make_session(path)
+
+
+class TestFindDetection(unittest.TestCase):
+    def _session_with_response(self, payload):
+        session = MagicMock()
+        response = MagicMock()
+        response.json.return_value = payload
+        response.raise_for_status.return_value = None
+        session.get.return_value = response
+        return session
+
+    def test_found(self):
+        session = self._session_with_response({"hits": {"hits": [{
+            "_id": "abc", "_index": "so-detection",
+            "_source": {"so_detection": {"overrides": [{"type": "suppress"}]}},
+        }]}})
+        doc_id, idx, existing = soi.find_detection(session, "so-detection", "2049201", "suricata")
+        self.assertEqual(doc_id, "abc")
+        self.assertEqual(idx, "so-detection")
+        self.assertEqual(len(existing), 1)
+
+    def test_not_found(self):
+        session = self._session_with_response({"hits": {"hits": []}})
+        doc_id, idx, existing = soi.find_detection(session, "so-detection", "x", "suricata")
+        self.assertIsNone(doc_id)
+        self.assertIsNone(idx)
+        self.assertIsNone(existing)
+
+    def test_no_overrides_field(self):
+        session = self._session_with_response({"hits": {"hits": [{
+            "_id": "abc", "_index": "so-detection",
+            "_source": {"so_detection": {}},
+        }]}})
+        _, _, existing = soi.find_detection(session, "so-detection", "x", "suricata")
+        self.assertEqual(existing, [])
+
+    def test_multiple_hits_warns(self):
+        session = self._session_with_response({"hits": {"hits": [
+            {"_id": "a", "_index": "i", "_source": {"so_detection": {"overrides": []}}},
+            {"_id": "b", "_index": "i", "_source": {"so_detection": {"overrides": []}}},
+        ]}})
+        with patch("sys.stdout", new=StringIO()) as out:
+            doc_id, _, _ = soi.find_detection(session, "i", "x", "suricata")
+        self.assertEqual(doc_id, "a")
+        self.assertIn("WARN", out.getvalue())
+
+
+class TestUpdateOverrides(unittest.TestCase):
+    def test_posts_to_update_endpoint(self):
+        session = MagicMock()
+        response = MagicMock()
+        response.raise_for_status.return_value = None
+        response.json.return_value = {"result": "updated"}
+        session.post.return_value = response
+
+        result = soi.update_overrides(session, "so-detection", "abc", [{"type": "suppress"}])
+
+        self.assertEqual(result, {"result": "updated"})
+        url = session.post.call_args[0][0]
+        self.assertIn("/_update/abc", url)
+        body = session.post.call_args[1]["json"]
+        self.assertEqual(body["doc"]["so_detection"]["overrides"], [{"type": "suppress"}])
+
+
+class TestConfirmProceed(unittest.TestCase):
+    def test_dry_run_skips_prompt(self):
+        args = MagicMock(dry_run=True)
+        with patch("sys.stdout", new=StringIO()):
+            self.assertTrue(soi.confirm_proceed(args))
+
+    def test_yes_input(self):
+        args = MagicMock(dry_run=False)
+        with patch("sys.stdout", new=StringIO()):
+            with patch("builtins.input", return_value="yes"):
+                self.assertTrue(soi.confirm_proceed(args))
+
+    def test_yes_input_case_insensitive(self):
+        args = MagicMock(dry_run=False)
+        with patch("sys.stdout", new=StringIO()):
+            with patch("builtins.input", return_value="YES"):
+                self.assertTrue(soi.confirm_proceed(args))
+
+    def test_no_input_aborts(self):
+        args = MagicMock(dry_run=False)
+        with patch("sys.stdout", new=StringIO()):
+            with patch("builtins.input", return_value="no"):
+                self.assertFalse(soi.confirm_proceed(args))
+
+    def test_empty_input_aborts(self):
+        args = MagicMock(dry_run=False)
+        with patch("sys.stdout", new=StringIO()):
+            with patch("builtins.input", return_value=""):
+                self.assertFalse(soi.confirm_proceed(args))
+
+
+class TestParseArgs(unittest.TestCase):
+    def test_defaults(self):
+        with patch.object(sys, "argv", ["cmd", "--source", "/some/path"]):
+            args = soi.parse_args()
+        self.assertEqual(args.source, "/some/path")
+        self.assertEqual(args.engine, "suricata")
+        self.assertFalse(args.dry_run)
+        self.assertFalse(args.no_import_note)
+        self.assertEqual(args.index, soi.DEFAULT_INDEX)
+
+    def test_all_options(self):
+        argv = ["cmd", "-s", "/x", "-e", "suricata", "-n",
+                "--no-import-note", "-i", "alt-index"]
+        with patch.object(sys, "argv", argv):
+            args = soi.parse_args()
+        self.assertEqual(args.source, "/x")
+        self.assertTrue(args.dry_run)
+        self.assertTrue(args.no_import_note)
+        self.assertEqual(args.index, "alt-index")
+
+
+class TestMain(unittest.TestCase):
+    def setUp(self):
+        self.tmpdir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.tmpdir, ignore_errors=True)
+        # Stub make_session so tests don't need /opt/so/conf/elasticsearch/curl.config.
+        p = patch.object(soi, "make_session", return_value=MagicMock())
+        p.start()
+        self.addCleanup(p.stop)
+
+    def _write_file(self, public_id, overrides, ext="txt"):
+        """Write an NDJSON override file. Entries may be dicts or raw strings (for malformed input)."""
+        path = os.path.join(self.tmpdir, f"{public_id}.{ext}")
+        with open(path, "w") as f:
+            for o in overrides:
+                f.write(o if isinstance(o, str) else json.dumps(o))
+                f.write("\n")
+        return path
+
+    def _run_main(self, *extra_argv, input_response="yes"):
+        """Run main() with stdout/stderr captured and input mocked. Returns (stdout, stderr, exit_code)."""
+        argv = ["cmd", "--source", self.tmpdir, *extra_argv]
+        out, err = StringIO(), StringIO()
+        with patch.object(sys, "argv", argv), \
+                patch("sys.stdout", new=out), \
+                patch("sys.stderr", new=err), \
+                patch("builtins.input", return_value=input_response):
+            with self.assertRaises(SystemExit) as cm:
+                soi.main()
+        return out.getvalue(), err.getvalue(), cm.exception.code
+
+    def test_source_dir_missing(self):
+        argv = ["cmd", "--source", "/no/such/path/here"]
+        err = StringIO()
+        with patch.object(sys, "argv", argv), patch("sys.stderr", new=err):
+            with self.assertRaises(SystemExit) as cm:
+                soi.main()
+        self.assertEqual(cm.exception.code, 1)
+        self.assertIn("source directory not found", err.getvalue())
+
+    def test_no_files_found(self):
+        out, _, code = self._run_main()
+        self.assertEqual(code, 0)
+        self.assertIn("No *.txt files found", out)
+
+    def test_user_aborts(self):
+        self._write_file("1001", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}])
+        out, _, code = self._run_main(input_response="no")
+        self.assertEqual(code, 1)
+        self.assertIn("Aborted", out)
+
+    def test_parse_error_increments_error(self):
+        # Malformed JSON line — parse_overrides_file raises JSONDecodeError.
+        self._write_file("1002", ["not json"])
+        out, _, code = self._run_main("--dry-run")
+        self.assertEqual(code, 1)  # invalid+error → non-zero
+        self.assertIn("could not parse", out)
+        self.assertIn("Errors:                    1", out)
+
+    def test_empty_file_skipped(self):
+        # Blank lines only — parse_overrides_file returns []; main reports "empty file" and continues.
+        path = os.path.join(self.tmpdir, "1003.txt")
+        with open(path, "w") as f:
+            f.write("\n\n")
+        out, _, code = self._run_main("--dry-run")
+        self.assertEqual(code, 0)
+        self.assertIn("empty file", out)
+
+    @patch.object(soi, "find_detection")
+    def test_search_http_error(self, mock_find):
+        mock_find.side_effect = requests.HTTPError("boom")
+        self._write_file("1004", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}])
+        out, _, code = self._run_main("--dry-run")
+        self.assertEqual(code, 1)
+        self.assertIn("search failed", out)
+
+    @patch.object(soi, "find_detection")
+    def test_no_detection_found(self, mock_find):
+        mock_find.return_value = (None, None, None)
+        self._write_file("1005", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}])
+        out, _, code = self._run_main("--dry-run")
+        self.assertEqual(code, 0)
+        self.assertIn("no detection found", out)
+        self.assertIn("Skipped (no detection):    1", out)
+
+    @patch.object(soi, "find_detection")
+    def test_all_duplicates_no_update(self, mock_find):
+        existing = [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}]
+        mock_find.return_value = ("doc1", "so-detection", existing)
+        self._write_file("1006", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}])
+        out, _, code = self._run_main("--dry-run")
+        self.assertEqual(code, 0)
+        self.assertIn("SKIP", out)
+        self.assertNotIn("DRY-RUN: would update", out)  # added_this_file == 0 branch
+
+    @patch.object(soi, "update_overrides")
+    @patch.object(soi, "find_detection")
+    def test_happy_path_full(self, mock_find, mock_update):
+        # Exercises: ADD, dedupe SKIP, INVALID, note prefix, UPDATE, custom-vars warning, exit=1 (invalid present)
+        existing = [{"type": "suppress", "track": "by_src", "ip": "9.9.9.9"}]
+        mock_find.return_value = ("doc1", "so-detection", existing)
+        mock_update.return_value = {"result": "updated"}
+        self._write_file("1007", [
+            {"type": "suppress", "track": "by_src", "ip": "1.2.3.4"},                # ADD
+            {"type": "suppress", "track": "by_src", "ip": "9.9.9.9"},                # SKIP (dupe of existing)
+            {"type": "suppress", "track": "bogus",  "ip": "1.2.3.4"},                # INVALID
+            {"type": "suppress", "track": "by_src", "ip": "$CONCOURSEWORKERS"},      # ADD + custom var
+        ])
+        out, _, code = self._run_main()
+        self.assertEqual(code, 1)  # one invalid -> non-zero
+
+        mock_update.assert_called_once()
+        merged = mock_update.call_args[0][3]
+        self.assertEqual(len(merged), 3)  # 1 existing + 2 new
+        new_notes = [o.get("note", "") for o in merged if o.get("ip") in ("1.2.3.4", "$CONCOURSEWORKERS")]
+        self.assertTrue(all(n.startswith("[Imported ") for n in new_notes))
+
+        self.assertIn("ADD", out)
+        self.assertIn("SKIP", out)
+        self.assertIn("INVALID", out)
+        self.assertIn("UPDATED", out)
+        self.assertIn("$CONCOURSEWORKERS", out)
+
+    @patch.object(soi, "update_overrides")
+    @patch.object(soi, "find_detection")
+    def test_no_import_note_preserves_note(self, mock_find, mock_update):
+        mock_find.return_value = ("doc1", "so-detection", [])
+        mock_update.return_value = {"result": "updated"}
+        self._write_file("1008", [
+            {"type": "suppress", "track": "by_src", "ip": "1.2.3.4", "note": "original"},
+        ])
+        _, _, code = self._run_main("--no-import-note")
+        self.assertEqual(code, 0)
+        merged = mock_update.call_args[0][3]
+        self.assertEqual(merged[0]["note"], "original")  # no prefix applied
+
+    @patch.object(soi, "find_detection")
+    def test_dry_run_skips_update(self, mock_find):
+        mock_find.return_value = ("doc1", "so-detection", [])
+        self._write_file("1009", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}])
+        with patch.object(soi, "update_overrides") as mock_update:
+            out, _, code = self._run_main("--dry-run")
+        self.assertEqual(code, 0)
+        mock_update.assert_not_called()
+        self.assertIn("DRY-RUN: would update", out)
+
+    @patch.object(soi, "update_overrides")
+    @patch.object(soi, "find_detection")
+    def test_update_http_error(self, mock_find, mock_update):
+        mock_find.return_value = ("doc1", "so-detection", [])
+        mock_update.side_effect = requests.HTTPError("nope")
+        self._write_file("1010", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}])
+        out, _, code = self._run_main()
+        self.assertEqual(code, 1)
+        self.assertIn("update failed", out)
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -314,24 +314,6 @@ EOSQL
 	fi
 }

-function sync_minion_config_to_db() {
-	log "INFO" "Syncing minion config to onionconfig for $MINION_ID"
-	/usr/sbin/so-config.py import-minion "$MINION_ID" --note "so-minion $OPERATION"
-	if [ $? -ne 0 ]; then
-		log "ERROR" "Failed to sync minion config to onionconfig for $MINION_ID"
-		return 1
-	fi
-}
-
-function purge_minion_config_from_db() {
-	log "INFO" "Purging minion config from onionconfig for $MINION_ID"
-	/usr/sbin/so-config.py purge-node "$MINION_ID" --note "so-minion delete"
-	if [ $? -ne 0 ]; then
-		log "ERROR" "Failed to purge minion config from onionconfig for $MINION_ID"
-		return 1
-	fi
-}
-
 # Create the minion file
 function ensure_socore_ownership() {
 	log "INFO" "Setting socore ownership on minion files"
@@ -1106,10 +1088,6 @@ case "$OPERATION" in
 			log "ERROR" "Failed to setup minion files for $MINION_ID"
 			exit 1
 		}
-		sync_minion_config_to_db || {
-			log "ERROR" "Failed to sync minion config to onionconfig for $MINION_ID"
-			exit 1
-		}
 		updateMineAndApplyStates || {
 			log "ERROR" "Failed to update mine and apply states for $MINION_ID"
 			exit 1
@@ -1130,20 +1108,12 @@ case "$OPERATION" in
 			log "ERROR" "Failed to setup VM minion files for $MINION_ID"
 			exit 1
 		}
-		sync_minion_config_to_db || {
-			log "ERROR" "Failed to sync VM minion config to onionconfig for $MINION_ID"
-			exit 1
-		}
 		log "INFO" "Successfully added VM minion $MINION_ID"
 		;;

 	"delete")
 		log "INFO" "Removing minion $MINION_ID"
 		remove_postgres_telegraf_from_minion
-		purge_minion_config_from_db || {
-			log "ERROR" "Failed to purge minion config from onionconfig for $MINION_ID"
-			exit 1
-		}
 		deleteMinionFiles || {
 			log "ERROR" "Failed to delete minion files for $MINION_ID"
 			exit 1
@@ -25,7 +25,6 @@ def showUsage(args):
    print('    get [-r]         - Displays (to stdout) the value stored in the given key. Requires KEY arg. Use -r for raw output without YAML formatting.', file=sys.stderr)
    print('    remove           - Removes a yaml key, if it exists. Requires KEY arg.', file=sys.stderr)
    print('    replace          - Replaces (or adds) a new key and set its value. Requires KEY and VALUE args.', file=sys.stderr)
-    print('    purge            - Delete the YAML file from disk (no KEY arg).', file=sys.stderr)
    print('    help             - Prints this usage information.', file=sys.stderr)
    print('', file=sys.stderr)
    print('  Where:', file=sys.stderr)
@@ -54,20 +53,7 @@ def loadYaml(filename):

 def writeYaml(filename, content):
    file = open(filename, "w")
-    result = yaml.safe_dump(content, file)
-    file.close()
-    return result
-
-
-def purgeFile(filename):
-    """Delete a YAML file from disk. Idempotent; missing files are success."""
-    if os.path.exists(filename):
-        try:
-            os.remove(filename)
-        except Exception as e:
-            print(f"Failed to remove {filename}: {e}", file=sys.stderr)
-            return 1
-    return 0
+    return yaml.safe_dump(content, file)


 def appendItem(content, key, listItem):
@@ -385,15 +371,6 @@ def get(args):
    return 0


-def purge(args):
-    """purge YAML_FILE - delete the file from disk."""
-    if len(args) != 1:
-        print('Missing filename arg', file=sys.stderr)
-        showUsage(None)
-        return 1
-    return purgeFile(args[0])
-
-
 def main():
    args = sys.argv[1:]

@@ -411,7 +388,6 @@ def main():
        "get": get,
        "remove": remove,
        "replace": replace,
-        "purge": purge,
    }

    code = 1
@@ -991,31 +991,3 @@ class TestLoadYaml(unittest.TestCase):
                    soyaml.loadYaml("/tmp/so-yaml_test-unreadable.yaml")
                    sysmock.assert_called_with(1)
                    self.assertIn("Error reading file", mock_stderr.getvalue())
-
-
-class TestPurge(unittest.TestCase):
-
-    def test_purge_missing_arg(self):
-        # showUsage calls sys.exit(1); patch it like the other tests do.
-        with patch('sys.exit', new=MagicMock()):
-            with patch('sys.stderr', new=StringIO()) as mock_stderr:
-                rc = soyaml.purge([])
-                self.assertEqual(rc, 1)
-                self.assertIn("Missing filename", mock_stderr.getvalue())
-
-    def test_purge_existing_file(self):
-        filename = "/tmp/so-yaml_test_purge.yaml"
-        with open(filename, "w") as f:
-            f.write("key: value\n")
-        rc = soyaml.purge([filename])
-        self.assertEqual(rc, 0)
-        import os as _os
-        self.assertFalse(_os.path.exists(filename))
-
-    def test_purge_missing_file_idempotent(self):
-        filename = "/tmp/so-yaml_test_purge_missing.yaml"
-        import os as _os
-        if _os.path.exists(filename):
-            _os.remove(filename)
-        rc = soyaml.purge([filename])
-        self.assertEqual(rc, 0)
@@ -485,6 +485,158 @@ elasticsearch_backup_index_templates() {
  tar -czf /nsm/backup/3.0.0_elasticsearch_index_templates.tar.gz -C /opt/so/conf/elasticsearch/templates/index/ .
 }

+elasticfleet_set_agent_logging_level_warn() {
+    . /usr/sbin/so-elastic-fleet-common
+
+    local current_agent_policies
+    if ! current_agent_policies=$(fleet_api "agent_policies?perPage=1000"); then
+        echo "Warning: unable to retrieve Fleet agent policies"
+        return 0
+    fi
+
+    # Only updating policies that are within Security Onion defaults and do not already have any user configured advanced_settings.
+    local policies_to_update
+    policies_to_update=$(jq -c '
+        .items[]
+        | select(has("advanced_settings") | not)
+        | select(
+            .id == "so-grid-nodes_general"
+            or .id == "so-grid-nodes_heavy"
+            or .id == "endpoints-initial"
+            or (.id | startswith("FleetServer_"))
+          )
+    ' <<< "$current_agent_policies")
+
+    if [[ -z "$policies_to_update" ]]; then
+        return 0
+    fi
+
+    while IFS= read -r policy; do
+        [[ -z "$policy" ]] && continue
+
+        local policy_id policy_name policy_namespace
+        policy_id=$(jq -r '.id' <<< "$policy")
+        policy_name=$(jq -r '.name' <<< "$policy")
+        policy_namespace=$(jq -r '.namespace' <<< "$policy")
+
+        local update_logging
+        update_logging=$(jq -n \
+            --arg name "$policy_name" \
+            --arg namespace "$policy_namespace" \
+            '{name: $name, namespace: $namespace, advanced_settings: {agent_logging_level: "warning"}}'
+        )
+
+        echo "Setting elastic agent_logging_level to warning on policy '$policy_name' ($policy_id)."
+        if ! fleet_api "agent_policies/$policy_id" -XPUT -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$update_logging" >/dev/null; then
+            echo "  warning: failed to update agent policy '$policy_name' ($policy_id)" >&2
+        fi
+    done <<< "$policies_to_update"
+}
+
+update_logstash_pipeline_name() {
+    local original_pipeline_name="$1"
+    local new_pipeline_name="$2"
+
+    echo "Checking for conflicting logstash defined_pipelines pillar value."
+    local LOGSTASH_FILE=/opt/so/saltstack/local/pillar/logstash/soc_logstash.sls
+    local MINIONDIR=/opt/so/saltstack/local/pillar/minions
+    for pillar_file in "$LOGSTASH_FILE" "$MINIONDIR"/*.sls; do
+        [[ -f "$pillar_file" ]] || continue
+        if grep -q "$original_pipeline_name$" "$pillar_file"; then
+            echo "Found conflicting defined_pipeline pillar value in $pillar_file. Updating to use the new logstash pipeline name."
+            sed -i "s#$original_pipeline_name\$#$new_pipeline_name#g" "$pillar_file"
+            chown socore:socore "$pillar_file"
+        fi
+    done
+}
+
+check_transform_health_and_reauthorize() {
+    . /usr/sbin/so-elastic-fleet-common
+
+    echo "Checking integration transform jobs for unhealthy / unauthorized status..."
+
+    local transforms_doc stats_doc installed_doc
+    if ! transforms_doc=$(so-elasticsearch-query "_transform/_all?size=1000" --fail --retry 3 --retry-delay 5 2>/dev/null); then
+        echo "Unable to query for transform jobs, skipping reauthorization."
+        return 0
+    fi
+    if ! stats_doc=$(so-elasticsearch-query "_transform/_all/_stats?size=1000" --fail --retry 3 --retry-delay 5 2>/dev/null); then
+        echo "Unable to query for transform job stats, skipping reauthorization."
+        return 0
+    fi
+    if ! installed_doc=$(fleet_api "epm/packages/installed?perPage=500"); then
+        echo "Unable to list installed Fleet packages, skipping reauthorization."
+        return 0
+    fi
+
+    # Get all transforms that meet the following
+    # - unhealthy (any non-green health status)
+    # - metadata has run_as_kibana_system: false (this fix is specific to transforms started prior to Kibana 9.3.3)
+    # - are not orphaned (integration is not somehow missing/corrupt/uninstalled)
+    local tmp_transforms tmp_stats tmp_installed
+    tmp_transforms=$(mktemp)
+    tmp_stats=$(mktemp)
+    tmp_installed=$(mktemp)
+
+    echo "$transforms_doc" > "$tmp_transforms"
+    echo "$stats_doc"      > "$tmp_stats"
+    echo "$installed_doc"  > "$tmp_installed"
+
+    local unhealthy_transforms
+    unhealthy_transforms=$(jq -c -n \
+        --slurpfile t "$tmp_transforms" \
+        --slurpfile s "$tmp_stats" \
+        --slurpfile i "$tmp_installed" '
+        ($i[0].items | map({key: .name, value: .version}) | from_entries) as $pkg_ver
+        | ($s[0].transforms | map({key: .id, value: .health.status}) | from_entries) as $health
+        | [ $t[0].transforms[]
+            | select(._meta.run_as_kibana_system == false)
+            | select(($health[.id] // "unknown") != "green")
+            | {id, pkg: ._meta.package.name, ver: ($pkg_ver[._meta.package.name])}
+          ]
+        | if length == 0 then empty else . end
+        | (map(select(.ver == null)) | map({orphan: .id})[]),
+          (map(select(.ver != null))
+           | group_by(.pkg)
+           | map({pkg: .[0].pkg, ver: .[0].ver, transformIds: map(.id)})[])
+    ')
+
+    if [[ -z "$unhealthy_transforms" ]]; then
+        return 0
+    fi
+
+    local unhealthy_count
+    unhealthy_count=$(jq -s '[.[].transformIds? // empty | .[]] | length' <<< "$unhealthy_transforms")
+    echo "Found $unhealthy_count transform(s) needing reauthorization."
+
+    local total_failures=0
+    while IFS= read -r transform; do
+        [[ -z "$transform" ]] && continue
+        if jq -e 'has("orphan")' <<< "$transform" >/dev/null 2>&1; then
+            echo "Skipping transform not owned by any installed Fleet package: $(jq -r '.orphan' <<< "$transform")"
+            continue
+        fi
+
+        local pkg ver body resp
+        pkg=$(jq -r '.pkg' <<< "$transform")
+        ver=$(jq -r '.ver' <<< "$transform")
+        body=$(jq -c '{transforms: (.transformIds | map({transformId: .}))}' <<< "$transform")
+
+        echo "Reauthorizing transform(s) for ${pkg}-${ver}..."
+        resp=$(fleet_api "epm/packages/${pkg}/${ver}/transforms/authorize" \
+                        -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' \
+                        -d "$body") || { echo "Could not reauthorize transform(s) for ${pkg}-${ver}"; continue; }
+
+        (( total_failures += $(jq 'map(select(.success != true)) | length' <<< "$resp" 2>/dev/null) ))
+    done <<< "$unhealthy_transforms"
+
+    rm -f "$tmp_transforms" "$tmp_stats" "$tmp_installed"
+
+    if [[ "$total_failures" -gt 0 ]]; then
+        echo "Some transform(s) failed to reauthorize."
+    fi
+}
+
 ensure_postgres_local_pillar() {
  # Postgres was added as a service after 3.0.0, so the new pillar/top.sls
  # references postgres.soc_postgres / postgres.adv_postgres unconditionally.
@@ -520,6 +672,31 @@ ensure_postgres_secret() {
  chown socore:socore "$secrets_file"
 }

+rename_strelka_scan_lnk() {
+  echo "Renaming strelka pillar ScanLNK to ScanLnk."
+  local STRELKA_FILE=/opt/so/saltstack/local/pillar/strelka/soc_strelka.sls
+  local MINIONDIR=/opt/so/saltstack/local/pillar/minions
+  local OLD_KEY=strelka.backend.config.backend.scanners.ScanLNK
+  local NEW_KEY=strelka.backend.config.backend.scanners.ScanLnk
+  local TMP_VALUE_FILE
+  TMP_VALUE_FILE=$(mktemp)
+
+  for pillar_file in "$STRELKA_FILE" "$MINIONDIR"/*.sls; do
+    [[ -f "$pillar_file" ]] || continue
+    # Skip if ScanLNK doesn't exist
+    so-yaml.py get "$pillar_file" "$OLD_KEY" > "$TMP_VALUE_FILE" 2>/dev/null || continue
+    echo "Found 'ScanLNK' key in $pillar_file. Renaming to 'ScanLnk'."
+    so-yaml.py add "$pillar_file" "$NEW_KEY" "file:$TMP_VALUE_FILE"
+    so-yaml.py remove "$pillar_file" "$OLD_KEY"
+  done
+
+  rm -f "$TMP_VALUE_FILE"
+}
+
+fix_logstash_0013_lumberjack_pipeline_name() {
+    update_logstash_pipeline_name "so/0013_input_lumberjack_fleet.conf" "so/0013_input_lumberjack_fleet.conf.jinja"
+}
+
 up_to_3.1.0() {
  ensure_postgres_local_pillar
  ensure_postgres_secret
@@ -527,7 +704,8 @@ up_to_3.1.0() {
  elasticsearch_backup_index_templates
  # Clear existing component template state file.
  rm -f /opt/so/state/esfleet_component_templates.json
-
+  rename_strelka_scan_lnk
+  fix_logstash_0013_lumberjack_pipeline_name

  INSTALLEDVERSION=3.1.0
 }
@@ -553,6 +731,12 @@ post_to_3.1.0() {
  # file_roots of its own and --local would fail with "No matching sls found".
  salt-call state.apply postgres.telegraf_users queue=True || true

+  # Update default agent policies to use logging level warn.
+  elasticfleet_set_agent_logging_level_warn || true
+
+  # Check for unhealthy / unauthorized integration transform jobs and attempt reauthorizations
+  check_transform_health_and_reauthorize || true
+
  POSTVERSION=3.1.0
 }

@@ -809,6 +993,9 @@ verify_es_version_compatibility() {
    local is_active_intermediate_upgrade=1
    # supported upgrade paths for SO-ES versions
    declare -A es_upgrade_map=(
+        ["8.18.4"]="8.18.6 8.18.8 9.0.8"
+	    ["8.18.6"]="8.18.8 9.0.8"
+	    ["8.18.8"]="9.0.8"
        ["9.0.8"]="9.3.3"
    )

@@ -832,6 +1019,171 @@ verify_es_version_compatibility() {
        exit 160
    fi

+    compatible_es_versions="$target_es_version"
+    for current_version in "${!es_upgrade_map[@]}"; do
+        # shellcheck disable=SC2076
+        if [[ " ${es_upgrade_map[$current_version]} " =~ " $target_es_version " ]]; then
+            compatible_es_versions+=" $current_version"
+        fi
+    done
+
+    # Check if the given ES version can directly upgrade to the target ES version. Used to assist with catching lagging nodes during the upgrade process
+    es_version_can_upgrade_to_target() {
+        local current_version="$1"
+        # shellcheck disable=SC2076
+        if [[ -n "$current_version" && " $compatible_es_versions " =~ " $current_version " ]]; then
+            return 0
+        fi
+
+        return 1
+    }
+
+    # Gather Elasticsearch cluster version info and verify that each node in the cluster is running a version compatible with the target ES version.
+    verify_searchnodes_es_target_compatibility() {
+        local retries=20
+        local retry_count=0
+        local delay=180
+        local expected_es_nodes searchnode_minions attempt
+        local searchnode_discovery_success=false
+        SEARCHNODE_ES_VERSIONS=""
+
+        for attempt in {1..3}; do
+            if searchnode_minions=$(set -o pipefail; salt-key --out=json --list=accepted 2> /dev/null | jq -r '.minions[]? | select(endswith("searchnode"))'); then
+                searchnode_discovery_success=true
+                break
+            fi
+
+            echo "Failed to retrieve grid searchnodes via salt-key... Retrying in 30 seconds. Attempt $attempt of 3."
+            sleep 30
+        done
+
+        if [[ "$searchnode_discovery_success" != "true" ]]; then
+            echo "Failed to retrieve grid searchnodes via salt-key."
+            return 1
+        fi
+
+        # Always add node running soup to expected es nodes
+        expected_es_nodes="${MINIONID%_*}"
+        while IFS= read -r searchnode_minion; do
+            [[ -z "$searchnode_minion" ]] && continue
+            expected_es_nodes+=$'\n'"${searchnode_minion%_searchnode}"
+        done <<< "$searchnode_minions"
+
+        while [[ $retry_count -lt $retries ]]; do
+            SEARCHNODE_ES_VERSIONS=$(so-elasticsearch-query _nodes/_all/version --retry 5 --retry-delay 10 --fail 2>&1)
+            local exit_status=$?
+
+            if [[ $exit_status -ne 0 ]]; then
+                echo "Failed to retrieve Elasticsearch versions from searchnodes... Retrying in $delay seconds. Attempt $((retry_count + 1)) of $retries."
+                ((retry_count++))
+                sleep $delay
+                continue
+            fi
+
+            local all_searchnodes_compatible=true
+            while IFS=$'\t' read -r node current_version; do
+                [[ -z "$node" ]] && continue
+                if ! es_version_can_upgrade_to_target "$current_version"; then
+                    echo "Searchnode $node is running Elasticsearch $current_version, which is not directly upgradable to Elasticsearch $target_es_version."
+                    all_searchnodes_compatible=false
+                fi
+            done < <(echo "$SEARCHNODE_ES_VERSIONS" | jq -r '.nodes | to_entries[] | [.value.name, .value.version] | @tsv')
+
+            while IFS= read -r expected_es_node; do
+                [[ -z "$expected_es_node" ]] && continue
+                if ! echo "$SEARCHNODE_ES_VERSIONS" | jq -e --arg node "$expected_es_node" '.nodes | to_entries | any(.value.name == $node)' > /dev/null; then
+                    echo "Searchnode $expected_es_node did not report an Elasticsearch version. It may be offline or still upgrading."
+                    all_searchnodes_compatible=false
+                fi
+            done <<< "$expected_es_nodes"
+
+            if [[ "$all_searchnodes_compatible" == true ]]; then
+                echo "All Searchnodes are upgradable to Elasticsearch $target_es_version."
+                return 0
+            fi
+
+            echo "One or more Searchnodes cannot upgrade directly to Elasticsearch $target_es_version. Rechecking in $delay seconds. Attempt $((retry_count + 1)) of $retries."
+            ((retry_count++))
+            sleep $delay
+        done
+
+        return 1
+    }
+
+    # Gather heavynode version info and verify that each node is running a version compatible with the target ES version.
+    verify_heavynodes_es_target_compatibility() {
+        local heavynode_minions attempt
+        local retries=20
+        local retry_count=0
+        local delay=180
+        local heavynode_discovery_success=false
+        HEAVYNODE_ES_VERSIONS=""
+
+        for attempt in {1..3}; do
+            if heavynode_minions=$(set -o pipefail; salt-key --out=json --list=accepted 2> /dev/null | jq -r '.minions[]? | select(endswith("heavynode"))'); then
+                heavynode_discovery_success=true
+                break
+            fi
+
+            echo "Failed to retrieve grid heavynodes via salt-key... Retrying in 30 seconds. Attempt $attempt of 3."
+            sleep 30
+        done
+
+        if [[ "$heavynode_discovery_success" != "true" ]]; then
+            echo "Failed to retrieve grid heavynodes via salt-key."
+            return 1
+        fi
+
+        if [[ -z "$heavynode_minions" ]]; then
+            echo "No heavynodes detected. Skipping heavynode Elasticsearch version compatibility check."
+            return 0
+        fi
+
+        while [[ $retry_count -lt $retries ]]; do
+            HEAVYNODE_ES_VERSIONS=$(salt -C 'G@role:so-heavynode' cmd.run 'set -o pipefail; so-elasticsearch-query / --retry 5 --retry-delay 10 | jq -er ".version.number"' shell=/bin/bash --out=json 2> /dev/null)
+            local exit_status=$?
+
+            if [[ $exit_status -ne 0 ]]; then
+                echo "Failed to retrieve Elasticsearch version from one or more heavynodes... Retrying in $delay seconds. Attempt $((retry_count + 1)) of $retries."
+                ((retry_count++))
+                sleep $delay
+                continue
+            fi
+
+            local all_heavynodes_compatible=true
+            while IFS=$'\t' read -r node current_version; do
+                [[ -z "$node" ]] && continue
+                if ! es_version_can_upgrade_to_target "$current_version"; then
+                    echo "Heavynode $node is running Elasticsearch $current_version, which is not directly upgradable to Elasticsearch $target_es_version."
+                    all_heavynodes_compatible=false
+                fi
+            done < <(echo "$HEAVYNODE_ES_VERSIONS" | jq -r 'to_entries[] | [.key, .value] | @tsv')
+
+            while IFS= read -r heavynode_minion; do
+                [[ -z "$heavynode_minion" ]] && continue
+                if ! echo "$HEAVYNODE_ES_VERSIONS" | jq -se --arg minion "$heavynode_minion" 'add | has($minion)' > /dev/null; then
+                    echo "Heavynode $heavynode_minion did not report an Elasticsearch version. It may be offline or still upgrading."
+                    all_heavynodes_compatible=false
+                fi
+            done <<< "$heavynode_minions"
+
+            if [[ "$all_heavynodes_compatible" == true ]]; then
+                echo -e "\nAll heavynodes can upgrade to Elasticsearch $target_es_version."
+                return 0
+            fi
+
+            echo "One or more heavynodes cannot upgrade directly to Elasticsearch $target_es_version. Rechecking in $delay seconds. Attempt $((retry_count + 1)) of $retries."
+            ((retry_count++))
+            sleep $delay
+        done
+
+        return 1
+    }
+
+    if [[ ! -f "$es_verification_script" ]]; then
+        create_intermediate_upgrade_verification_script "$es_verification_script"
+    fi
+
    for statefile in "${es_required_version_statefile_base}"-*; do
        [[ -f $statefile ]] || continue

@@ -850,10 +1202,6 @@ verify_es_version_compatibility() {
            continue
        fi

-        if [[ ! -f "$es_verification_script" ]]; then
-            create_intermediate_upgrade_verification_script "$es_verification_script"
-        fi
-
        echo -e "\n##############################################################################################################################\n"
        echo "A previously required intermediate Elasticsearch upgrade was detected. Verifying that all Searchnodes/Heavynodes have successfully upgraded Elasticsearch to $es_required_version_statefile_value before proceeding with soup to avoid potential data loss! This command can take up to an hour to complete."
        if ! timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$statefile"; then
@@ -875,6 +1223,26 @@ verify_es_version_compatibility() {

    # shellcheck disable=SC2076 # Do not want a regex here eg usage " 8.18.8 9.0.8 " =~ " 9.0.8 "
    if [[ " ${es_upgrade_map[$es_version]} " =~ " $target_es_version " || "$es_version" == "$target_es_version" ]]; then
+        if ! verify_searchnodes_es_target_compatibility || ! verify_heavynodes_es_target_compatibility; then
+            echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
+
+            echo "One or more Searchnode(s)/Heavynode(s) cannot upgrade directly to Elasticsearch $target_es_version. This can happen with soups that include Elasticsearch upgrades being run in quick succession. Typically, this will resolve itself as the grid synchronizes. Please allow time for all Searchnodes/Heavynodes to have upgraded Elasticsearch to a compatible version with $target_es_version before running soup again to avoid potential data loss!"
+
+            if [[ -n "$HEAVYNODE_ES_VERSIONS" ]]; then
+                echo "Current heavynode Elasticsearch versions:"
+                echo "$HEAVYNODE_ES_VERSIONS" | jq '.'
+            fi
+
+            if [[ -n "$SEARCHNODE_ES_VERSIONS" ]]; then
+                echo "Current searchnode Elasticsearch versions:"
+                echo "$SEARCHNODE_ES_VERSIONS" | jq '.nodes | to_entries | map({(.value.name): .value.version}) | sort | add'
+            fi
+
+            echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
+
+            exit 161
+        fi
+
        # supported upgrade
        return 0
    else
@@ -1160,7 +1528,13 @@ EOF

 # Keeping this block in case we need to do a hotfix that requires salt update
 apply_hotfix() {
-   echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)"
+    if [[ "$INSTALLEDVERSION" == "3.1.0" ]] ; then
+       # Do not remove this fix_logstash_0013_lumberjack_pipeline_name in future hotfixes without first validating older
+       #  installs referencing "so/0013_input_lumberjack_fleet.conf" via pillar are upgradable
+       fix_logstash_0013_lumberjack_pipeline_name
+    else
+        echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)"
+    fi
 }

 failed_soup_restore_items() {
@@ -1232,7 +1606,7 @@ main() {
  echo "Verifying we have the latest soup script."
  verify_latest_update_script

-  echo "Verifying Elasticsearch version compatibility before upgrading."
+  echo "Verifying Elasticsearch version compatibility across the grid before upgrading."
  verify_es_version_compatibility

  echo "Let's see if we need to update Security Onion."
@@ -33,11 +33,8 @@ so-elastic-fleet-stop --force

 status "Deleting Fleet Data from Pillars..."
 so-yaml.py remove /opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls elasticfleet
-/usr/sbin/so-config.py sync-yaml-mutation /opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls remove elasticfleet --note "so-elastic-fleet-reset"
 so-yaml.py remove /opt/so/saltstack/local/pillar/global/soc_global.sls global.fleet_grid_enrollment_token_general
-/usr/sbin/so-config.py sync-yaml-mutation /opt/so/saltstack/local/pillar/global/soc_global.sls remove global.fleet_grid_enrollment_token_general --note "so-elastic-fleet-reset"
 so-yaml.py remove /opt/so/saltstack/local/pillar/global/soc_global.sls global.fleet_grid_enrollment_token_heavy
-/usr/sbin/so-config.py sync-yaml-mutation /opt/so/saltstack/local/pillar/global/soc_global.sls remove global.fleet_grid_enrollment_token_heavy --note "so-elastic-fleet-reset"

 status "Restarting Kibana..."
 so-kibana-restart --force
@@ -225,6 +225,7 @@ http {
 			limit_req             zone=auth_throttle burst={{ NGINXMERGED.config.throttle_login_burst }} nodelay;
 			limit_req_status      429;
 			proxy_pass            http://{{ GLOBALS.manager }}:4433;
+			proxy_set_header      Connection "Close";
 			proxy_read_timeout    90;
 			proxy_connect_timeout 90;
 			proxy_set_header      Host $host;
@@ -237,6 +238,7 @@ http {
 		location ~ ^/auth/.*?(whoami|logout|settings|errors|webauthn.js) {
 			rewrite               /auth/(.*) /$1 break;
 			proxy_pass            http://{{ GLOBALS.manager }}:4433;
+			proxy_set_header      Connection "Close";
 			proxy_read_timeout    90;
 			proxy_connect_timeout 90;
 			proxy_set_header      Host $host;
@@ -3,7 +3,14 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-{% set hypervisor = pillar.minion_id %}
+{% set hypervisor = pillar.get('minion_id', '') %}
+
+{% if not hypervisor|regex_match('^([A-Za-z0-9._-]{1,253})$') %}
+{%   do salt.log.error('delete_hypervisor_orch: refusing unsafe minion_id=' ~ hypervisor) %}
+delete_hypervisor_invalid_minion_id:
+  test.fail_without_changes:
+    - name: delete_hypervisor_invalid_minion_id
+{% else %}

 ensure_hypervisor_mine_deleted:
  salt.function:
@@ -20,3 +27,5 @@ update_salt_cloud_profile:
    - sls:
      - salt.cloud.config
    - concurrent: True
+
+{% endif %}
@@ -12,7 +12,14 @@
 {% if 'vrt' in salt['pillar.get']('features', []) %}

 {%   do salt.log.debug('vm_pillar_clean_orch: Running') %}
-{%   set vm_name = pillar.get('vm_name') %}
+{%   set vm_name = pillar.get('vm_name', '') %}
+
+{%   if not vm_name|regex_match('^([A-Za-z0-9._-]{1,253})$') %}
+{%     do salt.log.error('vm_pillar_clean_orch: refusing unsafe vm_name=' ~ vm_name) %}
+vm_pillar_clean_invalid_name:
+  test.fail_without_changes:
+    - name: vm_pillar_clean_invalid_name
+{%   else %}

 delete_adv_{{ vm_name }}_pillar:
  module.run:
@@ -24,6 +31,8 @@ delete_{{ vm_name }}_pillar:
    - file.remove:
      - path: /opt/so/saltstack/local/pillar/minions/{{ vm_name }}.sls

+{%   endif %}
+
 {% else %}

 {%   do salt.log.error(
@@ -46,10 +46,10 @@ postgresinitdir:
    - require:
      - file: postgresconfdir

-postgresinitusers:
+postgresinitdb:
  file.managed:
-    - name: /opt/so/conf/postgres/init/init-users.sh
-    - source: salt://postgres/files/init-users.sh
+    - name: /opt/so/conf/postgres/init/init-db.sh
+    - source: salt://postgres/files/init-db.sh
    - user: 939
    - group: 939
    - mode: 755
@@ -31,7 +31,7 @@ so-postgres:
      - POSTGRES_DB=securityonion
      # Passwords are delivered via mounted 0600 secret files, not plaintext env vars.
      # The upstream postgres image resolves POSTGRES_PASSWORD_FILE; entrypoint.sh and
-      # init-users.sh resolve SO_POSTGRES_PASS_FILE the same way.
+      # init-db.sh resolve SO_POSTGRES_PASS_FILE the same way.
      - POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password
      - SO_POSTGRES_USER={{ SO_POSTGRES_USER }}
      - SO_POSTGRES_PASS_FILE=/run/secrets/so_postgres_pass
@@ -46,7 +46,7 @@ so-postgres:
      - /opt/so/conf/postgres/postgresql.conf:/conf/postgresql.conf:ro
      - /opt/so/conf/postgres/pg_hba.conf:/conf/pg_hba.conf:ro
      - /opt/so/conf/postgres/secrets:/run/secrets:ro
-      - /opt/so/conf/postgres/init/init-users.sh:/docker-entrypoint-initdb.d/init-users.sh:ro
+      - /opt/so/conf/postgres/init/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh:ro
      - /etc/pki/postgres.crt:/conf/postgres.crt:ro
      - /etc/pki/postgres.key:/conf/postgres.key:ro
      - /etc/pki/tls/certs/intca.crt:/conf/ca.crt:ro
@@ -70,7 +70,7 @@ so-postgres:
    - watch:
      - file: postgresconf
      - file: postgreshba
-      - file: postgresinitusers
+      - file: postgresinitdb
      - file: postgres_super_secret
      - file: postgres_app_secret
      - x509: postgres_crt
@@ -78,7 +78,7 @@ so-postgres:
    - require:
      - file: postgresconf
      - file: postgreshba
-      - file: postgresinitusers
+      - file: postgresinitdb
      - file: postgres_super_secret
      - file: postgres_app_secret
      - x509: postgres_crt
@@ -1,20 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-
-# Deprecated: the old so_pillar schema has been replaced by SOC-owned
-# onionconfig tables. SOC creates its schema on first startup.
-postgres_schema_pillar_deprecated:
-  test.nop
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}
@@ -39,17 +39,15 @@ postgres_wait_ready:
    - require:
      - docker_container: so-postgres

-# Ensure the shared Telegraf database exists. init-users.sh only runs on a
+# Ensure the shared Telegraf database exists. init-db.sh only runs on a
 # fresh data dir, so hosts upgraded onto an existing /nsm/postgres volume
 # would otherwise never get so_telegraf.
 postgres_create_telegraf_db:
  cmd.run:
-    - name: |
-        if ! docker exec so-postgres psql -U postgres -tAc "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1; then
-          docker exec so-postgres psql -v ON_ERROR_STOP=1 -U postgres -c "CREATE DATABASE so_telegraf"
-        fi
+    - name: /usr/sbin/so-telegraf-postgres create_db
    - require:
      - cmd: postgres_wait_ready
+      - file: postgres_sbin

 # Provision the shared group role and schema once. Every per-minion role is a
 # member of so_telegraf, and each Telegraf connection does SET ROLE so_telegraf
@@ -57,68 +55,26 @@ postgres_create_telegraf_db:
 # on first write are owned by the group role and every member can INSERT/SELECT.
 postgres_telegraf_group_role:
  cmd.run:
-    - name: |
-        docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
-        DO $$
-        BEGIN
-            IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'so_telegraf') THEN
-                CREATE ROLE so_telegraf NOLOGIN;
-            END IF;
-        END
-        $$;
-        GRANT CONNECT ON DATABASE so_telegraf TO so_telegraf;
-        CREATE SCHEMA IF NOT EXISTS telegraf AUTHORIZATION so_telegraf;
-        GRANT USAGE, CREATE ON SCHEMA telegraf TO so_telegraf;
-        CREATE SCHEMA IF NOT EXISTS partman;
-        CREATE EXTENSION IF NOT EXISTS pg_partman SCHEMA partman;
-        CREATE EXTENSION IF NOT EXISTS pg_cron;
-        -- Telegraf (running as so_telegraf) calls partman.create_parent()
-        -- on first write of each metric, which needs USAGE on the partman
-        -- schema, EXECUTE on its functions/procedures, and write access to
-        -- partman.part_config so it can register new partitioned parents.
-        GRANT USAGE, CREATE ON SCHEMA partman TO so_telegraf;
-        GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA partman TO so_telegraf;
-        GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA partman TO so_telegraf;
-        GRANT EXECUTE ON ALL PROCEDURES IN SCHEMA partman TO so_telegraf;
-        -- partman creates per-parent template tables (partman.template_*) at
-        -- runtime; default privileges extend DML/sequence access to them.
-        ALTER DEFAULT PRIVILEGES IN SCHEMA partman
-            GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO so_telegraf;
-        ALTER DEFAULT PRIVILEGES IN SCHEMA partman
-            GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO so_telegraf;
-        -- Hourly partman maintenance. cron.schedule is idempotent by jobname.
-        SELECT cron.schedule(
-          'telegraf-partman-maintenance',
-          '17 * * * *',
-          'CALL partman.run_maintenance_proc()'
-        );
-        EOSQL
+    - name: /usr/sbin/so-telegraf-postgres group_role
    - require:
      - cmd: postgres_create_telegraf_db
+      - file: postgres_sbin

 {%   set creds = salt['pillar.get']('telegraf:postgres_creds', {}) %}
 {%   for mid, entry in creds.items() %}
 {%     if entry.get('user') and entry.get('pass') %}
 {%       set u = entry.user %}
-{%       set p = entry.pass | replace("'", "''") %}
+{%       set p = entry.pass %}

 postgres_telegraf_role_{{ u }}:
  cmd.run:
-    - name: |
-        docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
-        DO $$
-        BEGIN
-            IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '{{ u }}') THEN
-                EXECUTE format('CREATE ROLE %I WITH LOGIN PASSWORD %L', '{{ u }}', '{{ p }}');
-            ELSE
-                EXECUTE format('ALTER ROLE %I WITH PASSWORD %L', '{{ u }}', '{{ p }}');
-            END IF;
-        END
-        $$;
-        GRANT CONNECT ON DATABASE so_telegraf TO "{{ u }}";
-        GRANT so_telegraf TO "{{ u }}";
-        EOSQL
+    - name: /usr/sbin/so-telegraf-postgres user
+    - env:
+      - ROLE_USER: {{ u | tojson }}
+      - ROLE_PASS: {{ p | tojson }}
+    - hide_output: True
    - require:
+      - file: postgres_sbin
      - cmd: postgres_telegraf_group_role

 {%     endif %}
@@ -130,21 +86,12 @@ postgres_telegraf_role_{{ u }}:
 {%   set retention = salt['pillar.get']('postgres:telegraf:retention_days', 14) | int %}
 postgres_telegraf_retention_reconcile:
  cmd.run:
-    - name: |
-        docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
-        DO $$
-        BEGIN
-            IF EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_partman') THEN
-                UPDATE partman.part_config
-                SET retention = '{{ retention }} days',
-                    retention_keep_table = false
-                WHERE parent_table LIKE 'telegraf.%';
-            END IF;
-        END
-        $$;
-        EOSQL
+    - name: /usr/sbin/so-telegraf-postgres retention
+    - env:
+      - RETENTION_DAYS: {{ retention }}
    - require:
      - cmd: postgres_telegraf_group_role
+      - file: postgres_sbin

 {% endif %}

@@ -7,15 +7,29 @@

 . /usr/sbin/so-common

+# Without pipefail, a pipeline's exit status is gzip's. A failed pg_dumpall would
+# otherwise be masked by a successful gzip, silently producing a valid .gz that
+# holds a truncated dump.
+set -o pipefail
+
 # Backups contain role password hashes and full chat data; keep them 0600.
 umask 0077

 TODAY=$(date '+%Y_%m_%d')
 BACKUPDIR=/nsm/backup
 BACKUPFILE="$BACKUPDIR/so-postgres-backup-$TODAY.sql.gz"
+TMPFILE="$BACKUPFILE.tmp"
 MAXBACKUPS=7
+LOGFILE=/opt/so/log/postgres/backup.log

-mkdir -p $BACKUPDIR
+log() {
+  echo "$(date '+%Y-%m-%d %H:%M:%S') $*" >> "$LOGFILE"
+}
+
+mkdir -p "$BACKUPDIR"
+
+# Remove any temp files left behind by a previously crashed run
+rm -f "$BACKUPDIR"/so-postgres-backup-*.sql.gz.tmp

 # Skip if already backed up today
 if [ -f "$BACKUPFILE" ]; then
@@ -27,13 +41,33 @@ if ! docker ps --format '{{.Names}}' | grep -q '^so-postgres$'; then
  exit 0
 fi

-# Dump all databases and roles, compress
-docker exec so-postgres pg_dumpall -U postgres | gzip > "$BACKUPFILE"
+# Always clean up the temp file on exit; the success path clears this trap
+# after the atomic rename so the finished backup is not deleted.
+trap 'rm -f "$TMPFILE"' EXIT

-# Retention cleanup
-NUMBACKUPS=$(find $BACKUPDIR -type f -name "so-postgres-backup*" | wc -l)
+# Dump all databases and roles, compress. Write to a temp file so the final
+# filename only ever appears for a complete, verified backup.
+if ! docker exec so-postgres pg_dumpall -U postgres | gzip > "$TMPFILE"; then
+  log "ERROR: pg_dumpall/gzip failed; backup aborted"
+  exit 1
+fi
+
+# Verify the compressed stream is intact before publishing it
+if ! gzip -t "$TMPFILE"; then
+  log "ERROR: backup failed gzip integrity check; backup aborted"
+  exit 1
+fi
+
+# Atomically publish the verified backup
+mv "$TMPFILE" "$BACKUPFILE"
+trap - EXIT
+log "OK: wrote $BACKUPFILE"
+
+# Retention cleanup (only reached after a successful backup). The glob is
+# restricted to finished backups so an in-progress .tmp can never be counted.
+NUMBACKUPS=$(find "$BACKUPDIR" -type f -name "so-postgres-backup-*.sql.gz" | wc -l)
 while [ "$NUMBACKUPS" -gt "$MAXBACKUPS" ]; do
-  OLDEST=$(find $BACKUPDIR -type f -name "so-postgres-backup*" -printf '%T+ %p\n' | sort | head -n 1 | awk -F" " '{print $2}')
+  OLDEST=$(find "$BACKUPDIR" -type f -name "so-postgres-backup-*.sql.gz" -printf '%T+ %p\n' | sort | head -n 1 | awk -F" " '{print $2}')
  rm -f "$OLDEST"
-  NUMBACKUPS=$(find $BACKUPDIR -type f -name "so-postgres-backup*" | wc -l)
+  NUMBACKUPS=$(find "$BACKUPDIR" -type f -name "so-postgres-backup-*.sql.gz" | wc -l)
 done
@@ -0,0 +1,110 @@
+#!/bin/bash
+set -e
+
+# Provision Telegraf state inside the so-postgres container.
+# Usage: so-telegraf-postgres <subcommand>
+#   create_db    Ensure the so_telegraf database exists.
+#   group_role   Provision the so_telegraf group role, telegraf/partman schemas,
+#                pg_partman, pg_cron, and the hourly partman maintenance job.
+#   user         Create or update a per-minion login role granted to so_telegraf.
+#                Env: ROLE_USER, ROLE_PASS.
+#   retention    Reconcile partman retention on telegraf parents.
+#                Env: RETENTION_DAYS.
+
+cmd="${1:?subcommand required}"
+
+case "$cmd" in
+  create_db)
+    if ! docker exec so-postgres psql -U postgres -tAc \
+        "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1; then
+      docker exec so-postgres psql -v ON_ERROR_STOP=1 -U postgres \
+        -c "CREATE DATABASE so_telegraf"
+    fi
+    ;;
+
+  group_role)
+    docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'so_telegraf') THEN
+        CREATE ROLE so_telegraf NOLOGIN;
+    END IF;
+END
+$$;
+GRANT CONNECT ON DATABASE so_telegraf TO so_telegraf;
+CREATE SCHEMA IF NOT EXISTS telegraf AUTHORIZATION so_telegraf;
+GRANT USAGE, CREATE ON SCHEMA telegraf TO so_telegraf;
+CREATE SCHEMA IF NOT EXISTS partman;
+CREATE EXTENSION IF NOT EXISTS pg_partman SCHEMA partman;
+CREATE EXTENSION IF NOT EXISTS pg_cron;
+-- Telegraf (running as so_telegraf) calls partman.create_parent()
+-- on first write of each metric, which needs USAGE on the partman
+-- schema, EXECUTE on its functions/procedures, and write access to
+-- partman.part_config so it can register new partitioned parents.
+GRANT USAGE, CREATE ON SCHEMA partman TO so_telegraf;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA partman TO so_telegraf;
+GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA partman TO so_telegraf;
+GRANT EXECUTE ON ALL PROCEDURES IN SCHEMA partman TO so_telegraf;
+-- partman creates per-parent template tables (partman.template_*) at
+-- runtime; default privileges extend DML/sequence access to them.
+ALTER DEFAULT PRIVILEGES IN SCHEMA partman
+    GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO so_telegraf;
+ALTER DEFAULT PRIVILEGES IN SCHEMA partman
+    GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO so_telegraf;
+-- Hourly partman maintenance. cron.schedule is idempotent by jobname.
+SELECT cron.schedule(
+  'telegraf-partman-maintenance',
+  '17 * * * *',
+  'CALL partman.run_maintenance_proc()'
+);
+EOSQL
+    ;;
+
+  user)
+    : "${ROLE_USER:?ROLE_USER is required}"
+    : "${ROLE_PASS:?ROLE_PASS is required}"
+    # psql does not substitute :vars inside dollar-quoted strings, so the
+    # conditional CREATE/ALTER is built outside any DO block and dispatched
+    # with \gexec. format() handles identifier/literal quoting.
+    docker exec -i so-postgres psql \
+      -v ON_ERROR_STOP=1 \
+      -v role_user="$ROLE_USER" \
+      -v role_pass="$ROLE_PASS" \
+      -U postgres -d so_telegraf <<'EOSQL'
+SELECT format(
+  CASE WHEN EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'role_user')
+       THEN 'ALTER ROLE %I WITH LOGIN PASSWORD %L'
+       ELSE 'CREATE ROLE %I WITH LOGIN PASSWORD %L'
+  END,
+  :'role_user',
+  :'role_pass'
+) \gexec
+GRANT CONNECT ON DATABASE so_telegraf TO :"role_user";
+GRANT so_telegraf TO :"role_user";
+EOSQL
+    ;;
+
+  retention)
+    : "${RETENTION_DAYS:?RETENTION_DAYS is required}"
+    # \gset + \if guards against a missing pg_partman without using a DO
+    # block (psql :var substitution doesn't reach into dollar-quoted code).
+    docker exec -i so-postgres psql \
+      -v ON_ERROR_STOP=1 \
+      -v retention_days="$RETENTION_DAYS" \
+      -U postgres -d so_telegraf <<'EOSQL'
+SELECT CASE WHEN EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_partman')
+            THEN 'true' ELSE 'false' END AS has_partman \gset
+\if :has_partman
+UPDATE partman.part_config
+SET retention = :'retention_days' || ' days',
+    retention_keep_table = false
+WHERE parent_table LIKE 'telegraf.%';
+\endif
+EOSQL
+    ;;
+
+  *)
+    echo "Unknown subcommand: $cmd" >&2
+    exit 1
+    ;;
+esac
@@ -3,12 +3,15 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-{% if data['id'].endswith('_hypervisor') and data['result'] == True %}
+{% set hid = data['id'] %}
+{% if hid|regex_match('^([A-Za-z0-9._-]{1,253})$')
+   and hid.endswith('_hypervisor')
+   and data['result'] == True %}

 {%   if data['act'] == 'accept' %}
 check_and_trigger:
  runner.setup_hypervisor.setup_environment:
-    - minion_id: {{ data['id'] }}
+    - minion_id: {{ hid }}
 {%   endif %}

 {%   if data['act'] == 'delete' %}
@@ -17,8 +20,7 @@ delete_hypervisor:
    - args:
      - mods: orch.delete_hypervisor
      - pillar:
-          minion_id: {{ data['id'] }}
+          minion_id: {{ hid }}
 {%   endif %}

 {% endif %}
-
@@ -1,7 +1,7 @@
 #!py

 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

@@ -9,30 +9,42 @@ import logging
 import os
 import pwd
 import grp
+import re
+
+log = logging.getLogger(__name__)
+
+PILLAR_ROOT = '/opt/so/saltstack/local/pillar/minions/'
+_VMNAME_RE = re.compile(r'^[A-Za-z0-9._-]{1,253}$')
+

 def run():
-  vm_name = data['kwargs']['name']
-  logging.error("createEmptyPillar reactor: vm_name: %s" % vm_name)
-  pillar_root = '/opt/so/saltstack/local/pillar/minions/'
+  vm_name = data.get('kwargs', {}).get('name', '')
+  if not _VMNAME_RE.match(str(vm_name)):
+    log.error("createEmptyPillar reactor: refusing unsafe vm_name=%r", vm_name)
+    return {}
+
+  log.info("createEmptyPillar reactor: vm_name: %s", vm_name)
  pillar_files = ['adv_' + vm_name + '.sls', vm_name + '.sls']

  try:
-    # Get socore user and group IDs
    socore_uid = pwd.getpwnam('socore').pw_uid
    socore_gid = grp.getgrnam('socore').gr_gid
+    pillar_root_real = os.path.realpath(PILLAR_ROOT)

    for f in pillar_files:
-      full_path = pillar_root + f
-      if not os.path.exists(full_path):
-        # Create empty file
-        os.mknod(full_path)
-        # Set ownership to socore:socore
-        os.chown(full_path, socore_uid, socore_gid)
-        # Set mode to 644 (rw-r--r--)
-        os.chmod(full_path, 0o640)
-        logging.error("createEmptyPillar reactor: created %s with socore:socore ownership and mode 644" % f)
+      full_path = os.path.join(PILLAR_ROOT, f)
+      resolved = os.path.realpath(full_path)
+      if os.path.dirname(resolved) != pillar_root_real:
+        log.error("createEmptyPillar reactor: refusing path outside pillar root: %s", resolved)
+        continue
+      if os.path.exists(resolved):
+        continue
+      os.mknod(resolved)
+      os.chown(resolved, socore_uid, socore_gid)
+      os.chmod(resolved, 0o640)
+      log.info("createEmptyPillar reactor: created %s with socore:socore ownership and mode 0640", f)

  except (KeyError, OSError) as e:
-    logging.error("createEmptyPillar reactor: Error setting ownership/permissions: %s" % str(e))
+    log.error("createEmptyPillar reactor: Error setting ownership/permissions: %s", e)

  return {}
@@ -1,18 +1,40 @@
+#!py
+
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-remove_key:
-  wheel.key.delete:
-    - args:
-      - match: {{ data['name'] }}
+import logging
+import re

-{{ data['name'] }}_pillar_clean:
-  runner.state.orchestrate:
-    - args:
-      - mods: orch.vm_pillar_clean
-      - pillar:
-          vm_name: {{ data['name'] }}
+log = logging.getLogger(__name__)

-{% do salt.log.info('deleteKey reactor: deleted minion key: %s' % data['name']) %}
+_VMNAME_RE = re.compile(r'^[A-Za-z0-9._-]{1,253}$')
+
+
+def run():
+  name = data.get('name', '')
+  if not _VMNAME_RE.match(str(name)):
+    log.error("deleteKey reactor: refusing unsafe name=%r", name)
+    return {}
+
+  log.info("deleteKey reactor: deleted minion key: %s", name)
+
+  return {
+    'remove_key': {
+      'wheel.key.delete': [
+        {'args': [
+          {'match': name},
+        ]},
+      ],
+    },
+    '%s_pillar_clean' % name: {
+      'runner.state.orchestrate': [
+        {'args': [
+          {'mods': 'orch.vm_pillar_clean'},
+          {'pillar': {'vm_name': name}},
+        ]},
+      ],
+    },
+  }
@@ -1,5 +1,6 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
-{% if GLOBALS.os == 'OEL' %}
+{# OL10 test path uses public repos; skip the SO repo state (which removes public repos and points at /nsm/repo) #}
+{% if GLOBALS.os == 'OEL' and GLOBALS.os_version|int == 9 %}
 include:
  - repo.client.oracle
 {% endif %}
@@ -17,7 +17,7 @@ engines:
                to:
                  'KAFKA':
                  - cmd.run:
-                      cmd: /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled True && /usr/sbin/so-config.py sync-yaml-mutation /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls replace kafka.enabled True --note "pillarWatch global.pipeline"
+                      cmd: /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled True
                  - cmd.run:
                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode' saltutil.kill_all_jobs
                  - cmd.run:
@@ -28,7 +28,7 @@ engines:
                to:
                  'REDIS':
                  - cmd.run:
-                      cmd: /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled False && /usr/sbin/so-config.py sync-yaml-mutation /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls replace kafka.enabled False --note "pillarWatch global.pipeline"
+                      cmd: /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled False
                  - cmd.run:
                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode' saltutil.kill_all_jobs
                  - cmd.run:
@@ -66,5 +66,5 @@ engines:
                  - cmd.run:
                      cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' state.apply kafka.disabled,kafka.reset
                  - cmd.run:
-                      cmd: /usr/sbin/so-yaml.py remove /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.reset && /usr/sbin/so-config.py sync-yaml-mutation /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls remove kafka.reset --note "pillarWatch kafka.reset"
+                      cmd: /usr/sbin/so-yaml.py remove /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.reset
      interval: 10
@@ -14,8 +14,6 @@

 include:
  - salt.minion
-  - salt.master.ext_pillar_postgres
-  - salt.master.pg_notify_pillar_engine
 {%   if 'vrt' in salt['pillar.get']('features', []) %}
  - salt.cloud
  - salt.cloud.reactor_config_hypervisor
@@ -1,24 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Deprecated. SOC/onionconfig owns the settings database now; this state only
-# removes the old so_pillar ext_pillar config if it was previously deployed.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-
-ext_pillar_postgres_config_absent:
-  file.absent:
-    - name: /etc/salt/master.d/ext_pillar_postgres.conf
-    - watch_in:
-      - service: salt_master_service
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}
@@ -1,37 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Deprecated. SOC/onionconfig owns the settings database now; this state only
-# removes the old so_pillar notify engine and reactor config if previously
-# deployed.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-
-pg_notify_pillar_engine_module_absent:
-  file.absent:
-    - name: /etc/salt/engines/pg_notify_pillar.py
-    - watch_in:
-      - service: salt_master_service
-
-pg_notify_pillar_engine_config_absent:
-  file.absent:
-    - name: /etc/salt/master.d/pg_notify_pillar_engine.conf
-    - watch_in:
-      - service: salt_master_service
-
-pg_notify_pillar_reactor_config_absent:
-  file.absent:
-    - name: /etc/salt/master.d/so_pillar_reactor.conf
-    - watch_in:
-      - service: salt_master_service
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}
@@ -24,11 +24,6 @@

 {% do SOCDEFAULTS.soc.config.server.modules.elastic.update({'username': GLOBALS.elasticsearch.auth.users.so_elastic_user.user, 'password': GLOBALS.elasticsearch.auth.users.so_elastic_user.pass}) %}

-{% if GLOBALS.postgres is defined and GLOBALS.postgres.auth is defined %}
-{%   set PG_ADMIN_PASS = salt['pillar.get']('secrets:postgres_pass', '') %}
-{% do SOCDEFAULTS.soc.config.server.modules.update({'postgres': {'hostUrl': GLOBALS.manager_ip, 'port': 5432, 'username': GLOBALS.postgres.auth.users.so_postgres_user.user, 'password': GLOBALS.postgres.auth.users.so_postgres_user.pass, 'adminUser': 'postgres', 'adminPassword': PG_ADMIN_PASS, 'dbname': 'securityonion', 'sslMode': 'require', 'assistantEnabled': true, 'esHostUrl': 'https://' ~ GLOBALS.manager_ip ~ ':9200', 'esUsername': GLOBALS.elasticsearch.auth.users.so_elastic_user.user, 'esPassword': GLOBALS.elasticsearch.auth.users.so_elastic_user.pass, 'esVerifyCert': false}}) %}
-{% endif %}
-
 {% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %}
 {% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'token': INFLUXDB_TOKEN}) %}
 {% for tool in SOCDEFAULTS.soc.config.server.client.tools %}
@@ -100,29 +100,6 @@ so-soc:
      - file: socusersroles
      - file: socclientsroles

-onionconfig_initial_import:
-  cmd.run:
-    - name: |
-        set -e
-        SOCONFIG=/usr/sbin/so-config.py
-        if [ ! -x "$SOCONFIG" ]; then
-          SOCONFIG=/opt/so/saltstack/default/salt/manager/tools/sbin/so-config.py
-        fi
-        for i in $(seq 1 60); do
-          if docker exec so-postgres pg_isready -h 127.0.0.1 -U postgres -q >/dev/null 2>&1 \
-             && curl -fsS --connect-timeout 2 http://{{ DOCKERMERGED.containers['so-soc'].ip }}:9822/ >/dev/null 2>&1; then
-            "$SOCONFIG" wait-schema --timeout 120
-            "$SOCONFIG" import-all --state-file /opt/so/state/onionconfig_initial_import.done
-            exit 0
-          fi
-          sleep 2
-        done
-        echo "so-soc or so-postgres did not become ready within 120s" >&2
-        exit 1
-    - unless: test -f /opt/so/state/onionconfig_initial_import.done
-    - require:
-      - docker_container: so-soc
-
 delete_so-soc_so-status.disabled:
  file.uncomment:
    - name: /opt/so/conf/so-status/so-status.conf
@@ -117,6 +117,121 @@ transformations:
      - type: logsource
        product: linux
        service: auth
+    # Maps M365 audit rules to Elastic Agent O365 integration logs
+    - id: m365_audit_field_mappings
+      type: field_name_mapping
+      mapping:
+        Operation: event.action
+        ResultStatus: event.outcome
+        ApplicationId: o365.audit.ApplicationId
+        ObjectId: o365.audit.ObjectId
+        RequestType: o365.audit.RequestType
+      rule_conditions:
+      - type: logsource
+        product: m365
+        service: audit
+    - id: m365_audit_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'o365.audit'
+        event.module: 'o365'
+      rule_conditions:
+      - type: logsource
+        product: m365
+        service: audit
+    # Maps M365 exchange rules to Elastic Agent O365 integration logs
+    - id: m365_exchange_field_mappings
+      type: field_name_mapping
+      mapping:
+        eventSource: event.provider
+        eventName: event.action
+        status: event.outcome
+      rule_conditions:
+      - type: logsource
+        product: m365
+        service: exchange
+    - id: m365_exchange_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'o365.audit'
+        event.module: 'o365'
+      rule_conditions:
+      - type: logsource
+        product: m365
+        service: exchange
+    # Maps M365 threat_management rules to Elastic Agent O365 integration logs
+    - id: m365_threat_management_field_mappings
+      type: field_name_mapping
+      mapping:
+        eventSource: event.provider
+        eventName: event.action
+        status: event.outcome
+      rule_conditions:
+      - type: logsource
+        product: m365
+        service: threat_management
+    - id: m365_threat_management_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'o365.audit'
+        event.module: 'o365'
+      rule_conditions:
+      - type: logsource
+        product: m365
+        service: threat_management
+    # Maps M365 threat_detection rules to Elastic Agent O365 integration logs
+    - id: m365_threat_detection_field_mappings
+      type: field_name_mapping
+      mapping:
+        eventSource: event.provider
+        eventName: event.action
+        status: event.outcome
+      rule_conditions:
+      - type: logsource
+        product: m365
+        service: threat_detection
+    - id: m365_threat_detection_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'o365.audit'
+        event.module: 'o365'
+      rule_conditions:
+      - type: logsource
+        product: m365
+        service: threat_detection
+    # Maps FortiGate event rules to Elastic Agent Fortinet integration logs
+    - id: fortigate_event_field_mappings
+      type: field_name_mapping
+      mapping:
+        action: fortinet.firewall.action
+        cfgpath: fortinet.firewall.cfgpath
+        cfgobj: fortinet.firewall.cfgobj
+        cfgattr: fortinet.firewall.cfgattr
+        devname: observer.name
+        devid: observer.serial_number
+        logid: event.code
+        type: fortinet.firewall.type
+        subtype: fortinet.firewall.subtype
+        level: log.level
+        vd: fortinet.firewall.vd
+        logdesc: fortinet.firewall.desc
+        user: user.name
+        ui: fortinet.firewall.ui
+        cfgtid: fortinet.firewall.cfgtid
+        msg: message
+      rule_conditions:
+      - type: logsource
+        product: fortigate
+        service: event
+    - id: fortigate_event_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'fortinet_fortigate.log'
+        event.module: 'fortinet_fortigate'
+      rule_conditions:
+      - type: logsource
+        product: fortigate
+        service: event
    # event.code should always be a string
    - id: convert_event_code_to_string
      type: convert_type
@@ -126,15 +241,36 @@ transformations:
          fields:
          - event.code
    # Maps process_creation rules to endpoint process creation logs
-    # This is an OS-agnostic mapping, to account for logs that don't specify source OS
    - id: endpoint_process_create_windows_add-fields
      type: add_condition
      conditions:
        event.category: 'process'
        event.type: 'start'
+        host.os.type: 'windows'
      rule_conditions:
      - type: logsource
        category: process_creation
+        product: windows
+    - id: endpoint_process_create_macos_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'process'
+        event.type: 'start'
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: process_creation
+        product: macos
+    - id: endpoint_process_create_linux_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'process'
+        event.type: 'start'
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: process_creation
+        product: linux
    # Maps file_event rules to endpoint file creation logs
    # This is an OS-agnostic mapping, to account for logs that don't specify source OS
    - id: endpoint_file_create_add-fields
@@ -261,7 +261,7 @@ strelka:
              priority: 5
              options:
                limit: 1000
-          'ScanLNK':
+          'ScanLnk':
            - positive:
                flavors:
                  - 'lnk_file'
@@ -15,7 +15,7 @@ from watchdog.observers import Observer
 from watchdog.events import FileSystemEventHandler

 with open("/opt/so/conf/strelka/filecheck.yaml", "r") as ymlfile:
-    cfg = yaml.load(ymlfile, Loader=yaml.Loader)
+    cfg = yaml.safe_load(ymlfile)

 extract_path = cfg["filecheck"]["extract_path"]
 historypath = cfg["filecheck"]["historypath"]
@@ -99,7 +99,7 @@ strelka:
          'ScanJpeg': *scannerOptions
          'ScanJson': *scannerOptions
          'ScanLibarchive': *scannerOptions
-          'ScanLNK': *scannerOptions
+          'ScanLnk': *scannerOptions
          'ScanLsb': *scannerOptions
          'ScanLzma': *scannerOptions
          'ScanMacho': *scannerOptions
@@ -1,6 +1,6 @@
 telegraf:
  enabled: False
-  output: BOTH
+  output: INFLUXDB
  config:
    interval: '30s'
    metric_batch_size: 1000
@@ -119,7 +119,7 @@ base:
    - kafka
    - pcap.cleanup

-  '*_manager or *_managerhype and G@saltversion:{{saltversion}} and not I@node_data:False':
+  '*_manager and G@saltversion:{{saltversion}} and not I@node_data:False':
    - match: compound
    - salt.master
    - registry
@@ -146,6 +146,32 @@ base:
    - stig
    - kafka

+  '*_managerhype and G@saltversion:{{saltversion}} and not I@node_data:False':
+    - match: compound
+    - salt.master
+    - registry
+    - nginx
+    - influxdb
+    - postgres
+    - strelka.manager
+    - soc
+    - kratos
+    - hydra
+    - firewall
+    - manager
+    - sensoroni
+    - telegraf
+    - backup.config_backup
+    - elasticsearch
+    - logstash
+    - redis
+    - elastic-fleet-package-registry
+    - kibana
+    - elastalert
+    - utility
+    - elasticfleet
+    - kafka
+
  '*_managerhype and I@features:vrt and G@saltversion:{{saltversion}}':
    - match: compound
    - manager.hypervisor
@@ -286,7 +312,6 @@ base:
    - libvirt
    - libvirt.images
    - elasticfleet.install_agent_grid
-    - stig
  
  '*_desktop and G@saltversion:{{saltversion}}':
    - sensoroni
@@ -31,6 +31,7 @@
    'so_model': INIT.GRAINS.get('sosmodel',''),
    'sensoroni_key': INIT.PILLAR.sensoroni.config.sensoronikey,
    'os': INIT.GRAINS.os,
+    'os_version': INIT.GRAINS.osmajorrelease,
    'os_family': INIT.GRAINS.os_family,
    'application_urls': {},
    'manager_roles': [
@@ -745,6 +745,56 @@ configure_network_sensor() {
 	return $err
 }

+configure_management_bond() {
+	local bond_name="bond1"
+	local bond_mode=${MBOND_MODE:-active-backup}
+
+	info "Setting up $bond_name management interface with mode $bond_mode"
+
+	if [[ ${#MBNICS[@]} -eq 0 ]]; then
+		error "[ERROR] No management bond NICs were selected."
+		fail_setup
+	fi
+
+	nmcli -t -f NAME con show | grep -Fxq "$bond_name"
+	local found_int=$?
+
+	if [[ $found_int != 0 ]]; then
+		nmcli con add type bond ifname "$bond_name" con-name "$bond_name" mode "$bond_mode" -- \
+			ipv6.method ignore \
+			connection.autoconnect yes >> "$setup_log" 2>&1
+	else
+		nmcli con mod "$bond_name" \
+			bond.options "mode=$bond_mode" \
+			ipv6.method ignore \
+			connection.autoconnect yes >> "$setup_log" 2>&1
+	fi
+
+	local err=0
+	for MBNIC in "${MBNICS[@]}"; do
+		local slave_name="$bond_name-slave-$MBNIC"
+
+		nmcli -t -f NAME con show | grep -Fxq "$slave_name"
+		found_int=$?
+
+		if [[ $found_int != 0 ]]; then
+			nmcli con add type ethernet ifname "$MBNIC" con-name "$slave_name" master "$bond_name" -- \
+				connection.autoconnect yes >> "$setup_log" 2>&1
+		else
+			nmcli con mod "$slave_name" \
+				connection.master "$bond_name" \
+				connection.slave-type bond \
+				connection.autoconnect yes >> "$setup_log" 2>&1
+		fi
+
+		nmcli con up "$slave_name" >> "$setup_log" 2>&1
+		local ret=$?
+		[[ $ret -eq 0 ]] || err=$ret
+	done
+
+	return $err
+}
+
 configure_hyper_bridge() {
 	info "Setting up hypervisor bridge"
 	info "Checking $MNIC ipv4.method is auto or manual"
@@ -853,14 +903,14 @@ detect_cloud() {

 detect_os() {
 	title "Detecting Base OS"
-	if [ -f /etc/redhat-release ] && grep -q "Red Hat Enterprise Linux release 9" /etc/redhat-release && [ -f /etc/oracle-release ]; then
+	if [ -f /etc/oracle-release ] && grep -qE "release (9|10)\b" /etc/oracle-release; then
 		OS=oracle
-		OSVER=9
+		OSVER=$(grep -oE "release [0-9]+" /etc/oracle-release | grep -oE "[0-9]+")
 		is_oracle=true
 		is_rpm=true
 		is_supported=true
 	else
-		info "This OS is not supported. Security Onion requires Oracle Linux 9."
+		info "This OS is not supported. Security Onion requires Oracle Linux 9 or 10."
 		fail_setup
 	fi

@@ -999,6 +1049,11 @@ filter_unused_nics() {
 			grep_string="$grep_string\|$BONDNIC"
 		done
 	fi
+	if [[ $MBNICS ]]; then
+		for BONDNIC in "${MBNICS[@]}"; do
+			grep_string="$grep_string\|$BONDNIC"
+		done
+	fi

 	# Finally, set filtered_nics to any NICs we aren't using (and ignore interfaces that aren't of use)
 	filtered_nics=$(ip link | awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2}' | grep -vwe "$grep_string"  | sed 's/ //g' | sed -r 's/(.*)(\.[0-9]+)@\1/\1\2/g')
@@ -1057,11 +1112,6 @@ generate_passwords(){
  POSTGRESPASS=$(get_random_value)
  SOCSRVKEY=$(get_random_value 64)
  IMPORTPASS=$(get_random_value)
-  # postsalt: salt-master connects to so_pillar.* as so_pillar_master, and the
-  # so-postgres container needs a symmetric key for pgcrypto-encrypted secrets.
-  # Both are generated here so they survive reinstall like the other secrets.
-  PILLARMASTERPASS=$(get_random_value)
-  SO_PILLAR_KEY=$(get_random_value 64)
 }

 generate_interface_vars() {
@@ -1393,7 +1443,7 @@ network_init() {
 	title "Initializing Network"
 	disable_ipv6
 	set_hostname
-	if [[ ( $is_iso || $is_desktop_iso ) ]]; then
+	if [[ $is_iso || $is_desktop_iso ]]; then
 		set_management_interface
 	fi
 }
@@ -1706,6 +1756,24 @@ remove_package() {
 	fi
 }

+ensure_pyyaml() {
+	title "Ensuring python3-pyyaml is installed"
+	if rpm -q python3-pyyaml >/dev/null 2>&1; then
+		info "python3-pyyaml already installed"
+		return 0
+	fi
+	info "python3-pyyaml not found, attempting to install"
+	set -o pipefail
+	dnf -y install python3-pyyaml 2>&1 | tee -a "$setup_log"
+	local result=$?
+	set +o pipefail
+	if [[ $result -ne 0 ]] || ! rpm -q python3-pyyaml >/dev/null 2>&1; then
+		error "Failed to install python3-pyyaml (exit=$result)"
+		fail_setup
+	fi
+	info "python3-pyyaml installed successfully"
+}
+
 # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and salt/salt/master.defaults.yaml and salt/salt/minion.defaults.yaml
 # CAUTION! SALT VERSION UDDATES - READ BELOW
 # When updating the salt version, also update the version in:
@@ -1715,6 +1783,15 @@ remove_package() {
 # - securityonion/salt/salt/minion.defaults.yaml

 securityonion_repo() {
+	if [[ "$OSVER" == "10" ]]; then
+		# TEST PATH: Oracle Linux 10 uses the public OL10 + EPEL + Docker CE repos.
+		# Keep the stock /etc/yum.repos.d/* in place, skip the SO mirror and local reposync.
+		gpg_rpm_import
+		logCmd "dnf -y install oracle-epel-release-el10"
+		logCmd "dnf -y config-manager --add-repo https://download.docker.com/linux/rhel/docker-ce.repo"
+		logCmd "dnf repolist"
+		return
+	fi
 	# Remove all the current repos
 	logCmd "dnf -v clean all"
 	logCmd "mkdir -vp /root/oldrepos"
@@ -1809,12 +1886,19 @@ saltify() {
 	info "Installing Salt $SALTVERSION"
 	chmod u+x ../salt/salt/scripts/bootstrap-salt.sh

+	# Normally Salt packages come from the SO mirror, so -r disables the bootstrap's own repo setup.
+	# On the OL10 test path there is no SO mirror, so let bootstrap configure the public Salt repo.
+	local saltrepoflag="-r"
+	if [[ "$OSVER" == "10" ]]; then
+		saltrepoflag=""
+	fi
+
 	if [[ $waitforstate ]]; then
 		# install all for a manager
-		retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -M -X stable $SALTVERSION" || fail_setup
+		retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh $saltrepoflag -M -X stable $SALTVERSION" || fail_setup
 	else
 		# just a minion
-		retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -X stable $SALTVERSION" || fail_setup
+		retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh $saltrepoflag -X stable $SALTVERSION" || fail_setup
 	fi

 	salt_install_module_deps
@@ -1858,34 +1942,7 @@ secrets_pillar(){
 		"secrets:"\
 		"  import_pass: $IMPORTPASS"\
 		"  influx_pass: $INFLUXPASS"\
-		"  pillar_master_pass: $PILLARMASTERPASS"\
 		"  postgres_pass: $POSTGRESPASS" > $local_salt_dir/pillar/secrets.sls
-  elif ! grep -q '^[[:space:]]*pillar_master_pass:' $local_salt_dir/pillar/secrets.sls; then
-	# Existing install pre-postsalt — append the new key without disturbing
-	# the values already on disk. Keys we already wrote stay; only the new
-	# pillar_master_pass is added.
-	info "Appending pillar_master_pass to existing Secrets Pillar"
-	if [ -z "$PILLARMASTERPASS" ]; then
-	  PILLARMASTERPASS=$(get_random_value)
-	fi
-	printf '  pillar_master_pass: %s\n' "$PILLARMASTERPASS" >> $local_salt_dir/pillar/secrets.sls
-  fi
-
-  # postsalt: write the so_pillar pgcrypto master key to a 0400 file owned by
-  # root. The key itself is never read by Salt — schema_pillar.sls loads it
-  # into the so-postgres container via ALTER ROLE so_pillar_secret_owner SET
-  # so_pillar.master_key = '<key>'; the file just lets the value survive
-  # container restarts.
-  if [ ! -f /opt/so/conf/postgres/so_pillar.key ]; then
-	info "Generating so_pillar pgcrypto master key"
-	mkdir -p /opt/so/conf/postgres
-	if [ -z "$SO_PILLAR_KEY" ]; then
-	  SO_PILLAR_KEY=$(get_random_value 64)
-	fi
-	umask 077
-	printf '%s' "$SO_PILLAR_KEY" > /opt/so/conf/postgres/so_pillar.key
-	chmod 0400 /opt/so/conf/postgres/so_pillar.key
-	chown root:root /opt/so/conf/postgres/so_pillar.key
  fi
 }

@@ -2116,8 +2173,12 @@ set_initial_firewall_access() {
 # Set up the management interface on the ISO
 set_management_interface() {
 	title "Setting up the main interface"
+	if [[ $MNIC == "bond1" ]]; then
+		configure_management_bond || fail_setup
+	fi
+
 	if [ "$address_type" = 'DHCP' ]; then
-		logCmd "nmcli con mod $MNIC connection.autoconnect yes"
+		logCmd "nmcli con mod $MNIC connection.autoconnect yes ipv4.method auto"
 		logCmd "nmcli con up $MNIC"
 		logCmd "nmcli -p connection show $MNIC"
 	else
@@ -66,6 +66,9 @@ set_timezone
 # Let's see what OS we are dealing with here
 detect_os

+# Ensure python3-pyyaml is available before any code that may need so-yaml/PyYAML
+ensure_pyyaml
+

 # Check to see if this is the setup type of "desktop".
 is_desktop=
@@ -845,18 +845,99 @@ whiptail_management_nic() {
 	[ -n "$TESTING" ] && return

 	filter_unused_nics
+	local management_nic_options=( "${nic_list_management[@]}" )
+	if [[ $is_iso || $is_desktop_iso ]]; then
+		management_nic_options+=( "BOND" "Configure a bonded management interface" )
+	fi

-	MNIC=$(whiptail --title "$whiptail_title" --menu "Please select the NIC you would like to use for management.\n\nUse the arrow keys to move around and the Enter key to select." 20 75 12 "${nic_list_management[@]}" 3>&1 1>&2 2>&3 )	
+	MNIC=$(whiptail --title "$whiptail_title" --menu "Please select the NIC you would like to use for management.\n\nUse the arrow keys to move around and the Enter key to select." 20 75 12 "${management_nic_options[@]}" 3>&1 1>&2 2>&3 )
 	local exitstatus=$?
 	whiptail_check_exitstatus $exitstatus

 	while [ -z "$MNIC" ]
 	do
-		MNIC=$(whiptail --title "$whiptail_title" --menu "Please select the NIC you would like to use for management.\n\nUse the arrow keys to move around and the Enter key to select." 22 75 12 "${nic_list_management[@]}" 3>&1 1>&2 2>&3 )	
+		MNIC=$(whiptail --title "$whiptail_title" --menu "Please select the NIC you would like to use for management.\n\nUse the arrow keys to move around and the Enter key to select." 22 75 12 "${management_nic_options[@]}" 3>&1 1>&2 2>&3 )
 		local exitstatus=$?
 		whiptail_check_exitstatus $exitstatus
 	done

+	if [[ $MNIC == "BOND" ]]; then
+		whiptail_management_bond
+	fi
+}
+
+whiptail_management_bond() {
+
+	[ -n "$TESTING" ] && return
+
+	MBOND_MODE=$(whiptail --title "$whiptail_title" --menu \
+	"Choose the bond mode for the management interface.\n\nThe management bond will be created as bond1." 20 75 7 \
+	"active-backup" "One active NIC with failover (recommended)" \
+	"balance-rr" "Round-robin transmit policy" \
+	"balance-xor" "Transmit based on selected hash policy" \
+	"broadcast" "Transmit everything on all slave interfaces" \
+	"802.3ad" "Dynamic link aggregation (requires switch support)" \
+	"balance-tlb" "Adaptive transmit load balancing" \
+	"balance-alb" "Adaptive load balancing" 3>&1 1>&2 2>&3)
+	local exitstatus=$?
+	whiptail_check_exitstatus $exitstatus
+
+	while [ -z "$MBOND_MODE" ]
+	do
+		MBOND_MODE=$(whiptail --title "$whiptail_title" --menu \
+		"Choose the bond mode for the management interface.\n\nThe management bond will be created as bond1." 20 75 7 \
+		"active-backup" "One active NIC with failover (recommended)" \
+		"balance-rr" "Round-robin transmit policy" \
+		"balance-xor" "Transmit based on selected hash policy" \
+		"broadcast" "Transmit everything on all slave interfaces" \
+		"802.3ad" "Dynamic link aggregation (requires switch support)" \
+		"balance-tlb" "Adaptive transmit load balancing" \
+		"balance-alb" "Adaptive load balancing" 3>&1 1>&2 2>&3)
+		local exitstatus=$?
+		whiptail_check_exitstatus $exitstatus
+	done
+
+	whiptail_management_bond_nics
+	MNIC="bond1"
+
+	export MBOND_MODE MNIC
+}
+
+whiptail_management_bond_nics() {
+
+	[ -n "$TESTING" ] && return
+
+	MBNICS=()
+	filter_unused_nics
+
+	MBNICS=$(whiptail --title "$whiptail_title" --checklist "Please add NICs to the Management Interface:" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3)
+	local exitstatus=$?
+	whiptail_check_exitstatus $exitstatus
+
+	while [ -z "$MBNICS" ]
+	do
+		MBNICS=$(whiptail --title "$whiptail_title" --checklist "Please add NICs to the Management Interface:" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3)
+		local exitstatus=$?
+		whiptail_check_exitstatus $exitstatus
+	done
+
+	MBNICS=$(echo "$MBNICS" | tr -d '"')
+
+	IFS=' ' read -ra MBNICS <<< "$MBNICS"
+
+	for bond_nic in "${MBNICS[@]}"; do
+		for dev_status in "${nmcli_dev_status_list[@]}"; do
+			if [[ $dev_status == "${bond_nic}:unmanaged" ]]; then
+				whiptail \
+					--title "$whiptail_title" \
+					--msgbox "$bond_nic is unmanaged by Network Manager. Please remove it from other network management tools then re-run setup." \
+					8 75
+				exit
+			fi
+		done
+	done
+
+	export MBNICS
 }

 whiptail_net_method() {
Author	SHA1	Message	Date
Mike Reeves	213afe4875	oracle 10 testing	2026-06-03 14:11:30 -04:00
Mike Reeves	2131e7d450	Merge pull request #15937 from Security-Onion-Solutions/hotfix/3.1.0 Hotfix/3.1.0	2026-05-28 10:20:53 -04:00
Mike Reeves	2a2d853ac4	Merge pull request #15936 from Security-Onion-Solutions/hotfix310 3.1.0 hotfix	2026-05-28 09:53:00 -04:00
Mike Reeves	5abd6de4b5	3.1.0 hotfix	2026-05-28 09:34:17 -04:00
Jorge Reyes	5599cce22c	Merge pull request #15934 from Security-Onion-Solutions/reyesj2-patch-1 keep logstash lumberjack pipeline name update unified	2026-05-27 13:37:41 -05:00
reyesj2	b2a82fec29	fix_logstash_0013_lumberjack_pipeline_name Before removing from apply_hotfix function first verify that older installs < 3.1.0 are still upgradable when referencing 'so/0013_input_lumberjack_fleet.conf' via pillar. Failure to do so will prevent logstash from starting	2026-05-27 13:24:23 -05:00
reyesj2	613eca52fc	update hotfix date	2026-05-27 13:24:10 -05:00
reyesj2	bf609a112e	LF	2026-05-27 12:21:44 -05:00
reyesj2	0b4a4de609	always run logstash pipeline rename	2026-05-27 12:21:22 -05:00
Jorge Reyes	ad376d2a43	Merge pull request #15930 from Security-Onion-Solutions/reyesj2-patch-1 check for stale logstash pipeline name in local pillar	2026-05-27 10:16:39 -05:00
reyesj2	0834998cca	usuable for next soup	2026-05-27 09:52:29 -05:00
reyesj2	473f93f0ee	check for stale logstash pipeline name in pillars	2026-05-27 09:33:15 -05:00
Jorge Reyes	7cc2e045fb	Merge pull request #15925 from Security-Onion-Solutions/reyesj2/soup-heavynode use multiple or combined input	2026-05-26 08:34:33 -05:00
Mike Reeves	6955ee73bf	Merge pull request #15924 from Security-Onion-Solutions/TOoSmOotH-patch-3 Add version number to HOTFIX file	2026-05-26 09:28:41 -04:00
Mike Reeves	c0272ddb81	Add version number to HOTFIX file	2026-05-26 09:24:10 -04:00
reyesj2	d72219c586	use multiple or combined input	2026-05-22 20:04:21 -05:00
Mike Reeves	c1d187599b	Merge pull request #15912 from Security-Onion-Solutions/3/dev 3.1.0	2026-05-21 15:41:50 -04:00
Mike Reeves	d87313db27	Merge pull request #15911 from Security-Onion-Solutions/3.1.0 3.1.0	2026-05-21 13:50:23 -04:00
Mike Reeves	141a61f5b5	3.1.0	2026-05-21 13:47:03 -04:00
Jorge Reyes	901cbf03e4	Merge pull request #15907 from Security-Onion-Solutions/reyesj2/es-verify-compat Verify compatibility for all ES nodes in the cluster	2026-05-20 14:16:41 -05:00
reyesj2	b485be4602	separate salt-key command from main es version compatiblity loop	2026-05-20 14:12:58 -05:00
reyesj2	7d13007aa9	block soup if all ES nodes are not online and reporting their ES version for compatibility check	2026-05-20 10:03:37 -05:00
reyesj2	d7a1b67095	use pipefail on heavynode versino command to pass through error	2026-05-20 09:16:57 -05:00
reyesj2	6c8997b28a	verify all heavynodes and all searchnodes are at compatible ES version before attempting an elasticsearch upgrade	2026-05-19 22:27:31 -05:00
Jorge Reyes	58f1d08ebe	Merge pull request #15902 from Security-Onion-Solutions/reyesj2/ea-fleet-sync sync elastic agent packages to fleet nodes	2026-05-19 11:08:48 -05:00
reyesj2	d0aa33a255	sync elastic agent packages to fleet nodes	2026-05-19 10:50:17 -05:00
Jorge Reyes	74b50f6009	Merge pull request #15899 from Security-Onion-Solutions/revert-15895-reyesj2/agentinstall Revert "use -verify flag during grid agent install to ensure agent health"	2026-05-16 10:01:58 -05:00
Jorge Reyes	e89c820b65	Revert "use -verify flag during grid agent install to ensure agent health"	2026-05-16 09:59:14 -05:00
Jorge Reyes	9ac05a6ad1	Merge pull request #15895 from Security-Onion-Solutions/reyesj2/agentinstall use -verify flag during grid agent install to ensure agent health	2026-05-15 12:58:09 -05:00
Jason Ertel	24ee3318bc	Merge pull request #15898 from Security-Onion-Solutions/jertel/logcheck exclude fps	2026-05-15 11:38:20 -04:00
Jason Ertel	ce566ba174	exclude fps	2026-05-15 11:36:46 -04:00
Mike Reeves	2635a60a8c	Merge pull request #15896 from Security-Onion-Solutions/quickfixes2 Make so-postgres-backup fail-safe against silent corruption	2026-05-15 09:32:15 -04:00
Mike Reeves	244a73b7a2	Make so-postgres-backup fail-safe against silent corruption The dump pipeline returned gzip's exit status, so a pg_dumpall that died mid-stream still produced a valid .gz holding a truncated dump, written straight to the final filename. The idempotency check then blocked retries for the day and the corrupt file counted toward retention, evicting a good backup each day until none remained. - set -o pipefail so a failed pg_dumpall fails the pipeline - dump to a .tmp file and atomically rename only after success, so the final filename appears only for a complete backup - gzip -t integrity check before publishing - trap-based cleanup of the temp file; sweep stale temps at startup - run retention only after a successful backup, with a glob restricted to finished backups - log timestamped OK/ERROR outcomes to /opt/so/log/postgres/backup.log	2026-05-15 08:48:54 -04:00
Mike Reeves	1189621ec5	Merge pull request #15893 from Security-Onion-Solutions/quickfixes2	2026-05-14 18:21:30 -04:00
reyesj2	d2524a593f	use -verify flag during grid agent install to ensure agent health	2026-05-14 17:12:02 -05:00
Josh Brower	f2ab2354fd	Merge pull request #15894 from Security-Onion-Solutions/3/nginx-fix Tweak for nginx upgrade	2026-05-14 23:20:57 +02:00
Mike Reeves	64731c73ba	Fix psql :var substitution in telegraf role and retention SQL psql does not substitute :var references inside dollar-quoted strings, so the DO blocks in the user and retention subcommands were receiving literal colons and failing (silently for user, via hide_output: True). Rewrite the conditional CREATE/ALTER ROLE with SELECT format(...) \\gexec and guard the retention UPDATE with \\gset + \\if.	2026-05-14 17:17:49 -04:00
Josh Brower	024fece607	Tweak for nginx upgrade	2026-05-14 17:08:57 -04:00
Mike Reeves	249b126312	Quote telegraf role env vars to survive YAML-special chars in passwords	2026-05-14 17:08:51 -04:00
Mike Reeves	8e38bff0c3	Rename telegraf_postgres.sh to so-telegraf-postgres	2026-05-14 16:55:53 -04:00
Mike Reeves	b9f2d56932	Consolidate telegraf postgres SQL into multi-mode script Replace inline psql heredocs in telegraf_users.sls with subcommand dispatcher telegraf_postgres.sh: create_db, group_role, user, retention.	2026-05-14 16:37:08 -04:00
Mike Reeves	03fa01a705	Move telegraf_role.sh to postgres tools/sbin	2026-05-14 16:18:01 -04:00
Mike Reeves	450eacca41	Move telegraf role provisioning to external script with env vars	2026-05-14 16:15:54 -04:00
Mike Reeves	b7a13899f7	Suppress output logging for postgres telegraf role provisioning	2026-05-14 15:56:04 -04:00
Mike Reeves	6f273d7d97	Rename init-users.sh to init-db.sh and update all references	2026-05-14 15:53:00 -04:00
Josh Brower	b328820c01	Merge pull request #15792 from Security-Onion-Solutions/3/strelkalnk Fix module name	2026-05-14 13:06:26 +02:00
Jorge Reyes	638aca97c8	Merge pull request #15877 from Security-Onion-Solutions/reyesj2-patch-1 update redis index template	2026-05-13 13:44:04 -05:00
Jorge Reyes	74a5c895e8	Merge pull request #15889 from Security-Onion-Solutions/reyesj2/zeek-ja4d add zeek.ja4d ingest pipeline	2026-05-13 13:43:56 -05:00
reyesj2	d56bf01823	add zeek.ja4d ingest pipeline	2026-05-13 12:32:54 -05:00
Mike Reeves	d29267d9c2	Merge pull request #15888 from Security-Onion-Solutions/TOoSmOotH-patch-1 Change Telegraf output from BOTH to INFLUXDB	2026-05-13 12:47:55 -04:00
Mike Reeves	72327285b2	Change Telegraf output from BOTH to INFLUXDB	2026-05-13 11:58:21 -04:00
Josh Patterson	cc7a237457	Merge pull request #15887 from Security-Onion-Solutions/m0duspwnens-patch-1 remove stig from hypervisor and managerhype	2026-05-13 10:57:58 -04:00
Josh Patterson	b068ad2b35	remove stig from hypervisor and managerhype	2026-05-13 10:53:11 -04:00
Jorge Reyes	b103f412b5	Merge pull request #15884 from Security-Onion-Solutions/reyesj2/strelkalnk rename strelka ScanLNK - ScanLnk	2026-05-13 09:46:52 -05:00
reyesj2	ef79c63858	Merge branch '3/dev' of github.com:Security-Onion-Solutions/securityonion into reyesj2/strelkalnk	2026-05-12 15:20:09 -05:00
reyesj2	01fb1aa156	check pillars for ScanLNK and rename to ScanLnk	2026-05-12 15:19:44 -05:00
Doug Burks	f19bdd7aae	Merge pull request #15883 from Security-Onion-Solutions/reyesj2/transformhealth use temp files to prevent jq arg too long	2026-05-12 15:36:12 -04:00
reyesj2	f637dc62d1	use temp files to prevent jq arg too long	2026-05-12 13:29:32 -05:00
Jorge Reyes	081f6fa1fb	Merge pull request #15878 from Security-Onion-Solutions/reyesj2/es-ingest-lag add ingest latency metrics	2026-05-12 10:21:04 -05:00
Josh Brower	d6d90d84cd	Merge pull request #15880 from Security-Onion-Solutions/feature/import-overrides Initial commit	2026-05-12 17:00:44 +02:00
Josh Brower	125610ed42	Additional test coverage	2026-05-12 10:11:22 -04:00
Josh Brower	306b0af4d0	Initial commit	2026-05-12 09:55:06 -04:00
reyesj2	492ae80da7	add ingest latency metrics	2026-05-11 16:51:38 -05:00
Jorge Reyes	4a2177c827	update redis index template missing redis integration component templates	2026-05-11 16:15:56 -05:00
Josh Brower	006ac31109	Merge pull request #15579 from marcopedrinazzi/3/dev New Sigma rules pipeline mapping for M365 and Fortigate	2026-05-11 21:03:53 +02:00
Josh Brower	49a643fff4	Merge pull request #15875 from Security-Onion-Solutions/3/sigma-fp-os proc_creation per OS type	2026-05-08 15:13:14 +02:00
Josh Brower	e1d830da76	proc_creation per OS type	2026-05-08 09:11:24 -04:00
Josh Brower	e847c46129	Merge pull request #15872 from Security-Onion-Solutions/3/soc-logs cleanup status code	2026-05-07 19:01:24 +02:00
Josh Brower	499f7102bd	cleanup status code	2026-05-07 11:27:49 -04:00
Josh Patterson	4bc19f91ce	Merge pull request #15867 from Security-Onion-Solutions/fixhype sanitize minion ids for hypervisor reactors / orchestration	2026-05-06 09:46:01 -04:00
Mike Reeves	4990d0ddea	Merge pull request #15866 from Security-Onion-Solutions/management-bond1 Management bond1	2026-05-05 17:17:58 -04:00
Mike Reeves	3e49322220	Allow preconfigured management bond in requirements	2026-05-05 15:35:12 -04:00
Mike Reeves	ecb92d43fc	Limit management bond setup to ISO installs	2026-05-05 15:30:09 -04:00
Mike Reeves	3b714db0bf	Show management bond option consistently	2026-05-05 15:22:40 -04:00
Mike Reeves	f17da4e68b	Add management bond setup option	2026-05-05 15:13:24 -04:00
Jorge Reyes	04cfc22e3f	Merge pull request #15864 from Security-Onion-Solutions/reyesj2/patch-2 update grok type conversion to convert processor	2026-05-05 13:58:39 -05:00
reyesj2	dceed421ae	update grok type conversion to convert processor	2026-05-05 13:41:00 -05:00
Josh Patterson	652ac5d61f	fix regex	2026-05-05 14:26:04 -04:00
Josh Patterson	f888a2ba6b	Merge remote-tracking branch 'origin/3/dev' into fixhype	2026-05-05 10:28:49 -04:00
Mike Reeves	8a1ee02335	Merge pull request #15846 from Security-Onion-Solutions/feature/ensure-pyyaml Ensure python3-pyyaml is installed before continuing setup	2026-05-05 10:24:25 -04:00
Josh Patterson	192f6cfe13	Merge remote-tracking branch 'origin/3/dev' into fixhype	2026-05-05 08:18:26 -04:00
Mike Reeves	5bca81d833	Merge pull request #15858 from Security-Onion-Solutions/security-fix Fix unsafe PyYAML load in filecheck	2026-05-04 16:16:40 -04:00
Josh Patterson	1c6574c694	ensure minion ids	2026-05-04 14:03:14 -04:00
Mike Reeves	b701664e04	Fix unsafe PyYAML load in filecheck	2026-05-04 12:09:35 -04:00
Jorge Reyes	bc64f1431d	Merge pull request #15857 from Security-Onion-Solutions/reyesj2/package-registry-health fleet package registry health check	2026-05-04 11:05:23 -05:00
reyesj2	2203037ce7	fleet package registry health check	2026-05-04 10:52:37 -05:00
Jorge Reyes	77a4ad877e	Merge pull request #15851 from Security-Onion-Solutions/reyesj2/integration-transforms	2026-05-01 14:11:12 -05:00
reyesj2	702b3585cc	excluding additional integration transform job failures	2026-05-01 12:57:59 -05:00
reyesj2	86966d2778	reauthorize unhealthy transform jobs using kibana 9.3.3 auth flow	2026-05-01 12:44:08 -05:00
Jorge Reyes	ce3ad3a895	Merge pull request #15844 from Security-Onion-Solutions/reyesj2/elastic-agent-warning update default elastic agent logging level to warning	2026-04-30 09:46:28 -05:00
Mike Reeves	3a4b7b50de	ensure python3-pyyaml is installed before continuing setup	2026-04-30 10:15:09 -04:00
reyesj2	39d0947102	update default elastic agent logging level to warning	2026-04-29 17:38:40 -05:00
Jorge Reyes	0085d9a353	Merge pull request #15842 from Security-Onion-Solutions/reyesj2-patch-1 so-elastic-fleet-outputs-update now checks for cert drift. Remove run…	2026-04-29 12:37:04 -05:00
Jorge Reyes	2f01ce3b23	so-elastic-fleet-outputs-update now checks for cert drift. Remove running --cert arg on cert change to prevent highstate from running outputs-update 2x	2026-04-29 12:33:28 -05:00
Mike Reeves	71b19c1b5f	Merge pull request #15840 from Security-Onion-Solutions/fix/import-postgres-firewall Open postgres in DOCKER-USER firewall everywhere influxdb is open	2026-04-29 09:20:03 -04:00
Mike Reeves	82e55ae87f	Open postgres on every hostgroup that opens influxdb The static defaults only listed postgres on each role's self-hostgroup, leaving sensor/searchnode/heavynode/receiver/fleet/idh/desktop/hypervisor hostgroups unable to reach the manager's so-postgres in distributed grids. A dynamic block in firewall/map.jinja added postgres to those hostgroups only when telegraf.output was switched to POSTGRES/BOTH, which left postgres unreachable by default. Mirror influxdb statically across manager/managerhype/managersearch/ standalone for every hostgroup that already lists influxdb, and drop the now-redundant telegraf-gated dynamic block from firewall/map.jinja.	2026-04-29 09:09:50 -04:00
Mike Reeves	3e02001544	Open postgres port for import role in DOCKER-USER firewall When so-postgres was wired in (`868cd1187`), the import role's firewall defaults were missed while every other manager-class role (manager, managerhype, managersearch, standalone, eval) had postgres added to their DOCKER-USER manager-hostgroup portgroups. As a result, on a fresh import install the so-postgres container starts but tcp/5432 is dropped at DOCKER-USER, so soc/kratos/telegraf can't reach it. Add postgres alongside the existing influxdb entry so import nodes match the other roles.	2026-04-29 08:48:45 -04:00
Mike Reeves	82f70bb53a	Merge pull request #15839 from Security-Onion-Solutions/fix/drop-postgres-soc-module-injection drop postgres module from soc defaults injection	2026-04-28 15:48:49 -04:00
Mike Reeves	2dcded6cca	drop postgres module from soc defaults injection The soc binary on 3/dev does not register a postgres module, so injecting postgres into soc.config.server.modules makes soc abort at launch with 'Module does not exist: postgres'. The soc-side module is staged on feature/postgres but is not landing this release. Drop the injection until the module ships; salt/postgres state and pillars are unchanged.	2026-04-28 15:46:56 -04:00
Mike Reeves	8ca59e6f0c	Merge pull request #15838 from Security-Onion-Solutions/fix/docker-refresh-multiarch-pull Fix/docker refresh multiarch pull	2026-04-28 15:14:27 -04:00
Mike Reeves	82dac82d15	drop platform/digest pull resolution The digest-pull logic was added to make `docker push` work for multi-arch upstream tags. Now that the push step is `docker buildx imagetools create` pinned to the gpg-verified RepoDigest, the registry-to-registry copy handles single- and multi-arch sources without help. Reverts the pull back to the original line and removes the unused PLATFORM_OS/_ARCH detection.	2026-04-28 14:54:25 -04:00
Mike Reeves	288a823edf	push images via buildx imagetools create Replaces `docker push` with a registry-to-registry copy. On Docker 29.x with the containerd image store, `docker push` of a freshly-pulled image hits a path that wraps single-platform manifests in a synthetic index and then can't push the layers it claims to reference, producing `NotFound: content digest ...` even when the image is fully present. Keep the local `docker tag` so so-image-pull's `docker images \| grep :5000` existence check continues to work.	2026-04-28 14:49:02 -04:00
Jorge Reyes	f9e3d30a71	Merge pull request #15837 from Security-Onion-Solutions/reyesj2/elastic-fleet-cert-check check current fleet policy cert against cert on disk	2026-04-28 13:47:55 -05:00
reyesj2	9cec79b299	check current fleet policy cert against cert on disk Co-authored-by: Copilot <copilot@github.com>	2026-04-28 13:34:39 -05:00
Mike Reeves	c86399327b	fix so-docker-refresh push for multi-arch source images docker pull of a multi-arch tag on Docker 29.x leaves the local tag pointing at the image index rather than the platform-specific manifest. The subsequent docker push then tries to push every sub-manifest the index references and fails on layers we never fetched. Resolve the local-platform manifest digest from the upstream index via docker buildx imagetools inspect, pull by that digest, and re-tag locally to the canonical tag. The signing flow and the existing tag/push to the embedded registry are unchanged.	2026-04-28 14:27:59 -04:00
Josh Brower	affede7f0a	Rename 'ScanLNK' to 'ScanLnk' in YAML config	2026-04-20 10:01:10 -04:00
Josh Brower	97366c0496	Rename 'ScanLNK' to 'ScanLnk' in defaults.yaml	2026-04-20 10:00:29 -04:00
Marco Pedrinazzi	d7e971a0fc	m365 and fortigate mappings sigma	2026-03-11 14:49:40 +01:00