Update so-nsm-clear

.github/DISCUSSION_TEMPLATE/3-0.yml

@@ -10,8 +10,6 @@ body:
       options:
         -
         - 3.0.0
-        - 3.1.0
-        - 3.2.0
         - Other (please provide detail below)
     validations:
       required: true

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 3.1.0-20260528 ISO image released on 2026/05/28
+### 3.0.0-20260331 ISO image released on 2026/03/31
 
 
 ### Download and Verify
 
-3.1.0-20260528 ISO image:  
+3.0.0-20260331 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-3.1.0-20260528.iso
+https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
  
-MD5: 9D6FF58DEEE24089D722C73169765B3E  
+MD5: ECD318A1662A6FDE0EF213F5A9BD4B07  
-SHA1: 2B8B816B6CEC3B7F96B3C5E040EBF502DD2C412F  
+SHA1: E55BE314440CCF3392DC0B06BC5E270B43176D9C  
-SHA256: 62FAB57E247C843D6A04F0796D8162C732B65D82FC3E4A59D087135B9FD32912  
+SHA256: 7FC47405E335CBE5C2B6C51FE7AC60248F35CBE504907B8B5A33822B23F8F4D5  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.1.0-20260528.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.1.0-20260528.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-3.1.0-20260528.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-3.1.0-20260528.iso.sig securityonion-3.1.0-20260528.iso
+gpg --verify securityonion-3.0.0-20260331.iso.sig securityonion-3.0.0-20260331.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Wed 27 May 2026 03:03:59 PM EDT using RSA key ID FE507013
+gpg: Signature made Mon 30 Mar 2026 06:22:14 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

HOTFIX

@@ -1 +0,0 @@
-

VERSION

@@ -1 +1 @@
-3.2.0
+3.0.0

pillar/elasticsearch/index_templates.sls

@@ -0,0 +1,2 @@
+elasticsearch:
+  index_settings:

pillar/telegraf/creds.sls

@@ -1,12 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Per-minion Telegraf Postgres credentials. so-telegraf-cred on the manager is
-# the single writer; it mutates /opt/so/saltstack/local/pillar/telegraf/creds.sls
-# under flock. Pillar_roots order (local before default) means the populated
-# copy shadows this default on any real grid; this file exists so the pillar
-# key is always defined on fresh installs and when no minions have creds yet.
-telegraf:
-  postgres_creds: {}

pillar/top.sls

@@ -17,7 +17,6 @@ base:
     - sensoroni.adv_sensoroni
     - telegraf.soc_telegraf
     - telegraf.adv_telegraf
-    - telegraf.creds
     - versionlock.soc_versionlock
     - versionlock.adv_versionlock
     - soc.license
@@ -39,9 +38,6 @@ base:
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
-    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
-    - postgres.auth
-    {% endif %}
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
     - kibana.secrets
     {% endif %}
@@ -64,8 +60,6 @@ base:
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
-    - postgres.soc_postgres
-    - postgres.adv_postgres
     - elasticsearch.nodes
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
@@ -103,12 +97,10 @@ base:
     - node_data.ips
     - secrets
     - healthcheck.eval
+    - elasticsearch.index_templates
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
-    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
-    - postgres.auth
-    {% endif %}
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
     - kibana.secrets
     {% endif %}
@@ -134,8 +126,6 @@ base:
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
-    - postgres.soc_postgres
-    - postgres.adv_postgres
     - backup.soc_backup
     - backup.adv_backup
     - zeek.soc_zeek
@@ -152,12 +142,10 @@ base:
     - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
+    - elasticsearch.index_templates
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
-    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
-    - postgres.auth
-    {% endif %}
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
     - kibana.secrets
     {% endif %}
@@ -172,8 +160,6 @@ base:
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
-    - postgres.soc_postgres
-    - postgres.adv_postgres
     - elasticsearch.nodes
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
@@ -270,12 +256,10 @@ base:
   '*_import':
     - node_data.ips
     - secrets
+    - elasticsearch.index_templates
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
-    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
-    - postgres.auth
-    {% endif %}
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
     - kibana.secrets
     {% endif %}
@@ -301,8 +285,6 @@ base:
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
-    - postgres.soc_postgres
-    - postgres.adv_postgres
     - zeek.soc_zeek
     - zeek.adv_zeek
     - bpf.soc_bpf

salt/_beacons/pillar_db.py

@@ -1,142 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Custom salt beacon that watches the SOC audit_settings table in postgres for
-# new settings changes and emits a beacon event per new row. This replaces the
-# inotify watch on /opt/so/saltstack/local/pillar -- instead of monitoring pillar
-# files on disk, we monitor the so_soc.audit_settings table that SOC writes to.
-#
-# Detection is poll-based with a monotonic `id` watermark persisted to
-# WATERMARK_FILE: each pass selects rows with id greater than the last id seen,
-# which makes it self-healing (a missed poll simply catches up on the next one).
-#
-# Each emitted event carries setting_id and node_id; the push_pillar reactor maps
-# setting_id -> app via pillar_push_map.yaml and writes a push intent, after which
-# the existing so-push-drainer / orch.push_batch pipeline takes over unchanged.
-
-import logging
-import os
-import subprocess
-
-log = logging.getLogger(__name__)
-
-WATERMARK_FILE = '/opt/so/state/pillar_db_watch.id'
-CONTAINER = 'so-postgres'
-DATABASE = 'so_soc'
-
-# Unaligned, tuples-only psql output with a field separator that cannot appear in
-# an id/setting_id/node_id, so we can split each row reliably.
-FIELD_SEP = '\x1f'
-
-
-def __virtual__():
-    return True
-
-
-def validate(config):
-    return True, 'valid'
-
-
-def _read_watermark():
-    # Returns the last processed id, or None if the watermark has not been seeded.
-    try:
-        with open(WATERMARK_FILE, 'r') as f:
-            return int((f.read() or '').strip())
-    except (IOError, ValueError):
-        return None
-
-
-def _write_watermark(value):
-    try:
-        os.makedirs(os.path.dirname(WATERMARK_FILE), exist_ok=True)
-        tmp = WATERMARK_FILE + '.tmp'
-        with open(tmp, 'w') as f:
-            f.write(str(int(value)))
-        os.rename(tmp, WATERMARK_FILE)
-    except OSError:
-        log.exception('pillar_db beacon: failed to persist watermark to %s', WATERMARK_FILE)
-
-
-def _query(sql):
-    # Run a query against so_soc inside the so-postgres container over the unix
-    # socket (trust auth, no password). Returns stdout on success, or None on any
-    # failure so the caller can no-op and retry on the next interval.
-    cmd = [
-        'docker', 'exec', CONTAINER,
-        'psql', '-U', 'postgres', '-d', DATABASE,
-        '-tA', '-F', FIELD_SEP, '-c', sql,
-    ]
-    try:
-        result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
-    except subprocess.TimeoutExpired:
-        log.warning('pillar_db beacon: psql timed out')
-        return None
-    except Exception:
-        log.exception('pillar_db beacon: failed to exec psql')
-        return None
-    if result.returncode != 0:
-        log.warning('pillar_db beacon: psql failed (rc=%s): %s',
-                    result.returncode, (result.stderr or '').strip())
-        return None
-    return result.stdout
-
-
-def beacon(config):
-    retval = []
-
-    watermark = _read_watermark()
-
-    # First run / missing watermark: seed to the current MAX(id) and emit nothing
-    # so we never replay the entire settings history into a fleetwide push.
-    if watermark is None:
-        seed = _query('SELECT COALESCE(MAX(id), 0) FROM audit_settings;')
-        if seed is None:
-            return retval  # postgres not ready yet; retry next interval
-        try:
-            _write_watermark(int((seed or '0').strip() or 0))
-        except ValueError:
-            log.warning('pillar_db beacon: could not parse MAX(id) seed: %r', seed)
-        return retval
-
-    rows = _query(
-        "SELECT id, setting_id, COALESCE(node_id, '') FROM audit_settings "
-        "WHERE id > %d ORDER BY id;" % watermark
-    )
-    if rows is None:
-        return retval
-
-    max_id = watermark
-    for line in rows.splitlines():
-        # Do NOT str.strip() the whole line: Python treats the \x1f field
-        # separator (and \x1c-\x1e) as whitespace, so stripping would eat an
-        # empty trailing node_id field and make the row look malformed.
-        if not line.strip():
-            continue
-        parts = line.split(FIELD_SEP)
-        if len(parts) < 3:
-            log.warning('pillar_db beacon: skipping malformed row: %r', line)
-            continue
-        try:
-            row_id = int(parts[0])
-        except ValueError:
-            log.warning('pillar_db beacon: skipping row with non-int id: %r', line)
-            continue
-        setting_id = parts[1]
-        node_id = parts[2]
-        retval.append({
-            'tag': 'audit_settings',
-            'id': row_id,
-            'setting_id': setting_id,
-            'node_id': node_id,
-        })
-        if row_id > max_id:
-            max_id = row_id
-
-    if max_id > watermark:
-        _write_watermark(max_id)
-        log.info('pillar_db beacon: emitted %d change(s), watermark %d -> %d',
-                 len(retval), watermark, max_id)
-
-    return retval

salt/allowed_states.map.jinja

@@ -29,14 +29,10 @@
     'manager',
     'nginx',
     'influxdb',
-    'postgres',
-    'postgres.auth',
     'soc',
     'kratos',
     'hydra',
     'elasticfleet',
-    'elasticfleet.manager',
-    'elasticsearch.cluster',
     'elastic-fleet-package-registry',
     'utility'
 ] %}
@@ -81,7 +77,7 @@
         ),
         'so-heavynode': (
             sensor_states +
-            ['elasticagent', 'elasticsearch', 'elasticsearch.cluster', 'logstash', 'redis', 'nginx']
+            ['elasticagent', 'elasticsearch', 'logstash', 'redis', 'nginx']
         ),
         'so-idh': (
             ['idh']

salt/backup/config_backup.sls

@@ -32,4 +32,3 @@ so_config_backup:
     - daymonth: '*'
     - month: '*'
     - dayweek: '*'
-

salt/backup/tools/sbin/so-config-backup.jinja

@@ -25,11 +25,9 @@ if [ ! -f $BACKUPFILE ]; then
   # Create empty backup file
   tar -cf $BACKUPFILE -T /dev/null
 
-  # Loop through all paths defined in global.sls, and append them to backup file if they exist
+  # Loop through all paths defined in global.sls, and append them to backup file
   {%- for LOCATION in BACKUPLOCATIONS %}
-  if [[ -d {{ LOCATION }} || -f {{ LOCATION }} ]]; then
   tar -rf $BACKUPFILE "${EXCLUSIONS[@]}" {{ LOCATION }}
-  fi
   {%- endfor %}
 
 fi

salt/ca/files/signing_policies.conf

@@ -54,20 +54,6 @@ x509_signing_policies:
     - extendedKeyUsage: serverAuth
     - days_valid: 820
     - copypath: /etc/pki/issued_certs/
-  postgres:
-    - minions: '*'
-    - signing_private_key: /etc/pki/ca.key
-    - signing_cert: /etc/pki/ca.crt
-    - C: US
-    - ST: Utah
-    - L: Salt Lake City
-    - basicConstraints: "critical CA:false"
-    - keyUsage: "critical keyEncipherment"
-    - subjectKeyIdentifier: hash
-    - authorityKeyIdentifier: keyid,issuer:always
-    - extendedKeyUsage: serverAuth
-    - days_valid: 820
-    - copypath: /etc/pki/issued_certs/
   elasticfleet:
     - minions: '*'
     - signing_private_key: /etc/pki/ca.key

salt/common/tools/sbin/so-image-common

@@ -31,7 +31,6 @@ container_list() {
       "so-hydra"
       "so-nginx"
       "so-pcaptools"
-      "so-postgres"
       "so-soc"
       "so-suricata"
       "so-telegraf"
@@ -56,7 +55,6 @@ container_list() {
       "so-logstash"
       "so-nginx"
       "so-pcaptools"
-      "so-postgres"
       "so-redis"
       "so-soc"
       "so-strelka-backend"
@@ -188,27 +186,8 @@ update_docker_containers() {
         if [ -z "$HOSTNAME" ]; then
           HOSTNAME=$(hostname)
         fi
-        docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$image $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 || {
+        docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$image $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 
-          echo "Unable to tag $image" >> "$LOG_FILE" 2>&1
+        docker push $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 
-          exit 1
-        }
-        # Push to the embedded registry via a registry-to-registry copy. Avoids
-        # `docker push`, which on Docker 29.x with the containerd image store
-        # represents freshly-pulled images as an index whose layer content
-        # isn't reachable through the push path. The local `docker tag` above
-        # is preserved so so-image-pull's `:5000` existence check still works.
-        # Pin to the digest already gpg-verified above so we copy exactly the
-        # bytes we approved.
-        local VERIFIED_REF
-        VERIFIED_REF=$(echo "$DOCKERINSPECT" | jq -r ".[0].RepoDigests[] | select(. | contains(\"$CONTAINER_REGISTRY\"))" | head -n 1)
-        if [ -z "$VERIFIED_REF" ] || [ "$VERIFIED_REF" = "null" ]; then
-          echo "Unable to determine verified digest for $image" >> "$LOG_FILE" 2>&1
-          exit 1
-        fi
-        docker buildx imagetools create --tag $HOSTNAME:5000/$IMAGEREPO/$image "$VERIFIED_REF" >> "$LOG_FILE" 2>&1 || {
-          echo "Unable to copy $image to embedded registry" >> "$LOG_FILE" 2>&1
-          exit 1
-        }
       fi
     else
       echo "There is a problem downloading the $image image. Details: " >> "$LOG_FILE" 2>&1 

salt/common/tools/sbin/so-log-check

@@ -165,8 +165,6 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading component template"  # false positive (elasticsearch index or template names contain 'error')
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading composable template" # false positive (elasticsearch composable template names contain 'error')
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error while parsing document for index \[.ds-logs-kratos-so-.*object mapping for \[file\]" # false positive (mapping error occuring BEFORE kratos index has rolled over in 2.4.210)
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|No such container"            # false positive (telegraf trying to run stats on an old container)
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|passwords do not match"       # false positive (automated hydra test)
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
@@ -229,7 +227,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|from NIC checksum offloading" # zeek reporter.log
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal"           # docker container getting recycled
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-(tychon|aws_billing|microsoft_defender_endpoint|armis|o365_metrics|microsoft_sentinel|snyk|cyera|island_browser).*user so_kibana lacks the required permissions \[(logs|metrics)-\1" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user. This error should not be seen on fresh ES 9.3.3 installs or after SO 3.1.0 with soups addition of check_transform_health_and_reauthorize()
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-(tychon|aws_billing|microsoft_defender_endpoint).*user so_kibana lacks the required permissions \[logs-\1" # Known issue with 3 integrations using kibana_system role vs creating unique api creds with proper permissions.
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown"             # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1
 fi
 

salt/common/tools/sbin/so-nsm-clear

@@ -66,11 +66,22 @@ delete_zeek() {
   ZEEK_LOG="/nsm/zeek/logs/"
   [ -d $ZEEK_LOG ] && so-zeek-stop && rm -rf $ZEEK_LOG/* && so-zeek-start
 }
+delete_import() {
+  IMPORT_DATA="/nsm/import/"
+  [ -d $IMPORT_DATA ] && rm -rf $IMPORT_DATA/*
+}
+delete_strelka() {
+  STRELKA_HISTORY_DATA="/nsm/strelka/history/"
+  STRELKA_PROCESSED_DATA="/nsm/strelka/processed/"
+  [ -d $STRELKA_HISTORY_DATA ] && rm -rf $STRELKA_HISTORY_DATA/*
+  [ -d $STRELKA_PROCESSED_DATA ] && rm -rf $STRELKA_PROCESSED_DATA/*
+}
+
 
 so-suricata-stop
 delete_pcap
 delete_suricata
 delete_zeek
 so-suricata-start
-
+delete_import
-
+delete_strelka

salt/common/tools/sbin_jinja/so-raid-status

@@ -9,7 +9,7 @@
 
 . /usr/sbin/so-common
 
-software_raid=("SOSMN" "SOSMN-DE02" "SOSSNNV" "SOSSNNV-DE02" "SOS10k-DE02" "SOS10KNV" "SOS10KNV-DE02" "SOS10KNV-DE02" "SOS2000-DE02" "SOS-GOFAST-LT-DE02" "SOS-GOFAST-MD-DE02" "SOS-GOFAST-HV-DE02" "HVGUEST")
+software_raid=("SOSMN" "SOSMN-DE02" "SOSSNNV" "SOSSNNV-DE02" "SOS10k-DE02" "SOS10KNV" "SOS10KNV-DE02" "SOS10KNV-DE02" "SOS2000-DE02" "SOS-GOFAST-LT-DE02" "SOS-GOFAST-MD-DE02" "SOS-GOFAST-HV-DE02")
 hardware_raid=("SOS1000" "SOS1000F" "SOSSN7200" "SOS5000" "SOS4000")
 
 {%- if salt['grains.get']('sosmodel', '') %}
@@ -87,11 +87,6 @@ check_boss_raid() {
 }
 
 check_software_raid() {
-  if [[ ! -f /proc/mdstat ]]; then
-    SWRAID=0
-    return
-  fi
-
   SWRC=$(grep "_" /proc/mdstat)
   if [[ -n $SWRC ]]; then
       # RAID is failed in some way
@@ -112,10 +107,8 @@ if [[ "$is_hwraid" == "true" ]]; then
 fi
 if [[ "$is_softwareraid" == "true" ]]; then
 	check_software_raid
-  if [ "$model" != "HVGUEST" ]; then
   check_boss_raid
 fi
-fi
 
 sum=$(($SWRAID + $BOSSRAID + $HWRAID))
 

salt/common/tools/sbin_jinja/so-salt-minion-check

@@ -1,3 +1,5 @@
+{% import_yaml 'salt/minion.defaults.yaml' as SALT_MINION_DEFAULTS -%}
+
 #!/bin/bash
 #
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
@@ -23,8 +25,7 @@ SYSTEM_START_TIME=$(date -d "$(</proc/uptime awk '{print $1}') seconds ago" +%s)
 LAST_HIGHSTATE_END=$([ -e "/opt/so/log/salt/lasthighstate" ] && date -r /opt/so/log/salt/lasthighstate +%s || echo 0)
 LAST_HEALTHCHECK_STATE_APPLY=$([ -e "/opt/so/log/salt/state-apply-test" ] && date -r /opt/so/log/salt/state-apply-test +%s || echo 0)
 # SETTING THRESHOLD TO ANYTHING UNDER 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default
-# THRESHOLD is derived from the global push highstate interval + 1 hour, so the minion-check grace period tracks the schedule automatically.
+THRESHOLD={{SALT_MINION_DEFAULTS.salt.minion.check_threshold}} #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted
-THRESHOLD=$(( ({{ salt['pillar.get']('global:push:highstate_interval_hours', 2) }} + 1) * 3600 )) #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted
 THRESHOLD_DATE=$((LAST_HEALTHCHECK_STATE_APPLY+THRESHOLD))
 
 logCmd() {

salt/docker/defaults.yaml

@@ -237,11 +237,3 @@ docker:
       extra_hosts: []
       extra_env: []
       ulimits: []
-    'so-postgres':
-      final_octet: 47
-      port_bindings:
-        - 0.0.0.0:5432:5432
-      custom_bind_mounts: []
-      extra_hosts: []
-      extra_env: []
-      ulimits: []

salt/docker_clean/init.sls

@@ -9,8 +9,7 @@
 prune_images:
   cmd.run:
     - name: so-docker-prune
-    - onlyif: command -v /usr/sbin/so-docker-prune >/dev/null 2>&1
+    - order: last
-    - order: 9000
 
 {% else %}
 

salt/elastalert/enabled.sls

@@ -19,7 +19,6 @@ wait_for_elasticsearch:
 so-elastalert:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastalert:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: elastalert
     - name: so-elastalert
     - user: so-elastalert

salt/elastic-fleet-package-registry/enabled.sls

@@ -15,7 +15,6 @@ include:
 so-elastic-fleet-package-registry:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-fleet-package-registry:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - name: so-elastic-fleet-package-registry
     - hostname: Fleet-package-reg-{{ GLOBALS.hostname }}
     - detach: True
@@ -52,16 +51,6 @@ so-elastic-fleet-package-registry:
       - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
     {%   endfor %}
     {% endif %}
-
-wait_for_so-elastic-fleet-package-registry:
-  http.wait_for_successful_query:
-    - name: "http://localhost:8080/health"
-    - status: 200
-    - wait_for: 300
-    - request_interval: 15
-    - require:
-      - docker_container: so-elastic-fleet-package-registry
-
 delete_so-elastic-fleet-package-registry_so-status.disabled:
   file.uncomment:
     - name: /opt/so/conf/so-status/so-status.conf

salt/elasticagent/enabled.sls

@@ -16,7 +16,6 @@ include:
 so-elastic-agent:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - name: so-elastic-agent
     - hostname: {{ GLOBALS.hostname }}
     - detach: True

salt/elasticfleet/content-defaults.map.jinja

@@ -1,123 +0,0 @@
-{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-  or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
-  this file except in compliance with the Elastic License 2.0. #}
-
-
-{% import_json '/opt/so/state/esfleet_content_package_components.json' as ADDON_CONTENT_PACKAGE_COMPONENTS %}
-{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
-{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
-
-{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
-{% set ADDON_CONTENT_INTEGRATION_DEFAULTS = {} %}
-{% set DEBUG_STUFF = {} %}
-
-{% for pkg in ADDON_CONTENT_PACKAGE_COMPONENTS %}
-{%   if pkg.name in CORE_ESFLEET_PACKAGES %}
-{#     skip core content packages #}
-{%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
-{#     generate defaults for each content package #}
-{%     if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0%}
-{%       for pattern in pkg.dataStreams %}
-{#         in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config)
-             title: generic is an artifact of that and is not in use #}
-{%         if pattern.title == "generic" %}
-{%           continue %}
-{%         endif %}
-{%         if "metrics-" in pattern.name %}
-{%           set integration_type = "metrics-" %}
-{%         elif "logs-" in pattern.name %}
-{%           set integration_type = "logs-" %}
-{%         else %}
-{%           set integration_type = "" %}
-{%         endif %}
-{#         on content integrations the component name is user defined at the time it is added to an agent policy #}
-{%         set component_name = pattern.title %}
-{%         set index_pattern = pattern.name %}
-{#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
-{%           set component_name_x = component_name.replace(".","_x_") %}
-{#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
-{%           set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %}
-{#           Default integration settings #}
-{%           set integration_defaults = {
-               "index_sorting": false,
-               "index_template": {
-                 "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
-                 "data_stream": {
-                   "allow_custom_routing": false,
-                   "hidden": false
-                 },
-                 "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
-                 "index_patterns": [index_pattern],
-                 "priority": 501,
-                 "template": {
-                   "settings": {
-                     "index": {
-                       "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
-                         "number_of_replicas": 0
-                     }
-                   }
-                 }
-               },
-               "policy": {
-                 "phases": {
-                   "cold": {
-                     "actions": {
-                        "allocate":{
-                          "number_of_replicas": ""
-                        },
-                       "set_priority": {"priority": 0}
-                     },
-                     "min_age": "60d"
-                   },
-                   "delete": {
-                     "actions": {
-                       "delete": {}
-                     },
-                     "min_age": "365d"
-                   },
-                   "hot": {
-                     "actions": {
-                       "rollover": {
-                         "max_age": "30d",
-                         "max_primary_shard_size": "50gb"
-                       },
-                       "forcemerge":{
-                         "max_num_segments": ""
-                       },
-                       "shrink":{
-                         "max_primary_shard_size": "",
-                         "method": "COUNT",
-                         "number_of_shards": ""
-                       },
-                       "set_priority": {"priority": 100}
-                      },
-                      "min_age": "0ms"
-                   },
-                   "warm": {
-                     "actions": {
-                        "allocate": {
-                          "number_of_replicas": ""
-                        },
-                        "forcemerge": {
-                          "max_num_segments": ""
-                        },
-                        "shrink":{
-                          "max_primary_shard_size": "",
-                          "method": "COUNT",
-                          "number_of_shards": ""
-                        },
-                       "set_priority": {"priority": 50}
-                     },
-                     "min_age": "30d"
-                   }
-                 }
-               }
-             } %}
-
-
-{%         do ADDON_CONTENT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
-{%       endfor %}
-{%     else %}
-{%     endif %}
-{%   endif %}
-{% endfor %}

salt/elasticfleet/defaults.yaml

@@ -1,6 +1,5 @@
 elasticfleet:
   enabled: False
-  patch_version: 9.3.3+build202604082258 # Elastic Agent specific patch release.
   enable_manager_output: True
   config:
     server:

salt/elasticfleet/enabled.sls

@@ -17,19 +17,65 @@ include:
   - logstash.ssl
   - elasticfleet.config
   - elasticfleet.sostatus
-{%- if GLOBALS.role != "so-fleet" %}
-  - elasticfleet.manager
-{%- endif %}
 
-{% if GLOBALS.role != "so-fleet" %}
+{% if grains.role not in ['so-fleet'] %}
 # Wait for Elasticsearch to be ready - no reason to try running Elastic Fleet server if ES is not ready
 wait_for_elasticsearch_elasticfleet:
   cmd.run:
     - name: so-elasticsearch-wait
 {% endif %}
 
-{% if GLOBALS.role == "so-fleet" %}
+# If enabled, automatically update Fleet Logstash Outputs
+{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval', 'so-fleet'] %}
+so-elastic-fleet-auto-configure-logstash-outputs:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-outputs-update
+    - retry:
+        attempts: 4
+        interval: 30
+
+{# Separate from above in order to catch elasticfleet-logstash.crt changes and force update to fleet output policy #}
+so-elastic-fleet-auto-configure-logstash-outputs-force:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-outputs-update --certs
+    - retry:
+        attempts: 4
+        interval: 30
+    - onchanges:
+        - x509: etc_elasticfleet_logstash_crt
+        - x509: elasticfleet_kafka_crt
+{% endif %}
+
+# If enabled, automatically update Fleet Server URLs & ES Connection
+{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-fleet'] %}
+so-elastic-fleet-auto-configure-server-urls:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-urls-update
+    - retry:
+        attempts: 4
+        interval: 30
+{% endif %}
+
+# Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
+{% if grains.role not in ['so-fleet'] %}
+so-elastic-fleet-auto-configure-elasticsearch-urls:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-es-url-update
+    - retry:
+        attempts: 4
+        interval: 30
+
+so-elastic-fleet-auto-configure-artifact-urls:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-artifacts-url-update
+    - retry:
+        attempts: 4
+        interval: 30
+
+{% endif %}
+
 # Sync Elastic Agent artifacts to Fleet Node
+{% if grains.role in ['so-fleet'] %}
 elasticagent_syncartifacts:
   file.recurse:
     - name: /nsm/elastic-fleet/artifacts/beats
@@ -42,7 +88,6 @@ elasticagent_syncartifacts:
 so-elastic-fleet:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - name: so-elastic-fleet
     - hostname: FleetServer-{{ GLOBALS.hostname }}
     - detach: True
@@ -104,6 +149,57 @@ so-elastic-fleet:
       - x509: etc_elasticfleet_crt
 {%   endif %}
 
+{%  if GLOBALS.role != "so-fleet" %}
+so-elastic-fleet-package-statefile:
+  file.managed:
+    - name: /opt/so/state/elastic_fleet_packages.txt
+    - contents: {{ELASTICFLEETMERGED.packages}}
+
+so-elastic-fleet-package-upgrade:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-package-upgrade
+    - retry:
+        attempts: 3
+        interval: 10
+    - onchanges:
+      - file: /opt/so/state/elastic_fleet_packages.txt
+
+so-elastic-fleet-integrations:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-integration-policy-load
+    - retry:
+        attempts: 3
+        interval: 10
+
+so-elastic-agent-grid-upgrade:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-agent-grid-upgrade
+    - retry:
+        attempts: 12
+        interval: 5
+
+so-elastic-fleet-integration-upgrade:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-integration-upgrade
+    - retry:
+        attempts: 3
+        interval: 10
+
+{# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
+so-elastic-fleet-addon-integrations:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
+
+{%   if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
+so-elastic-defend-manage-filters-file-watch:
+  cmd.run:
+    - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log
+    - onchanges:
+      - file: elasticdefendcustom
+      - file: elasticdefenddisabled
+{%    endif %}
+{%  endif %}
+
 delete_so-elastic-fleet_so-status.disabled:
   file.uncomment:
     - name: /opt/so/conf/so-status/so-status.conf

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json

@@ -9,22 +9,16 @@
   "namespace": "so",
   "description": "Zeek Import logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/zeek/logs/*.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "import",
             "pipeline": "",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -40,8 +34,7 @@
             "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
+            "include_lines": []
-            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json

@@ -15,25 +15,19 @@
     "version": ""
   },
   "name": "kratos-logs",
-  "namespace": "so",
   "description": "Kratos logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
+  "namespace": "so",
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/kratos/kratos.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "kratos",
             "pipeline": "kratos",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -54,10 +48,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
+            "include_lines": []
-            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json

@@ -9,22 +9,16 @@
   "namespace": "so",
   "description": "Zeek logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/zeek/logs/current/*.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "zeek",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"],
@@ -36,10 +30,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
+            "include_lines": []
-            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -5,7 +5,7 @@
   "package": {
     "name": "endpoint",
     "title": "Elastic Defend",
-    "version": "9.3.0",
+    "version": "9.0.2",
     "requires_root": true
   },
   "enabled": true,

salt/elasticfleet/files/integrations/grid-nodes_general/elastic-agent-monitor.json

@@ -6,23 +6,21 @@
   "name": "agent-monitor",
   "namespace": "",
   "description": "",
-  "policy_id": "so-grid-nodes_general",
   "policy_ids": [
     "so-grid-nodes_general"
   ],
+  "output_id": null,
   "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/agents/agent-monitor.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "agentmonitor",
             "pipeline": "elasticagent.monitor",
             "parsers": "",
@@ -36,16 +34,15 @@
             "ignore_older": "72h",
             "clean_inactive": -1,
             "harvester_limit": 0,
-            "fingerprint": false,
+            "fingerprint": true,
             "fingerprint_offset": 0,
-            "file_identity_native": true,
+            "fingerprint_length": 64,
+            "file_identity_native": false,
             "exclude_lines": [],
-            "include_lines": [],
+            "include_lines": []
-            "delete_enabled": false
+          }
         }
       }
     }
   }
-  },
-  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json

@@ -4,25 +4,19 @@
     "version": ""
   },
   "name": "hydra-logs",
-  "namespace": "so",
   "description": "Hydra logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
+  "namespace": "so",
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/hydra/hydra.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "hydra",
             "pipeline": "hydra",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -40,10 +34,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
+            "include_lines": []
-            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json

@@ -4,25 +4,19 @@
     "version": ""
   },
   "name": "idh-logs",
-  "namespace": "so",
   "description": "IDH integration",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
+  "namespace": "so",
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/idh/opencanary.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "idh",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -37,10 +31,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
+            "include_lines": []
-            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -4,32 +4,26 @@
     "version": ""
   },
   "name": "import-evtx-logs",
-  "namespace": "so",
   "description": "Import Windows EVTX logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
+  "namespace": "so",
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/evtx/*.json"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "import",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": [
               "\\.gz$"
             ],
             "include_files": [],
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.15.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.15.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.15.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.6.1\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.1.2\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.6.1\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.6.1\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.1.2\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
             ],
@@ -39,10 +33,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
+            "include_lines": []
-            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json

@@ -4,25 +4,19 @@
     "version": ""
   },
   "name": "import-suricata-logs",
-  "namespace": "so",
   "description": "Import Suricata logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
+  "namespace": "so",
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/suricata/eve*.json"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "import",
             "pipeline": "suricata.common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -38,10 +32,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
+            "include_lines": []
-            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json

@@ -4,18 +4,14 @@
     "version": ""
   },
   "name": "rita-logs",
-  "namespace": "so",
   "description": "RITA Logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
+  "namespace": "so",
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
@@ -23,8 +19,6 @@
               "/nsm/rita/exploded-dns.csv",
               "/nsm/rita/long-connections.csv"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "rita",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": [
@@ -39,10 +33,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
+            "include_lines": []
-            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json

@@ -4,25 +4,19 @@
     "version": ""
   },
   "name": "so-ip-mappings",
-  "namespace": "so",
   "description": "IP Description mappings",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
+  "namespace": "so",
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/custom-mappings/ip-descriptions.csv"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "hostnamemappings",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": [
@@ -38,10 +32,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
+            "include_lines": []
-            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json

@@ -4,25 +4,19 @@
     "version": ""
   },
   "name": "soc-auth-sync-logs",
-  "namespace": "so",
   "description": "Security Onion - Elastic Auth Sync - Logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
+  "namespace": "so",
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/sync.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -37,10 +31,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
+            "include_lines": []
-            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json

@@ -4,26 +4,20 @@
     "version": ""
   },
   "name": "soc-detections-logs",
-  "namespace": "so",
   "description": "Security Onion Console - Detections Logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
+  "namespace": "so",
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/detections_runtime-status_sigma.log",
               "/opt/so/log/soc/detections_runtime-status_yara.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -41,10 +35,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
+            "include_lines": []
-            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json

@@ -4,25 +4,19 @@
     "version": ""
   },
   "name": "soc-salt-relay-logs",
-  "namespace": "so",
   "description": "Security Onion - Salt Relay - Logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
+  "namespace": "so",
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/salt-relay.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -39,10 +33,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
+            "include_lines": []
-            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json

@@ -4,25 +4,19 @@
     "version": ""
   },
   "name": "soc-sensoroni-logs",
-  "namespace": "so",
   "description": "Security Onion - Sensoroni - Logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
+  "namespace": "so",
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/sensoroni/sensoroni.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -37,10 +31,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
+            "include_lines": []
-            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json

@@ -4,25 +4,19 @@
     "version": ""
   },
   "name": "soc-server-logs",
-  "namespace": "so",
   "description": "Security Onion Console Logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
+  "namespace": "so",
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/sensoroni-server.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -39,10 +33,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
+            "include_lines": []
-            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json

@@ -4,25 +4,19 @@
     "version": ""
   },
   "name": "strelka-logs",
-  "namespace": "so",
   "description": "Strelka Logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
+  "namespace": "so",
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/strelka/log/strelka.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "strelka",
             "pipeline": "strelka.file",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -37,10 +31,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
+            "include_lines": []
-            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json

@@ -4,25 +4,19 @@
     "version": ""
   },
   "name": "suricata-logs",
-  "namespace": "so",
   "description": "Suricata integration",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
+  "namespace": "so",
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/suricata/eve*.json"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "suricata",
             "pipeline": "suricata.common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -37,10 +31,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
+            "include_lines": []
-            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/input-defaults.map.jinja

@@ -1,123 +0,0 @@
-{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-  or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
-  this file except in compliance with the Elastic License 2.0. #}
-
-
-{% import_json '/opt/so/state/esfleet_input_package_components.json' as ADDON_INPUT_PACKAGE_COMPONENTS %}
-{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
-{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
-
-{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
-{% set ADDON_INPUT_INTEGRATION_DEFAULTS = {} %}
-{% set DEBUG_STUFF = {} %}
-
-{% for pkg in ADDON_INPUT_PACKAGE_COMPONENTS %}
-{%   if pkg.name in CORE_ESFLEET_PACKAGES %}
-{#     skip core input packages #}
-{%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
-{#     generate defaults for each input package #}
-{%     if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %}
-{%       for pattern in pkg.dataStreams %}
-{#         in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config)
-             title: generic is an artifact of that and is not in use #}
-{%         if pattern.title == "generic" %}
-{%           continue %}
-{%         endif %}
-{%         if "metrics-" in pattern.name %}
-{%           set integration_type = "metrics-" %}
-{%         elif "logs-" in pattern.name %}
-{%           set integration_type = "logs-" %}
-{%         else %}
-{%           set integration_type = "" %}
-{%         endif %}
-{#         on input integrations the component name is user defined at the time it is added to an agent policy #}
-{%         set component_name = pattern.title %}
-{%         set index_pattern = pattern.name %}
-{#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
-{%           set component_name_x = component_name.replace(".","_x_") %}
-{#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
-{%           set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %}
-{#           Default integration settings #}
-{%           set integration_defaults = {
-               "index_sorting": false,
-               "index_template": {
-                 "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
-                 "data_stream": {
-                   "allow_custom_routing": false,
-                   "hidden": false
-                 },
-                 "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
-                 "index_patterns": [index_pattern],
-                 "priority": 501,
-                 "template": {
-                   "settings": {
-                     "index": {
-                       "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
-                         "number_of_replicas": 0
-                     }
-                   }
-                 }
-               },
-               "policy": {
-                 "phases": {
-                   "cold": {
-                     "actions": {
-                        "allocate":{
-                          "number_of_replicas": ""
-                        },
-                       "set_priority": {"priority": 0}
-                     },
-                     "min_age": "60d"
-                   },
-                   "delete": {
-                     "actions": {
-                       "delete": {}
-                     },
-                     "min_age": "365d"
-                   },
-                   "hot": {
-                     "actions": {
-                       "rollover": {
-                         "max_age": "30d",
-                         "max_primary_shard_size": "50gb"
-                       },
-                       "forcemerge":{
-                         "max_num_segments": ""
-                       },
-                       "shrink":{
-                         "max_primary_shard_size": "",
-                         "method": "COUNT",
-                         "number_of_shards": ""
-                       },
-                       "set_priority": {"priority": 100}
-                      },
-                      "min_age": "0ms"
-                   },
-                   "warm": {
-                     "actions": {
-                        "allocate": {
-                          "number_of_replicas": ""
-                        },
-                        "forcemerge": {
-                          "max_num_segments": ""
-                        },
-                        "shrink":{
-                          "max_primary_shard_size": "",
-                          "method": "COUNT",
-                          "number_of_shards": ""
-                        },
-                       "set_priority": {"priority": 50}
-                     },
-                     "min_age": "30d"
-                   }
-                 }
-               }
-             } %}
-
-
-{%         do ADDON_INPUT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
-{%         do DEBUG_STUFF.update({integration_key: "Generating defaults for "+ pkg.name })%}
-{%       endfor %}
-{%     endif %}
-{%   endif %}
-{% endfor %}

salt/elasticfleet/integration-defaults.map.jinja

@@ -59,8 +59,8 @@
 {#     skip core integrations #}
 {%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
 {#     generate defaults for each integration #}
-{%     if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %}
+{%     if pkg.es_index_patterns is defined and pkg.es_index_patterns is not none %}
-{%       for pattern in pkg.dataStreams %}
+{%       for pattern in pkg.es_index_patterns %}
 {%         if "metrics-" in pattern.name %}
 {%           set integration_type = "metrics-" %}
 {%         elif "logs-" in pattern.name %}
@@ -75,27 +75,44 @@
 {%         if component_name in WEIRD_INTEGRATIONS %}
 {%           set component_name = WEIRD_INTEGRATIONS[component_name] %}
 {%         endif %}
+
+{#         create duplicate of component_name, so we can split generics from @custom component templates in the index template below and overwrite the default @package when needed
+           eg. having to replace unifiedlogs.generic@package with filestream.generic@package, but keep the ability to customize unifiedlogs.generic@custom and its ILM policy #}
+{%         set custom_component_name = component_name %}
+
+{#         duplicate integration_type to assist with sometimes needing to overwrite component templates with 'logs-filestream.generic@package' (there is no metrics-filestream.generic@package) #}
+{%         set generic_integration_type = integration_type %}
+
 {#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
 {%           set component_name_x = component_name.replace(".","_x_") %}
 {#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
 {%           set integration_key = "so-" ~ integration_type ~ component_name_x %}
 
+{#         if its a .generic template make sure that a .generic@package for the integration exists. Else default to logs-filestream.generic@package #}
+{%         if ".generic" in component_name and integration_type ~ component_name ~ "@package" not in INSTALLED_COMPONENT_TEMPLATES %}
+{#           these generic templates by default are directed to index_pattern of 'logs-generic-*', overwrite that here to point to eg gcp_pubsub.generic-* #}
+{%           set index_pattern = integration_type ~ component_name ~ "-*" %}
+{#           includes use of .generic component template, but it doesn't exist in installed component templates. Redirect it to filestream.generic@package #}
+{%           set component_name = "filestream.generic" %}
+{%           set generic_integration_type = "logs-" %}
+{%         endif %}
+
 {#           Default integration settings #}
 {%           set integration_defaults = {
                "index_sorting": false,
                "index_template": {
-                 "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
+                 "composed_of": [generic_integration_type ~ component_name ~ "@package", integration_type ~ custom_component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
                  "data_stream": {
                    "allow_custom_routing": false,
                    "hidden": false
                  },
-                 "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
+                 "ignore_missing_component_templates": [integration_type ~ custom_component_name ~ "@custom"],
                  "index_patterns": [index_pattern],
                  "priority": 501,
                  "template": {
                    "settings": {
                      "index": {
-                       "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
+                       "lifecycle": {"name": "so-" ~ integration_type ~ custom_component_name ~ "-logs"},
                          "number_of_replicas": 0
                      }
                    }

salt/elasticfleet/manager.sls

@@ -1,103 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls in allowed_states %}
-{%   from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
-
-include:
-  - elasticfleet.config
-
-# If enabled, automatically update Fleet Logstash Outputs
-{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration %}
-{%   if grains.role not in ['so-import', 'so-eval']%}
-so-elastic-fleet-auto-configure-logstash-outputs:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-outputs-update
-    - retry:
-        attempts: 4
-        interval: 30
-{%   endif %}
-
-# If enabled, automatically update Fleet Server URLs & ES Connection
-so-elastic-fleet-auto-configure-server-urls:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-urls-update
-    - retry:
-        attempts: 4
-        interval: 30
-{% endif %}
-
-# Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
-so-elastic-fleet-auto-configure-elasticsearch-urls:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-es-url-update
-    - retry:
-        attempts: 4
-        interval: 30
-
-so-elastic-fleet-auto-configure-artifact-urls:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-artifacts-url-update
-    - retry:
-        attempts: 4
-        interval: 30
-
-so-elastic-fleet-package-statefile:
-  file.managed:
-    - name: /opt/so/state/elastic_fleet_packages.txt
-    - contents: {{ELASTICFLEETMERGED.packages}}
-
-so-elastic-fleet-package-upgrade:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-package-upgrade
-    - retry:
-        attempts: 3
-        interval: 10
-    - onchanges:
-      - file: /opt/so/state/elastic_fleet_packages.txt
-
-so-elastic-fleet-integrations:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-integration-policy-load
-    - retry:
-        attempts: 3
-        interval: 10
-
-so-elastic-agent-grid-upgrade:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-agent-grid-upgrade
-    - retry:
-        attempts: 12
-        interval: 5
-
-so-elastic-fleet-integration-upgrade:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-integration-upgrade
-    - retry:
-        attempts: 3
-        interval: 10
-
-{# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
-so-elastic-fleet-addon-integrations:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
-
-{% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
-so-elastic-defend-manage-filters-file-watch:
-  cmd.run:
-    - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log
-    - onchanges:
-      - file: elasticdefendcustom
-      - file: elasticdefenddisabled
-{% endif %}
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/elasticfleet/tools/sbin/so-elastic-fleet-common

@@ -135,33 +135,9 @@ elastic_fleet_bulk_package_install() {
     fi
 }
 
-elastic_fleet_get_package_list_by_type() {
+elastic_fleet_installed_packages() {
-    if ! output=$(fleet_api "epm/packages"); then
+    if ! fleet_api "epm/packages/installed?perPage=500"; then
         return 1
-    else
-        is_integration=$(jq '[.items[] | select(.type=="integration") | .name ]' <<< "$output")
-        is_input=$(jq '[.items[] | select(.type=="input") | .name ]' <<< "$output")
-        is_content=$(jq '[.items[] | select(.type=="content") | .name ]' <<< "$output")
-        jq -n --argjson is_integration "${is_integration:-[]}" \
-              --argjson is_input "${is_input:-[]}" \
-              --argjson is_content "${is_content:-[]}" \
-              '{"integration": $is_integration,"input": $is_input, "content": $is_content}'
-    fi
-}
-elastic_fleet_installed_packages_components() {
-    package_type=${1,,}
-    if [[ "$package_type" != "integration" && "$package_type" != "input" && "$package_type" != "content" ]]; then
-        echo "Error: Invalid package type ${package_type}. Valid types are 'integration', 'input', or 'content'."
-        return 1
-    fi
-
-    packages_by_type=$(elastic_fleet_get_package_list_by_type)
-    packages=$(jq --arg package_type "$package_type" '.[$package_type]' <<< "$packages_by_type")
-
-    if ! output=$(fleet_api "epm/packages/installed?perPage=500"); then
-        return 1
-    else
-        jq -c --argjson packages "$packages" '[.items[] | select(.name | IN($packages[])) | {name: .name, dataStreams: .dataStreams}]' <<< "$output"
     fi
 }
 
@@ -240,7 +216,7 @@ elastic_fleet_policy_create() {
         --arg DESC "$DESC" \
         --arg TIMEOUT $TIMEOUT \
         --arg FLEETSERVER "$FLEETSERVER" \
-            '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER,"advanced_settings":{"agent_logging_level": "warning"}}'
+            '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER}'
         )
     # Create Fleet Policy
     if ! fleet_api "agent_policies" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade

@@ -5,13 +5,7 @@
 # this file except in compliance with the Elastic License 2.0.
 
 . /usr/sbin/so-common
-. /usr/sbin/so-elastic-fleet-common
 {%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
-{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
-{# Optionally override Elasticsearch version for Elastic Agent patch releases #}
-{%- if ELASTICFLEETDEFAULTS.elasticfleet.patch_version is defined %}
-{%-   do ELASTICSEARCHDEFAULTS.elasticsearch.update({'version': ELASTICFLEETDEFAULTS.elasticfleet.patch_version}) %}
-{%- endif %}
 
 # Only run on Managers
 if ! is_manager_node; then
@@ -20,8 +14,11 @@ if ! is_manager_node; then
 fi
 
 # Get current list of Grid Node Agents that need to be upgraded
-if ! RAW_JSON=$(fleet_api "agents?perPage=20&page=1&kuery=NOT%20agent.version%3A%20{{ELASTICSEARCHDEFAULTS.elasticsearch.version | urlencode }}%20AND%20policy_id%3A%20so-grid-nodes_%2A&showInactive=false&getStatusSummary=true" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
+RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=NOT%20agent.version%3A%20{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}%20AND%20policy_id%3A%20so-grid-nodes_%2A&showInactive=false&getStatusSummary=true" --retry 3 --retry-delay 30 --fail 2>/dev/null)
 
+# Check to make sure that the server responded with good data - else, bail from script
+CHECKSUM=$(jq -r '.page' <<< "$RAW_JSON")
+if [ "$CHECKSUM" -ne 1 ]; then
  printf "Failed to query for current Grid Agents...\n"
  exit 1
 fi
@@ -34,12 +31,10 @@ if [ "$OUTDATED_LIST" != '[]' ]; then
    printf "Initiating upgrades for $AGENTNUMBERS Agents to Elastic {{ELASTICSEARCHDEFAULTS.elasticsearch.version}}...\n\n"
 
    # Generate updated JSON payload
-   JSON_STRING=$(jq -n --arg ELASTICVERSION "{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}" --argjson UPDATELIST "$OUTDATED_LIST" '{"version": $ELASTICVERSION,"agents": $UPDATELIST }')
+   JSON_STRING=$(jq -n --arg ELASTICVERSION {{ELASTICSEARCHDEFAULTS.elasticsearch.version}} --arg UPDATELIST $OUTDATED_LIST '{"version": $ELASTICVERSION,"agents": $UPDATELIST }')
 
    # Update Node Agents
-   if ! fleet_api "agents/bulk_upgrade" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
+   curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "http://localhost:5601/api/fleet/agents/bulk_upgrade" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
-       printf "Failed to initiate Agent upgrades...\n"
-   fi
 else
     printf "No Agents need updates... Exiting\n\n"
     exit 0

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load

@@ -18,9 +18,7 @@ INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json
 BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
 BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
 BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json
-INTEGRATION_PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
+PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
-INPUT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_input_package_components.json
-CONTENT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_content_package_components.json
 COMPONENT_TEMPLATES=/opt/so/state/esfleet_component_templates.json
 
 PENDING_UPDATE=false
@@ -181,13 +179,10 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
         else
             echo "Elastic integrations don't appear to need installation/updating..."
         fi
-        # Write out file for generating index/component/ilm templates, keeping each package type separate
+        # Write out file for generating index/component/ilm templates
-        for package_type in "INTEGRATION" "INPUT" "CONTENT"; do
+        if latest_installed_package_list=$(elastic_fleet_installed_packages); then
-            if latest_installed_package_list=$(elastic_fleet_installed_packages_components "$package_type"); then
+            echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
-                outfile="${package_type}_PACKAGE_COMPONENTS"
-                echo $latest_installed_package_list > "${!outfile}"
         fi
-        done
         if retry 3 1 "so-elasticsearch-query / --fail --output /dev/null"; then
             # Refresh installed component template list
             latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.')

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update

@@ -235,16 +235,6 @@ function update_kafka_outputs() {
 
 {% endif %}
 
-# Compare the current Elastic Fleet certificate against what is on disk
-POLICY_CERT_SHA=$(jq -r '.item.ssl.certificate' <<< $RAW_JSON | openssl x509 -noout -sha256 -fingerprint)
-DISK_CERT_SHA=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt -noout -sha256 -fingerprint)
-
-if [[ "$POLICY_CERT_SHA" != "$DISK_CERT_SHA" ]]; then
-    printf "Certificate on disk doesn't match certificate in policy - forcing update\n"
-    UPDATE_CERTS=true
-    FORCE_UPDATE=true
-fi
-
 # Sort & hash the new list of Logstash Outputs
 NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}")
 NEW_HASH=$(sha256sum <<< "$NEW_LIST_JSON" | awk '{print $1}')

salt/elasticsearch/cluster.sls

@@ -1,181 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls in allowed_states %}
-{%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
-{%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %}
-{%   if GLOBALS.role != 'so-heavynode' %}
-{%     from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS, ADDON_INDICES %}
-{%   endif %}
-
-include:
-  - elasticsearch.enabled
-
-escomponenttemplates:
-  file.recurse:
-    - name: /opt/so/conf/elasticsearch/templates/component
-    - source: salt://elasticsearch/templates/component
-    - user: 930
-    - group: 939
-    - clean: True
-    - onchanges_in:
-      - file: so-elasticsearch-templates-reload
-    - show_changes: False
-
-# Clean up legacy and non-SO managed templates from the elasticsearch/templates/index/ directory
-so_index_template_dir:
-  file.directory:
-    - name: /opt/so/conf/elasticsearch/templates/index
-    - clean: True
-    {%- if SO_MANAGED_INDICES %}
-    - require:
-      {%- for index in SO_MANAGED_INDICES %}
-      - file: so_index_template_{{index}}
-      {%- endfor %}
-    {%- endif %}
-
-{%  if GLOBALS.role != "so-heavynode" %}
-# Clean up legacy and non-SO managed templates from the elasticsearch/templates/addon-index/ directory
-addon_index_template_dir:
-  file.directory:
-    - name: /opt/so/conf/elasticsearch/templates/addon-index
-    - clean: True
-    {%- if ADDON_INDICES %}
-    - require:
-      {%- for index in ADDON_INDICES %}
-      - file: addon_index_template_{{index}}
-      {%- endfor %}
-    {%- endif %}
-{%  endif %}
-
-# Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml)
-#   These index templates are for the core SO datasets and are always required
-{%  for index, settings in ES_INDEX_SETTINGS.items() %}
-{%    if settings.index_template is defined %}
-so_index_template_{{index}}:
-  file.managed:
-    - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json
-    - source: salt://elasticsearch/base-template.json.jinja
-    - defaults:
-        TEMPLATE_CONFIG: {{ settings.index_template }}
-    - template: jinja
-    - onchanges_in:
-      - file: so-elasticsearch-templates-reload
-{%    endif %}
-{%  endfor %}
-
-{%  if GLOBALS.role != "so-heavynode" %}
-# Auto-generate optional index templates for integration | input | content packages
-#   These index templates are not used by default (until user adds package to an agent policy).
-#   Pre-configured with standard defaults, and incorporated into SOC configuration for user customization.
-{%    for index,settings in ALL_ADDON_SETTINGS.items() %}
-{%      if settings.index_template is defined %}
-addon_index_template_{{index}}:
-  file.managed:
-    - name: /opt/so/conf/elasticsearch/templates/addon-index/{{ index }}-template.json
-    - source: salt://elasticsearch/base-template.json.jinja
-    - defaults:
-        TEMPLATE_CONFIG: {{ settings.index_template }}
-    - template: jinja
-    - show_changes: False
-    - onchanges_in:
-      - file: addon-elasticsearch-templates-reload
-{%       endif %}
-{%     endfor %}
-{%   endif %}
-
-{%  if GLOBALS.role in GLOBALS.manager_roles %}
-so-es-cluster-settings:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-cluster-settings
-    - cwd: /opt/so
-    - template: jinja
-    - require:
-      - docker_container: so-elasticsearch
-      - file: elasticsearch_sbin_jinja
-      - http: wait_for_so-elasticsearch
-{%  endif %}
-
-# heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)
-so-elasticsearch-ilm-policy-load:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-ilm-policy-load
-    - cwd: /opt/so
-    - require:
-      - docker_container: so-elasticsearch
-      - file: so-elasticsearch-ilm-policy-load-script
-    - onchanges:
-      - file: so-elasticsearch-ilm-policy-load-script
-
-so-elasticsearch-templates-reload:
-  file.absent:
-    - name: /opt/so/state/estemplates.txt
-
-addon-elasticsearch-templates-reload:
-  file.absent:
-    - name: /opt/so/state/addon_estemplates.txt
-
-# so-elasticsearch-templates-load will have its first successful run during the 'so-elastic-fleet-setup' script
-so-elasticsearch-templates:
-  cmd.run:
-{%-  if GLOBALS.role == "so-heavynode" %}
-    - name: /usr/sbin/so-elasticsearch-templates-load --heavynode
-{%-  else %}
-    - name: /usr/sbin/so-elasticsearch-templates-load
-{%-  endif %}
-    - cwd: /opt/so
-    - template: jinja
-    - require:
-      - docker_container: so-elasticsearch
-      - file: elasticsearch_sbin_jinja
-
-so-elasticsearch-pipelines:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }}
-    - require:
-      - docker_container: so-elasticsearch
-      - file: so-elasticsearch-pipelines-script
-
-so-elasticsearch-roles-load:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-roles-load
-    - cwd: /opt/so
-    - template: jinja
-    - require:
-      - docker_container: so-elasticsearch
-      - file: elasticsearch_sbin_jinja
-
-{%  if grains.role in ['so-managersearch', 'so-manager', 'so-managerhype'] %}
-{%    set ap = "absent" %}
-{%  endif %}
-{%  if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %}
-{%    if ELASTICSEARCHMERGED.index_clean %}
-{%      set ap = "present" %}
-{%    else %}
-{%      set ap = "absent" %}
-{%    endif %}
-{%  endif %}
-{%  if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
-so-elasticsearch-indices-delete:
-  cron.{{ap}}:
-    - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1
-    - identifier: so-elasticsearch-indices-delete
-    - user: root
-    - minute: '*/5'
-    - hour: '*'
-    - daymonth: '*'
-    - month: '*'
-    - dayweek: '*'
-{%  endif %}
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/elasticsearch/config.sls

@@ -66,8 +66,6 @@ so-elasticsearch-ilm-policy-load-script:
     - group: 939
     - mode: 754
     - template: jinja
-    - defaults:
-        GLOBALS: {{ GLOBALS }}
     - show_changes: False
 
 so-elasticsearch-pipelines-script:
@@ -93,13 +91,6 @@ estemplatedir:
     - group: 939
     - makedirs: True
 
-esaddontemplatedir:
-  file.directory:
-    - name: /opt/so/conf/elasticsearch/templates/addon-index
-    - user: 930
-    - group: 939
-    - makedirs: True
-
 esrolesdir:
   file.directory:
     - name: /opt/so/conf/elasticsearch/roles

salt/elasticsearch/defaults.yaml

@@ -1,6 +1,6 @@
 elasticsearch:
   enabled: false
-  version: 9.3.3
+  version: 9.0.8
   index_clean: true
   vm:
     max_map_count: 1048576
@@ -3958,13 +3958,10 @@ elasticsearch:
         - vulnerability-mappings
         - common-settings
         - common-dynamic-mappings
-        - logs-redis.log@package
-        - logs-redis.log@custom
         data_stream:
           allow_custom_routing: false
           hidden: false
-        ignore_missing_component_templates:
+        ignore_missing_component_templates: []
-        - logs-redis.log@custom
         index_patterns:
         - logs-redis.log*
         priority: 501

salt/elasticsearch/enabled.sls

@@ -10,6 +10,8 @@
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
+{%   set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
+{%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
 
 include:
   - ca
@@ -17,14 +19,10 @@ include:
   - elasticsearch.ssl
   - elasticsearch.config
   - elasticsearch.sostatus
-{%- if GLOBALS.role != "so-searchnode" %}
-  - elasticsearch.cluster
-{%- endif%}
 
 so-elasticsearch:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:{{ ELASTICSEARCHMERGED.version }}
-    - restart_policy: unless-stopped
     - hostname: elasticsearch
     - name: so-elasticsearch
     - user: elasticsearch
@@ -103,24 +101,134 @@ so-elasticsearch:
       - cmd: auth_users_roles_inode
       - cmd: auth_users_inode
 
-wait_for_so-elasticsearch:
-  http.wait_for_successful_query:
-    - name: "https://localhost:9200/"
-    - username: 'so_elastic'
-    - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}'
-    - ssl: True
-    - verify_ssl: False
-    - status: 200
-    - wait_for: 300
-    - request_interval: 15
-    - require:
-      - docker_container: so-elasticsearch
-
 delete_so-elasticsearch_so-status.disabled:
   file.uncomment:
     - name: /opt/so/conf/so-status/so-status.conf
     - regex: ^so-elasticsearch$
 
+{%   if GLOBALS.role != "so-searchnode" %}
+escomponenttemplates:
+  file.recurse:
+    - name: /opt/so/conf/elasticsearch/templates/component
+    - source: salt://elasticsearch/templates/component
+    - user: 930
+    - group: 939
+    - clean: True
+    - onchanges_in:
+      - file: so-elasticsearch-templates-reload
+    - show_changes: False
+      
+# Auto-generate templates from defaults file
+{%     for index, settings in ES_INDEX_SETTINGS.items() %}
+ {%      if settings.index_template is defined %}
+es_index_template_{{index}}:
+  file.managed:
+    - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json
+    - source: salt://elasticsearch/base-template.json.jinja
+    - defaults:
+      TEMPLATE_CONFIG: {{ settings.index_template }}
+    - template: jinja
+    - show_changes: False
+    - onchanges_in:
+      - file: so-elasticsearch-templates-reload
+{%       endif %}
+{%     endfor %}
+
+{%     if TEMPLATES %}
+# Sync custom templates to /opt/so/conf/elasticsearch/templates
+{%       for TEMPLATE in TEMPLATES %}
+es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
+  file.managed:
+    - source: salt://elasticsearch/templates/index/{{TEMPLATE}}
+{%         if 'jinja' in TEMPLATE.split('.')[-1] %}
+    - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}}
+    - template: jinja
+{%         else %}
+    - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1]}}
+{%         endif %}
+    - user: 930
+    - group: 939
+    - show_changes: False
+    - onchanges_in:
+      - file: so-elasticsearch-templates-reload
+{%       endfor %}
+{%     endif %}
+
+{%     if GLOBALS.role in GLOBALS.manager_roles %}
+so-es-cluster-settings:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-cluster-settings
+    - cwd: /opt/so
+    - template: jinja
+    - require:
+      - docker_container: so-elasticsearch
+      - file: elasticsearch_sbin_jinja
+{%     endif %}
+
+so-elasticsearch-ilm-policy-load:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-ilm-policy-load
+    - cwd: /opt/so
+    - require:
+      - docker_container: so-elasticsearch
+      - file: so-elasticsearch-ilm-policy-load-script
+    - onchanges:
+      - file: so-elasticsearch-ilm-policy-load-script
+
+so-elasticsearch-templates-reload:
+  file.absent:
+    - name: /opt/so/state/estemplates.txt
+
+so-elasticsearch-templates:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-templates-load
+    - cwd: /opt/so
+    - template: jinja
+    - require:
+      - docker_container: so-elasticsearch
+      - file: elasticsearch_sbin_jinja
+
+so-elasticsearch-pipelines:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }}
+    - require:
+      - docker_container: so-elasticsearch
+      - file: so-elasticsearch-pipelines-script
+
+so-elasticsearch-roles-load:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-roles-load
+    - cwd: /opt/so
+    - template: jinja
+    - require:
+      - docker_container: so-elasticsearch
+      - file: elasticsearch_sbin_jinja
+
+{%     if grains.role in ['so-managersearch', 'so-manager', 'so-managerhype'] %}
+{%       set ap = "absent" %}
+{%     endif %}
+{%     if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %}
+{%       if ELASTICSEARCHMERGED.index_clean %}
+{%         set ap = "present" %}
+{%       else %}
+{%         set ap = "absent" %}
+{%       endif %}
+{%     endif %}  
+{%     if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
+so-elasticsearch-indices-delete:
+  cron.{{ap}}:
+    - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1
+    - identifier: so-elasticsearch-indices-delete
+    - user: root
+    - minute: '*/5'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+{%     endif %}
+
+{%   endif %}
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/elasticsearch/files/ingest/common

@@ -63,8 +63,7 @@
     { "set":             { "if": "ctx.event?.dataset != null && !ctx.event.dataset.contains('.')", "field": "event.dataset", "value": "{{event.module}}.{{event.dataset}}" } },
     { "split":           { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "dataset_tag_temp" } },
     { "append":          { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}"  } },
-    { "grok":            { "if": "ctx.http?.response?.status_code instanceof String", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long}(?:\\s+%{GREEDYDATA})?"], "ignore_failure": true } },
+    { "grok":            { "if": "ctx.http?.response?.status_code != null", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long} %{GREEDYDATA}"]} },
-    { "convert":         { "if": "ctx.http?.response?.status_code != null && !(ctx.http.response.status_code instanceof Number)", "field": "http.response.status_code", "type": "long", "ignore_failure": true } },
     { "set":             { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
     { "remove":          { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } },
     { "pipeline": { "name": "global@custom", "ignore_missing_pipeline": true, "description": "[Fleet] Global pipeline for all data streams" } }

salt/elasticsearch/files/ingest/global@custom

@@ -177,84 +177,12 @@
                 "description": "Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"
             }
         },
-        {
-            "script": {
-                "description": "Snapshot event.ingested into _tmp.event_ingested_pre_fleet before .fleet_final_pipeline-1 overwrites it with ES ingest time",
-                "lang": "painless",
-                "if": "ctx.event?.ingested != null && ctx.event?.created == null",
-                "ignore_failure": true,
-                "source": "ctx.putIfAbsent('_tmp', [:]); ctx._tmp.event_ingested_pre_fleet = ctx.event.ingested;"
-            }
-        },
         {
             "pipeline": {
                 "name": ".fleet_final_pipeline-1",
                 "ignore_missing_pipeline": true
             }
         },
-        {
-            "script": {
-                "description": "Calculate time from Elastic Agent to Logstash.",
-                "lang": "painless",
-                "if": "ctx._tmp?.logstash_from_agent != null",
-                "ignore_failure": true,
-                "source": "ZonedDateTime start = ctx._tmp.event_ingested_pre_fleet != null ? ZonedDateTime.parse(ctx._tmp.event_ingested_pre_fleet) : ZonedDateTime.parse(ctx['@timestamp']); ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_elasticagent_to_logstash = ChronoUnit.SECONDS.between(start, ZonedDateTime.parse(ctx._tmp.logstash_from_agent));"
-            }
-        },
-        {
-            "script": {
-                "description": "Calculate time from Logstash to Redis",
-                "lang": "painless",
-                "if": "ctx._tmp?.logstash_from_agent != null && ctx._tmp?.logstash_to_redis != null",
-                "ignore_failure": true,
-                "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_logstash_to_redis = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_from_agent), ZonedDateTime.parse(ctx._tmp.logstash_to_redis));"
-            }
-        },
-        {
-            "script": {
-                "description": "Calculate time message spends in redis queue (logstash delay in pulling event).",
-                "lang": "painless",
-                "if": "ctx._tmp?.logstash_to_redis != null && ctx._tmp?.logstash_from_redis != null",
-                "ignore_failure": true,
-                "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_redis_to_logstash = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_to_redis), ZonedDateTime.parse(ctx._tmp.logstash_from_redis));"
-            }
-        },
-        {
-            "script": {
-                "description": "Calculate time from Logstash to Elasticsearch (after read from Redis).",
-                "lang": "painless",
-                "if": "ctx._tmp?.logstash_from_redis != null",
-                "ignore_failure": true,
-                "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_logstash_to_elasticsearch = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_from_redis), metadata().now);"
-            }
-        },
-        {
-            "script": {
-                "description": "Calculate time from Elastic Agent to Kafka.",
-                "lang": "painless",
-                "if": "ctx._tmp?.logstash_from_kafka != null && ctx._tmp?.logstash_from_agent == null",
-                "ignore_failure": true,
-                "source": "ZonedDateTime start = ctx._tmp.event_ingested_pre_fleet != null ? ZonedDateTime.parse(ctx._tmp.event_ingested_pre_fleet) : ZonedDateTime.parse(ctx['@timestamp']); ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_elasticagent_to_kafka = ChronoUnit.SECONDS.between(start, ZonedDateTime.parse(ctx._tmp.logstash_from_kafka));"
-            }
-        },
-        {
-            "script": {
-                "description": "Calculate time message spends in Kafka queue (logstash delay in pulling event).",
-                "lang": "painless",
-                "if": "ctx._tmp?.logstash_from_kafka != null && ctx.metadata?.kafka?.timestamp != null && ctx._tmp?.logstash_from_agent == null",
-                "ignore_failure": true,
-                "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_kafka_queue = ChronoUnit.SECONDS.between(ZonedDateTime.ofInstant(Instant.ofEpochMilli(Long.parseLong(ctx.metadata.kafka.timestamp.toString())), ZoneId.of('UTC')), ZonedDateTime.parse(ctx._tmp.logstash_from_kafka));"
-            }
-        },
-        {
-            "script": {
-                "description": "Calculate time from Logstash to Elasticsearch (after read from Kafka).",
-                "lang": "painless",
-                "if": "ctx._tmp?.logstash_from_kafka != null && ctx._tmp?.logstash_from_agent == null",
-                "ignore_failure": true,
-                "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_kafka_to_elasticsearch = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_from_kafka), metadata().now);"
-            }
-        },
         {
             "remove": {
                 "field": "event.agent_id_status",
@@ -274,8 +202,7 @@
                     "event.dataset_temp",
                     "dataset_tag_temp",
                     "module_temp",
-                    "datastream_dataset_temp",
+                    "datastream_dataset_temp"
-                    "_tmp"
                 ],
                 "ignore_missing": true,
                 "ignore_failure": true

salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1

@@ -10,28 +10,24 @@
   "processors": [
     {
       "set": {
-        "tag": "set_ecs_version_f5923549",
         "field": "ecs.version",
         "value": "8.17.0"
       }
     },
     {
       "set": {
-        "tag": "set_observer_vendor_ad9d35cc",
         "field": "observer.vendor",
         "value": "netgate"
       }
     },
     {
       "set": {
-        "tag": "set_observer_type_5dddf3ba",
         "field": "observer.type",
         "value": "firewall"
       }
     },
     {
       "rename": {
-        "tag": "rename_message_to_event_original_56a77271",
         "field": "message",
         "target_field": "event.original",
         "ignore_missing": true,
@@ -40,14 +36,12 @@
     },
     {
       "set": {
-        "tag": "set_event_kind_de80643c",
         "field": "event.kind",
         "value": "event"
       }
     },
     {
       "set": {
-        "tag": "set_event_timezone_4ca44cac",
         "field": "event.timezone",
         "value": "{{{_tmp.tz_offset}}}",
         "if": "ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local'"
@@ -55,7 +49,6 @@
     },
     {
       "grok": {
-        "tag": "grok_event_original_27d9c8c7",
         "description": "Parse syslog header",
         "field": "event.original",
         "patterns": [
@@ -79,7 +72,6 @@
     },
     {
       "date": {
-        "tag": "date__tmp_timestamp8601_to_timestamp_6ac9d3ce",
         "if": "ctx._tmp.timestamp8601 != null",
         "field": "_tmp.timestamp8601",
         "target_field": "@timestamp",
@@ -90,7 +82,6 @@
     },
     {
       "date": {
-        "tag": "date__tmp_timestamp_to_timestamp_f21e536e",
         "if": "ctx.event?.timezone != null && ctx._tmp?.timestamp != null",
         "field": "_tmp.timestamp",
         "target_field": "@timestamp",
@@ -104,7 +95,6 @@
     },
     {
       "grok": {
-        "tag": "grok_process_name_cef3d489",
         "description": "Set Event Provider",
         "field": "process.name",
         "patterns": [
@@ -117,83 +107,71 @@
     },
     {
       "pipeline": {
-        "tag": "pipeline_e16851a7",
+        "name": "logs-pfsense.log-1.23.1-firewall",
-        "name": "logs-pfsense.log-1.25.2-firewall",
         "if": "ctx.event.provider == 'filterlog'"
       }
     },
     {
       "pipeline": {
-        "tag": "pipeline_828590b5",
+        "name": "logs-pfsense.log-1.23.1-openvpn",
-        "name": "logs-pfsense.log-1.25.2-openvpn",
         "if": "ctx.event.provider == 'openvpn'"
       }
     },
     {
       "pipeline": {
-        "tag": "pipeline_9d37039c",
+        "name": "logs-pfsense.log-1.23.1-ipsec",
-        "name": "logs-pfsense.log-1.25.2-ipsec",
         "if": "ctx.event.provider == 'charon'"
       }
     },
     {
       "pipeline": {
-        "tag": "pipeline_ad56bbca",
+        "name": "logs-pfsense.log-1.23.1-dhcp",
-        "name": "logs-pfsense.log-1.25.2-dhcp",
+        "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)"
-        "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\", \"dnsmasq-dhcp\"].contains(ctx.event.provider)"
       }
     },
     {
       "pipeline": {
-        "tag": "pipeline_dd85553d",
+        "name": "logs-pfsense.log-1.23.1-unbound",
-        "name": "logs-pfsense.log-1.25.2-unbound",
         "if": "ctx.event.provider == 'unbound'"
       }
     },
     {
       "pipeline": {
-        "tag": "pipeline_720ed255",
+        "name": "logs-pfsense.log-1.23.1-haproxy",
-        "name": "logs-pfsense.log-1.25.2-haproxy",
         "if": "ctx.event.provider == 'haproxy'"
       }
     },
     {
       "pipeline": {
-        "tag": "pipeline_456beba5",
+        "name": "logs-pfsense.log-1.23.1-php-fpm",
-        "name": "logs-pfsense.log-1.25.2-php-fpm",
         "if": "ctx.event.provider == 'php-fpm'"
       }
     },
     {
       "pipeline": {
-        "tag": "pipeline_a0d89375",
+        "name": "logs-pfsense.log-1.23.1-squid",
-        "name": "logs-pfsense.log-1.25.2-squid",
         "if": "ctx.event.provider == 'squid'"
       }
     },
     {
       "pipeline": {
-        "tag": "pipeline_c2f1ed55",
+        "name": "logs-pfsense.log-1.23.1-snort",
-        "name": "logs-pfsense.log-1.25.2-snort",
         "if": "ctx.event.provider == 'snort'"
       }
     },
     {
       "pipeline": {
-        "tag":"pipeline_33db1c9e",
+        "name": "logs-pfsense.log-1.23.1-suricata",
-        "name": "logs-pfsense.log-1.25.2-suricata",
         "if": "ctx.event.provider == 'suricata'"
       }
     },
     {
       "drop": {
-        "tag": "drop_9d7c46f8",
+        "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)"
-        "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dnsmasq-dhcp\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)"
       }
     },
     {
       "append": {
-        "tag": "append_event_category_4780a983",
         "field": "event.category",
         "value": "network",
         "if": "ctx.network != null"
@@ -201,7 +179,6 @@
     },
     {
       "convert": {
-        "tag": "convert_source_address_to_source_ip_f5632a20",
         "field": "source.address",
         "target_field": "source.ip",
         "type": "ip",
@@ -211,7 +188,6 @@
     },
     {
       "convert": {
-        "tag": "convert_destination_address_to_destination_ip_f1388f0c",
         "field": "destination.address",
         "target_field": "destination.ip",
         "type": "ip",
@@ -221,7 +197,6 @@
     },
     {
       "set": {
-        "tag": "set_network_type_1f1d940a",
         "field": "network.type",
         "value": "ipv6",
         "if": "ctx.source?.ip != null && ctx.source.ip.contains(\":\")"
@@ -229,7 +204,6 @@
     },
     {
       "set": {
-        "tag": "set_network_type_69deca38",
         "field": "network.type",
         "value": "ipv4",
         "if": "ctx.source?.ip != null && ctx.source.ip.contains(\".\")"
@@ -237,7 +211,6 @@
     },
     {
       "geoip": {
-        "tag": "geoip_source_ip_to_source_geo_da2e41b2",
         "field": "source.ip",
         "target_field": "source.geo",
         "ignore_missing": true
@@ -245,7 +218,6 @@
     },
     {
       "geoip": {
-        "tag": "geoip_destination_ip_to_destination_geo_ab5e2968",
         "field": "destination.ip",
         "target_field": "destination.geo",
         "ignore_missing": true
@@ -253,7 +225,6 @@
     },
     {
       "geoip": {
-        "tag": "geoip_source_ip_to_source_as_28d69883",
         "ignore_missing": true,
         "database_file": "GeoLite2-ASN.mmdb",
         "field": "source.ip",
@@ -266,7 +237,6 @@
     },
     {
       "geoip": {
-        "tag": "geoip_destination_ip_to_destination_as_8a007787",
         "database_file": "GeoLite2-ASN.mmdb",
         "field": "destination.ip",
         "target_field": "destination.as",
@@ -279,7 +249,6 @@
     },
     {
       "rename": {
-        "tag": "rename_source_as_asn_to_source_as_number_a917047d",
         "field": "source.as.asn",
         "target_field": "source.as.number",
         "ignore_missing": true
@@ -287,7 +256,6 @@
     },
     {
       "rename": {
-        "tag": "rename_source_as_organization_name_to_source_as_organization_name_f1362d0b",
         "field": "source.as.organization_name",
         "target_field": "source.as.organization.name",
         "ignore_missing": true
@@ -295,7 +263,6 @@
     },
     {
       "rename": {
-        "tag": "rename_destination_as_asn_to_destination_as_number_3b459fcd",
         "field": "destination.as.asn",
         "target_field": "destination.as.number",
         "ignore_missing": true
@@ -303,7 +270,6 @@
     },
     {
       "rename": {
-        "tag": "rename_destination_as_organization_name_to_destination_as_organization_name_814bd459",
         "field": "destination.as.organization_name",
         "target_field": "destination.as.organization.name",
         "ignore_missing": true
@@ -311,14 +277,12 @@
     },
     {
       "community_id": {
-        "tag": "community_id_d2308e7a",
         "target_field": "network.community_id",
         "ignore_failure": true
       }
     },
     {
       "grok": {
-        "tag": "grok_observer_ingress_interface_name_968018d3",
         "field": "observer.ingress.interface.name",
         "patterns": [
           "%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}"
@@ -329,7 +293,6 @@
     },
     {
       "set": {
-        "tag": "set_network_vlan_id_efd4d96a",
         "field": "network.vlan.id",
         "copy_from": "observer.ingress.vlan.id",
         "ignore_empty_value": true
@@ -337,7 +300,6 @@
     },
     {
       "append": {
-        "tag": "append_related_ip_c1a6356b",
         "field": "related.ip",
         "value": "{{{destination.ip}}}",
         "allow_duplicates": false,
@@ -346,7 +308,6 @@
     },
     {
       "append": {
-        "tag": "append_related_ip_8121c591",
         "field": "related.ip",
         "value": "{{{source.ip}}}",
         "allow_duplicates": false,
@@ -355,7 +316,6 @@
     },
     {
       "append": {
-        "tag": "append_related_ip_53b62ed8",
         "field": "related.ip",
         "value": "{{{source.nat.ip}}}",
         "allow_duplicates": false,
@@ -364,7 +324,6 @@
     },
     {
       "append": {
-        "tag": "append_related_hosts_6f162628",
         "field": "related.hosts",
         "value": "{{{destination.domain}}}",
         "if": "ctx.destination?.domain != null"
@@ -372,7 +331,6 @@
     },
     {
       "append": {
-        "tag": "append_related_user_c036eec2",
         "field": "related.user",
         "value": "{{{user.name}}}",
         "if": "ctx.user?.name != null"
@@ -380,7 +338,6 @@
     },
     {
       "set": {
-        "tag": "set_network_direction_cb1e3125",
         "field": "network.direction",
         "value": "{{{network.direction}}}bound",
         "if": "ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/"
@@ -388,7 +345,6 @@
     },
     {
       "remove": {
-        "tag": "remove_a82e20f2",
         "field": [
           "_tmp"
         ],
@@ -397,21 +353,11 @@
     },
     {
       "script": {
-        "tag": "script_a7f2c062",
         "lang": "painless",
         "description": "This script processor iterates over the whole document to remove fields with null values.",
         "source": "void handleMap(Map map) {\n  for (def x : map.values()) {\n    if (x instanceof Map) {\n        handleMap(x);\n    } else if (x instanceof List) {\n        handleList(x);\n    }\n  }\n  map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n  for (def x : list) {\n      if (x instanceof Map) {\n          handleMap(x);\n      } else if (x instanceof List) {\n          handleList(x);\n      }\n  }\n}\nhandleMap(ctx);\n"
       }
     },
-    {
-      "append": {
-        "tag": "append_preserve_original_event_on_error",
-        "field": "tags",
-        "value": "preserve_original_event",
-        "allow_duplicates": false,
-        "if": "ctx.error?.message != null"
-      }
-    },
     {
       "pipeline": {
         "name": "global@custom",
@@ -459,14 +405,7 @@
     {
       "append": {
         "field": "error.message",
-        "value": "Processor '{{{ _ingest.on_failure_processor_type }}}' {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}'"
+        "value": "{{{ _ingest.on_failure_message }}}"
-      }
-    },
-    {
-      "append": {
-        "field": "tags",
-        "value": "preserve_original_event",
-        "allow_duplicates": false
       }
     }
   ]

salt/elasticsearch/files/ingest/zeek.ja4d

@@ -1,71 +0,0 @@
-{
-    "description": "zeek.ja4d",
-    "processors": [
-        {
-            "set": {
-                "field": "event.dataset",
-                "value": "ja4d"
-            }
-        },
-        {
-            "remove": {
-                "field": [
-                    "host"
-                ],
-                "ignore_failure": true
-            }
-        },
-        {
-            "json": {
-                "field": "message",
-                "target_field": "message2",
-                "ignore_failure": true
-            }
-        },
-        {
-            "rename": {
-                "field": "message2.ja4d",
-                "target_field": "hash.ja4d",
-                "ignore_missing": true,
-                "if": "ctx?.message2?.ja4d != null && ctx.message2.ja4d.length() > 0"
-            }
-        },
-        {
-            "rename": {
-                "field": "message2.client_mac",
-                "target_field": "host.mac",
-                "ignore_missing": true,
-                "if": "ctx?.message2?.client_mac != null && ctx.message2.client_mac.length() > 0"
-            }
-        },
-        {
-            "rename": {
-                "field": "message2.hostname",
-                "target_field": "host.hostname",
-                "ignore_missing": true,
-                "if": "ctx?.message2?.hostname != null && ctx.message2.hostname.length() > 0"
-            }
-        },
-        {
-            "rename": {
-                "field": "message2.requested_ip",
-                "target_field": "dhcp.requested_address",
-                "ignore_missing": true,
-                "if": "ctx?.message2?.requested_ip != null && ctx.message2.requested_ip.length() > 0"
-            }
-        },
-        {
-            "rename": {
-                "field": "message2.vendor_class_id",
-                "target_field": "zeek.ja4d.vendor_class_id",
-                "ignore_missing": true,
-                "if": "ctx?.message2?.vendor_class_id != null && ctx.message2.vendor_class_id.length() > 0"
-            }
-        },
-        {
-            "pipeline": {
-                "name": "zeek.common"
-            }
-        }
-    ]
-}

salt/elasticsearch/files/log4j2.properties

@@ -45,7 +45,3 @@ appender.rolling_json.strategy.action.condition.nested_condition.age = 1D
 rootLogger.level = info
 rootLogger.appenderRef.rolling.ref = rolling
 rootLogger.appenderRef.rolling_json.ref = rolling_json
-
-# Suppress NotEntitledException WARNs (ES 9.3.3 bug)
-logger.entitlement_security.name = org.elasticsearch.entitlement.runtime.policy.PolicyManager.x-pack-security.org.elasticsearch.security.org.elasticsearch.xpack.security
-logger.entitlement_security.level = error

salt/elasticsearch/template.map.jinja

@@ -14,43 +14,16 @@
 
 {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %}
 
-{% set ALL_ADDON_INTEGRATION_DEFAULTS = {} %}
-{% set ALL_ADDON_SETTINGS_ORIG = {} %}
-{% set ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES = {} %}
-{% set ALL_ADDON_SETTINGS = {} %}
 {# start generation of integration default index_settings #}
-{% if salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %}
+{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') and salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %}
-{#   import integration type defaults #}
+{%   set check_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %}
-{%   if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') %}
+{%   if check_package_components.size > 1 %}
-{%     set check_integration_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %}
-{%     if check_integration_package_components.size > 1 %}
 {%    from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}
-{%       do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INTEGRATION_DEFAULTS) %}
+{%    for index, settings in ADDON_INTEGRATION_DEFAULTS.items() %}
-{%     endif %}
+{%      do ES_INDEX_SETTINGS_ORIG.update({index: settings}) %}
-{%   endif %}
-
-{#   import input type defaults #}
-{%   if salt['file.file_exists']('/opt/so/state/esfleet_input_package_components.json') %}
-{%     set check_input_package_components = salt['file.stats']('/opt/so/state/esfleet_input_package_components.json') %}
-{%     if check_input_package_components.size > 1 %}
-{%       from 'elasticfleet/input-defaults.map.jinja' import ADDON_INPUT_INTEGRATION_DEFAULTS %}
-{%       do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INPUT_INTEGRATION_DEFAULTS) %}
-{%     endif %}
-{%   endif %}
-
-{#   import content type defaults #}
-{%   if salt['file.file_exists']('/opt/so/state/esfleet_content_package_components.json') %}
-{%     set check_content_package_components = salt['file.stats']('/opt/so/state/esfleet_content_package_components.json') %}
-{%     if check_content_package_components.size > 1 %}
-{%       from 'elasticfleet/content-defaults.map.jinja' import ADDON_CONTENT_INTEGRATION_DEFAULTS %}
-{%       do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_CONTENT_INTEGRATION_DEFAULTS) %}
-{%     endif %}
-{%   endif %}
-
-{%   for index, settings in ALL_ADDON_INTEGRATION_DEFAULTS.items() %}
-{%     do ALL_ADDON_SETTINGS_ORIG.update({index: settings}) %}
 {%    endfor %}
 {%   endif%}
+{% endif %}
 {# end generation of integration default index_settings #}
 
 {% set ES_INDEX_SETTINGS_GLOBAL_OVERRIDES = {} %}
@@ -58,43 +31,25 @@
 {%   do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}
 {% endfor %}
 
-{% if ALL_ADDON_SETTINGS_ORIG.keys() | length > 0 %}
-{%   for index in ALL_ADDON_SETTINGS_ORIG.keys() %}
-{%     do ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ALL_ADDON_SETTINGS_ORIG[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}
-{#     Explicitly excluding addon indices from ES_INDEX_SETTINGS_ORIG
-         When manager.soc_managed_annotations runs, new entries are added to the salt/elasticsearch/defaults.yaml file to support 'revert to default' functionality.
-         Subsequent map renders will then incorrectly include 'integration X' in 'ES_INDEX_SETTINGS_ORIG' due to being in the defaults.yaml file. #}
-{%     if index in ES_INDEX_SETTINGS_ORIG.keys() %}
-{%       do ES_INDEX_SETTINGS_ORIG.pop(index) %}
-{%     endif %}
-{%   endfor %}
-{% endif %}
-
 {% set ES_INDEX_SETTINGS = {} %}
-{% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS, EXCLUDE_INDICES=[]) %}
+{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
-
+{% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %}
-{% do GLOBAL_OVERRIDES.update(salt['defaults.merge'](GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
-{% for index, settings in GLOBAL_OVERRIDES.items() %}
-
-{%   if index in EXCLUDE_INDICES %}
-{%     continue %}
-{%   endif %}
 
 {#   prevent this action from being performed on custom defined indices. #}
 {#   the custom defined index is not present in either of the dictionaries and fails to reder. #}
-{%   if index in DEFINED_SETTINGS and index in GLOBAL_OVERRIDES %}
+{%   if index in ES_INDEX_SETTINGS_ORIG and index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES %}
 
 {#     dont merge policy from the global_overrides if policy isn't defined in the original index settingss #}
 {#     this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #}
-{%     if not DEFINED_SETTINGS[index].policy is defined and GLOBAL_OVERRIDES[index].policy is defined %}
+{%     if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
-{%       do GLOBAL_OVERRIDES[index].pop('policy') %}
+{%       do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %}
 {%     endif %}
 
 {#     this prevents and index from inderiting a policy phase from global overrides if it wasnt defined in the defaults. #}
-{%     if GLOBAL_OVERRIDES[index].policy is defined %}
+{%     if ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
-{%       for phase in GLOBAL_OVERRIDES[index].policy.phases.copy() %}
+{%       for phase in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.copy() %}
-{%         if DEFINED_SETTINGS[index].policy.phases[phase] is not defined %}
+{%         if ES_INDEX_SETTINGS_ORIG[index].policy.phases[phase] is not defined %}
-{%           do GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %}
+{%           do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %}
 {%         endif %}
 {%       endfor %}
 {%     endif %}
@@ -156,23 +111,5 @@
 {%     endfor %}
 {%   endif %}
 
-{%   do FINAL_INDEX_SETTINGS.update({index | replace("_x_", "."): GLOBAL_OVERRIDES[index]}) %}
+{%   do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %}
-{% endfor %}
-{% endmacro %}
-
-{# Exclude addon integrations from final ES_INDEX_SETTINGS #}
-{{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS, ALL_ADDON_SETTINGS_ORIG.keys() | list ) }}
-
-{# Exclude SO managed indices, otherwise ALL_ADDON_SETTINGS will include pillar values
-  of core integrations without merging defaults, resulting in an overlapping, but bad index template being generated. #}
-{{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS, ES_INDEX_SETTINGS_ORIG.keys() | list ) }}
-
-{% set SO_MANAGED_INDICES = [] %}
-{% for index, settings in ES_INDEX_SETTINGS.items() %}
-{%   do SO_MANAGED_INDICES.append(index) %}
-{% endfor %}
-
-{% set ADDON_INDICES = [] %}
-{% for index, settings in ALL_ADDON_SETTINGS.items() %}
-{%   do ADDON_INDICES.append(index) %}
 {% endfor %}

salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list

@@ -6,19 +6,8 @@
 # Elastic License 2.0.
 
 . /usr/sbin/so-common
-
+if [ "$1" == "" ]; then
-if [[ -z "$1" ]]; then
+    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template | jq '.component_templates[] |.name'| sort
-    if output=$(so-elasticsearch-query "_component_template" --retry 3 --retry-delay 1 --fail); then
-        jq '[.component_templates[] | .name] | sort' <<< "$output"
 else
-        echo "Failed to retrieve component templates from Elasticsearch."
+    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template/$1 | jq
-        exit 1
-    fi
-else
-    if output=$(so-elasticsearch-query "_component_template/$1" --retry 3 --retry-delay 1 --fail); then
-        jq <<< "$output"
-    else
-        echo "Failed to retrieve component template '$1' from Elasticsearch."
-        exit 1
-    fi
 fi

salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load

@@ -1,276 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-
-SO_STATEFILE_SUCCESS=/opt/so/state/estemplates.txt
-ADDON_STATEFILE_SUCCESS=/opt/so/state/addon_estemplates.txt
-ELASTICSEARCH_TEMPLATES_DIR="/opt/so/conf/elasticsearch/templates"
-SO_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/index"
-ADDON_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/addon-index"
-SO_LOAD_FAILURES=0
-ADDON_LOAD_FAILURES=0
-SO_LOAD_FAILURES_NAMES=()
-ADDON_LOAD_FAILURES_NAMES=()
-IS_HEAVYNODE="false"
-FORCE="false"
-VERBOSE="false"
-SHOULD_EXIT_ON_FAILURE="true"
-
-# If soup is running, ignore errors
-pgrep soup >/dev/null && SHOULD_EXIT_ON_FAILURE="false"
-
-while [[ $# -gt 0 ]]; do
-    case "$1" in
-    --heavynode)
-        IS_HEAVYNODE="true"
-        ;;
-    --force)
-        FORCE="true"
-        ;;
-    --verbose)
-        VERBOSE="true"
-        ;;
-    *)
-        echo "Usage: $0 [options]"
-        echo "Options:"
-        echo "  --heavynode     Only loads index templates specific to heavynodes"
-        echo "  --force     Force reload all templates regardless of statefiles (default: false)"
-        echo "  --verbose     Enable verbose output"
-        exit 1
-        ;;
-    esac
-    shift
-done
-
-load_template() {
-    local uri="$1"
-    local file="$2"
-
-    echo "Loading template file $file"
-    if ! output=$(retry 3 3 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"); then
-        echo "$output"
-
-        return 1
-
-    elif [[ "$VERBOSE" == "true" ]]; then
-        echo "$output"
-    fi
-
-}
-
-check_required_component_template_exists() {
-    local required
-    local missing
-    local file=$1
-
-    required=$(jq '[((.composed_of //[]) - (.ignore_missing_component_templates // []))[]]' "$file")
-    missing=$(jq -n --argjson required "$required" --argjson component_templates "$component_templates" '(($required) - ($component_templates))')
-
-    if [[ $(jq length <<<"$missing") -gt 0 ]]; then
-
-        return 1
-    fi
-}
-
-check_heavynode_compatiable_index_template() {
-    # The only templates that are relevant to heavynodes are from datasets defined in elasticagent/files/elastic-agent.yml.jinja.
-    # Heavynodes do not have fleet server packages installed and do not support elastic agents reporting directly to them.
-    local -A heavynode_index_templates=(
-        ["so-import"]=1
-        ["so-syslog"]=1
-        ["so-logs-soc"]=1
-        ["so-suricata"]=1
-        ["so-suricata.alerts"]=1
-        ["so-zeek"]=1
-        ["so-strelka"]=1
-    )
-
-    local template_name="$1"
-
-    if [[ ! -v heavynode_index_templates["$template_name"] ]]; then
-
-        return 1
-    fi
-
-}
-
-load_component_templates() {
-    local printed_name="$1"
-    local pattern="${ELASTICSEARCH_TEMPLATES_DIR}/component/$2"
-    local append_mappings="${3:-"false"}"
-
-    echo -e "\nLoading $printed_name component templates...\n"
-
-    if ! compgen -G "${pattern}/*.json" > /dev/null; then
-        echo "No $printed_name component templates found in ${pattern}, skipping."
-        return
-    fi
-
-    for component in "$pattern"/*.json; do
-        tmpl_name=$(basename "${component%.json}")
-
-        if [[ "$append_mappings" == "true" ]]; then
-            # avoid duplicating "-mappings" if it already exists in the component template filename
-            tmpl_name="${tmpl_name%-mappings}-mappings"
-        fi
-
-        if ! load_template "_component_template/${tmpl_name}" "$component"; then
-            SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
-            SO_LOAD_FAILURES_NAMES+=("$component")
-        fi
-    done
-}
-
-check_elasticsearch_responsive() {
-    # Cannot load templates if Elasticsearch is not responding.
-    #  NOTE: Slightly faster exit w/ failure than previous "retry 240 1" if there is a problem with Elasticsearch the
-    #    script should exit sooner rather than hang at the 'so-elasticsearch-templates' salt state.
-    retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" ||
-        fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot."
-}
-
-index_templates_exist() {
-    local templates_dir="$1"
-
-    if [[ ! -d "$templates_dir" ]]; then
-        return 1
-    fi
-
-    compgen -G "${templates_dir}/*.json" > /dev/null
-}
-
-should_load_addon_templates() {
-    if [[ "$IS_HEAVYNODE" == "true" ]]; then
-        return 1
-    fi
-
-    # Skip statefile checks when forcing template load
-    if [[ "$FORCE" != "true" ]]; then
-        if [[ ! -f "$SO_STATEFILE_SUCCESS" || -f "$ADDON_STATEFILE_SUCCESS" ]]; then
-            return 1
-        fi
-    fi
-
-    index_templates_exist "$ADDON_TEMPLATES_DIR"
-}
-
-if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]] && index_templates_exist "$SO_TEMPLATES_DIR"; then
-    check_elasticsearch_responsive
-
-    if [[ "$IS_HEAVYNODE" == "false" ]]; then
-        # TODO: Better way to check if fleet server is installed vs checking for Elastic Defend component template.
-        fleet_check="logs-endpoint.alerts@package"
-        if ! so-elasticsearch-query "_component_template/$fleet_check" --output /dev/null --retry 5 --retry-delay 3 --fail; then
-            # This check prevents so-elasticsearch-templates-load from running before so-elastic-fleet-setup has run.
-            echo -e "\nPackage $fleet_check not yet installed. Fleet Server may not be fully configured yet."
-            # Fleet Server is required because some SO index templates depend on components installed via
-            #  specific integrations eg Elastic Defend. These are components that we do not manually create / manage
-            #  via /opt/so/saltstack/salt/elasticsearch/templates/component/
-
-            exit 0
-        fi
-    fi
-
-    # load_component_templates "Name" "directory" "append '-mappings'?"
-    load_component_templates "ECS" "ecs" "true"
-    load_component_templates "Elastic Agent" "elastic-agent"
-    load_component_templates "Security Onion" "so"
-
-    component_templates=$(so-elasticsearch-component-templates-list)
-    echo -e "Loading Security Onion index templates...\n"
-    for so_idx_tmpl in "${SO_TEMPLATES_DIR}"/*.json; do
-        tmpl_name=$(basename "${so_idx_tmpl%-template.json}")
-
-        if [[ "$IS_HEAVYNODE" == "true" ]]; then
-            # TODO: Better way to load only heavynode specific templates
-            if ! check_heavynode_compatiable_index_template "$tmpl_name"; then
-                if [[ "$VERBOSE" == "true" ]]; then
-                    echo "Skipping over $so_idx_tmpl, template is not a heavynode specific index template."
-                fi
-
-                continue
-            fi
-        fi
-
-        if check_required_component_template_exists "$so_idx_tmpl"; then
-            if ! load_template "_index_template/$tmpl_name" "$so_idx_tmpl"; then
-                SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
-                SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl")
-            fi
-        else
-            echo "Skipping over $so_idx_tmpl due to missing required component template(s)."
-            SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
-            SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl")
-
-            continue
-        fi
-    done
-
-    if [[ $SO_LOAD_FAILURES -eq 0 ]]; then
-        echo "All Security Onion core templates loaded successfully."
-
-        touch "$SO_STATEFILE_SUCCESS"
-    else
-        echo "Encountered $SO_LOAD_FAILURES failure(s) loading templates:"
-        for failed_template in "${SO_LOAD_FAILURES_NAMES[@]}"; do
-            echo "  - $failed_template"
-        done
-        if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then
-            fail "Failed to load all Security Onion core templates successfully."
-        fi
-    fi
-elif ! index_templates_exist "$SO_TEMPLATES_DIR"; then
-    echo "No Security Onion core index templates found in ${SO_TEMPLATES_DIR}, skipping."
-elif [[ -f "$SO_STATEFILE_SUCCESS" ]]; then
-    echo "Security Onion core templates already loaded"
-fi
-
-# Start loading addon templates
-if should_load_addon_templates; then
-
-    check_elasticsearch_responsive
-
-    echo -e "\nLoading addon integration index templates...\n"
-    component_templates=$(so-elasticsearch-component-templates-list)
-
-    for addon_idx_tmpl in "${ADDON_TEMPLATES_DIR}"/*.json; do
-        tmpl_name=$(basename "${addon_idx_tmpl%-template.json}")
-
-        if check_required_component_template_exists "$addon_idx_tmpl"; then
-            if ! load_template "_index_template/${tmpl_name}" "$addon_idx_tmpl"; then
-                ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1))
-                ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl")
-            fi
-        else
-            echo "Skipping over $addon_idx_tmpl due to missing required component template(s)."
-            ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1))
-            ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl")
-
-            continue
-        fi
-    done
-
-    if [[ $ADDON_LOAD_FAILURES -eq 0 ]]; then
-        echo "All addon integration templates loaded successfully."
-
-        touch "$ADDON_STATEFILE_SUCCESS"
-    else
-        echo "Encountered $ADDON_LOAD_FAILURES failure(s) loading addon integration templates:"
-        for failed_template in "${ADDON_LOAD_FAILURES_NAMES[@]}"; do
-            echo "  - $failed_template"
-        done
-        if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then
-            fail "Failed to load all addon integration templates successfully."
-        fi
-    fi
-
-elif [[ ! -f "$SO_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" ]]; then
-    echo "Skipping loading addon integration templates until Security Onion core templates have been loaded."
-
-elif [[ -f "$ADDON_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" && "$FORCE" == "false" ]]; then
-    echo "Addon integration templates already loaded"
-fi

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load

@@ -7,9 +7,6 @@
 . /usr/sbin/so-common
 
 {%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
-{%- if GLOBALS.role != "so-heavynode" %}
-{%-   from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
-{%- endif %}
 
 {%- for index, settings in ES_INDEX_SETTINGS.items() %}
 {%-   if settings.policy is defined %}
@@ -36,13 +33,3 @@
 {%-   endif %}
 {%- endfor %}
 echo
-{%- if GLOBALS.role != "so-heavynode" %}
-{%-   for index, settings in ALL_ADDON_SETTINGS.items() %}
-{%-     if settings.policy is defined %}
-  echo
-  echo "Setting up {{ index }}-logs policy..."
-  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
-  echo
-{%-     endif %}
-{%-   endfor %}
-{%- endif %}

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load

@@ -0,0 +1,165 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+{%  from 'vars/globals.map.jinja' import GLOBALS %}
+
+STATE_FILE_INITIAL=/opt/so/state/estemplates_initial_load_attempt.txt
+STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt
+
+if [[ -f $STATE_FILE_INITIAL ]]; then
+  # The initial template load has already run. As this is a subsequent load, all dependencies should 
+  # already be satisified. Therefore, immediately exit/abort this script upon any template load failure
+  # since this is an unrecoverable failure.
+  should_exit_on_failure=1
+else
+  # This is the initial template load, and there likely are some components not yet setup in Elasticsearch.
+  # Therefore load as many templates as possible at this time and if an error occurs proceed to the next 
+  # template. But if at least one template fails to load do not mark the templates as having been loaded.
+  # This will allow the next load to resume the load of the templates that failed to load initially.
+  should_exit_on_failure=0
+  echo "This is the initial template load"
+fi
+
+# If soup is running, ignore errors
+pgrep soup > /dev/null && should_exit_on_failure=0
+
+load_failures=0
+
+load_template() {
+  uri=$1
+  file=$2
+
+  echo "Loading template file $i" 
+  if ! retry 3 1 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"; then
+    if [[ $should_exit_on_failure -eq 1 ]]; then
+      fail "Could not load template file: $file"
+    else
+      load_failures=$((load_failures+1))
+      echo "Incremented load failure counter: $load_failures"
+    fi
+  fi
+}
+
+if [ ! -f $STATE_FILE_SUCCESS ]; then
+  echo "State file $STATE_FILE_SUCCESS not found. Running so-elasticsearch-templates-load."
+
+  . /usr/sbin/so-common
+
+  {% if GLOBALS.role != 'so-heavynode' %}
+  if [ -f /usr/sbin/so-elastic-fleet-common ]; then
+    . /usr/sbin/so-elastic-fleet-common
+  fi
+  {% endif %}
+
+  default_conf_dir=/opt/so/conf
+
+  # Define a default directory to load pipelines from
+  ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/"
+
+  {% if GLOBALS.role == 'so-heavynode' %}
+  file="/opt/so/conf/elasticsearch/templates/index/so-common-template.json"
+  {% else %}
+  file="/usr/sbin/so-elastic-fleet-common"
+  {% endif %}
+
+  if [ -f "$file" ]; then
+    # Wait for ElasticSearch to initialize
+    echo -n "Waiting for ElasticSearch..."
+    retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
+    {% if GLOBALS.role != 'so-heavynode' %}
+    TEMPLATE="logs-endpoint.alerts@package"
+    INSTALLED=$(so-elasticsearch-query _component_template/$TEMPLATE | jq -r .component_templates[0].name)
+    if [ "$INSTALLED" != "$TEMPLATE" ]; then
+      echo
+      echo "Packages not yet installed."
+      echo
+      exit 0
+    fi
+    {% endif %}
+
+    touch $STATE_FILE_INITIAL
+
+    cd ${ELASTICSEARCH_TEMPLATES}/component/ecs
+
+    echo "Loading ECS component templates..."
+    for i in *; do 
+      TEMPLATE=$(echo $i | cut -d '.' -f1) 
+      load_template "_component_template/${TEMPLATE}-mappings" "$i"
+    done
+    echo
+
+    cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent
+
+    echo "Loading Elastic Agent component templates..."
+    {% if GLOBALS.role == 'so-heavynode' %}
+    component_pattern="so-*"
+    {% else %}
+    component_pattern="*"
+    {% endif %}
+    for i in $component_pattern; do 
+      TEMPLATE=${i::-5} 
+      load_template "_component_template/$TEMPLATE" "$i"
+    done
+    echo
+	    
+    # Load SO-specific component templates
+    cd ${ELASTICSEARCH_TEMPLATES}/component/so
+
+    echo "Loading Security Onion component templates..."
+    for i in *; do 
+      TEMPLATE=$(echo $i | cut -d '.' -f1); 
+      load_template "_component_template/$TEMPLATE" "$i"
+    done
+    echo
+
+    # Load SO index templates
+    cd ${ELASTICSEARCH_TEMPLATES}/index
+
+    echo "Loading Security Onion index templates..."
+    shopt -s extglob
+    {% if GLOBALS.role == 'so-heavynode' %}
+    pattern="!(*1password*|*aws*|*azure*|*cloudflare*|*elastic_agent*|*fim*|*github*|*google*|*osquery*|*system*|*windows*|*endpoint*|*elasticsearch*|*generic*|*fleet_server*|*soc*)"
+    {% else %}
+    pattern="*"
+    {% endif %}
+    # Index templates will be skipped if the following conditions are met:
+    # 1. The template is part of the "so-logs-" template group
+    # 2. The template name does not correlate to at least one existing component template
+    # In this situation, the script will treat the skipped template as a temporary failure
+    # and allow the templates to be loaded again on the next run or highstate, whichever 
+    # comes first.
+    COMPONENT_LIST=$(so-elasticsearch-component-templates-list)
+    for i in $pattern; do
+      TEMPLATE=${i::-14}
+      COMPONENT_PATTERN=${TEMPLATE:3}
+      MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery")
+      if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ \.generic|logs-winlog\.winlog ]]; then
+        load_failures=$((load_failures+1))
+        echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures"
+      else
+        load_template "_index_template/$TEMPLATE" "$i"
+      fi
+    done
+  else
+    {% if GLOBALS.role == 'so-heavynode' %}
+    echo "Common template does not exist. Exiting..."
+    {% else %}
+    echo "Elastic Fleet not configured. Exiting..."
+    {% endif %}
+    exit 0
+  fi
+  
+  cd - >/dev/null
+
+  if [[ $load_failures -eq 0 ]]; then
+    echo "All templates loaded successfully"
+    touch $STATE_FILE_SUCCESS
+  else
+    echo "Encountered $load_failures templates that were unable to load, likely due to missing dependencies that will be available later; will retry on next highstate"
+  fi
+else
+  echo "Templates already loaded"
+fi

salt/firewall/containers.map.jinja

@@ -11,7 +11,6 @@
      'so-kratos',
      'so-hydra',
      'so-nginx',
-     'so-postgres',
      'so-redis',
      'so-soc',
      'so-strelka-coordinator',
@@ -35,7 +34,6 @@
      'so-hydra',
      'so-logstash',
      'so-nginx',
-     'so-postgres',
      'so-redis',
      'so-soc',
      'so-strelka-coordinator',
@@ -79,7 +77,6 @@
      'so-kratos',
      'so-hydra',
      'so-nginx',
-     'so-postgres',
      'so-soc'
 ] %}
 

salt/firewall/defaults.yaml

@@ -98,10 +98,6 @@ firewall:
       tcp:
         - 8086
       udp: []
-    postgres:
-      tcp:
-        - 5432
-      udp: []
     kafka_controller:
       tcp:
         - 9093
@@ -197,7 +193,6 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
-                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - localrules
@@ -384,7 +379,6 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
-                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - docker_registry
@@ -398,7 +392,6 @@ firewall:
                 - elasticsearch_rest
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -411,7 +404,6 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -429,7 +421,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
             searchnode:
               portgroups:
@@ -440,7 +431,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -454,7 +444,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -464,7 +453,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -498,7 +486,6 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - elastic_agent_control
@@ -509,7 +496,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -604,7 +590,6 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
-                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - docker_registry
@@ -618,7 +603,6 @@ firewall:
                 - elasticsearch_rest
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -631,7 +615,6 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -649,7 +632,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
             searchnode:
               portgroups:
@@ -660,7 +642,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -674,7 +655,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -684,7 +664,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -716,7 +695,6 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - elastic_agent_control
@@ -727,7 +705,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -822,7 +799,6 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
-                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - docker_registry
@@ -836,7 +812,6 @@ firewall:
                 - elasticsearch_rest
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -849,7 +824,6 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -867,7 +841,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
             searchnode:
               portgroups:
@@ -877,7 +850,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -890,7 +862,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -900,7 +871,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -934,7 +904,6 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - elastic_agent_control
@@ -945,7 +914,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -1043,7 +1011,6 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
-                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - docker_registry
@@ -1064,7 +1031,6 @@ firewall:
                 - elasticsearch_rest
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -1077,7 +1043,6 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -1089,7 +1054,6 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -1101,7 +1065,6 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - redis
@@ -1111,7 +1074,6 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - redis
@@ -1122,7 +1084,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -1159,7 +1120,6 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - elastic_agent_control
@@ -1170,7 +1130,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -1514,7 +1473,6 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
-                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - elastic_agent_control

salt/global/defaults.yaml

@@ -1,10 +1,3 @@
 global:
   pcapengine: SURICATA
   pipeline: REDIS
-  push:
-    enabled: true
-    highstate_interval_hours: 2
-    debounce_seconds: 30
-    drain_interval: 15
-    batch: '25%'
-    batch_wait: 15

salt/global/soc_global.yaml

@@ -11,14 +11,18 @@ global:
     regexFailureMessage: You must enter a valid IP address or CIDR.
   mdengine:
     description: Which engine to use for meta data generation. Options are ZEEK and SURICATA.
+    regex: ^(ZEEK|SURICATA)$
     options:
       - ZEEK
       - SURICATA
+    regexFailureMessage: You must enter either ZEEK or SURICATA.
     global: True
   pcapengine:
     description: Which engine to use for generating pcap. Currently only SURICATA is supported.
+    regex: ^(SURICATA)$
     options:
       - SURICATA
+    regexFailureMessage: You must enter either SURICATA.
     global: True
   ids:
     description: Which IDS engine to use. Currently only Suricata is supported.
@@ -38,9 +42,11 @@ global:
     advanced: True
   pipeline:
     description: Sets which pipeline technology for events to use. The use of Kafka requires a Security Onion Pro license.
+    regex: ^(REDIS|KAFKA)$
     options:
       - REDIS
       - KAFKA
+    regexFailureMessage: You must enter either REDIS or KAFKA.
     global: True
     advanced: True
   repo_host:
@@ -59,41 +65,4 @@ global:
     description: Allows use of Endgame with Security Onion. This feature requires a license from Endgame.
     global: True
     advanced: True
-  push:
-    enabled:
-      description: Master kill-switch for the active push feature. When disabled, rule and pillar changes are picked up at the next scheduled highstate instead of being pushed immediately.
-      forcedType: bool
-      helpLink: push
-      global: True
-    highstate_interval_hours:
-      description: How often every minion in the grid runs a scheduled state.highstate, in hours. Lower values keep minions closer in sync at the cost of more load; higher values reduce load but increase worst-case latency for non-pushed changes. The salt-minion health check restarts a minion if its last highstate is older than this value plus one hour.
-      forcedType: int
-      helpLink: push
-      global: True
-      advanced: True
-    debounce_seconds:
-      description: Trailing-edge debounce window in seconds. A push intent must be quiet for this long before the drainer dispatches. Rapid bursts of edits within this window coalesce into one dispatch.
-      forcedType: int
-      helpLink: push
-      global: True
-      advanced: True
-    drain_interval:
-      description: How often the push drainer checks for ready intents, in seconds. Small values lower dispatch latency at the cost of more background work on the manager.
-      forcedType: int
-      helpLink: push
-      global: True
-      advanced: True
-    batch:
-      description: "Host batch size for push orchestrations. A number (e.g. '10') or a percentage (e.g. '25%'). Limits how many minions run the push state at once so large fleets don't thundering-herd."
-      helpLink: push
-      global: True
-      advanced: True
-      regex: '^([0-9]+%?)$'
-      regexFailureMessage: Enter a whole number or a whole-number percentage (e.g. 10 or 25%).
-    batch_wait:
-      description: Seconds to wait between host batches in a push orchestration. Gives the fleet time to breathe between waves.
-      forcedType: int
-      helpLink: push
-      global: True
-      advanced: True
 

salt/hydra/enabled.sls

@@ -58,7 +58,6 @@ so-hydra:
       - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
     {%   endfor %}
     {% endif %}
-    # Intentionally unless-stopped -- matches the fleet default.
     - restart_policy: unless-stopped
     - watch:
       - file: hydraconfig

salt/idh/enabled.sls

@@ -15,7 +15,6 @@ include:
 so-idh:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-idh:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - name: so-idh
     - detach: True
     - network_mode: host

salt/influxdb/enabled.sls

@@ -18,7 +18,6 @@ include:
 so-influxdb:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-influxdb:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: influxdb
     - networks:
       - sobridge:

salt/influxdb/soc_influxdb.yaml

@@ -85,10 +85,7 @@ influxdb:
       description: The log level to use for outputting log statements. Allowed values are debug, info, or error.
       global: True
       advanced: false
-      options:
+      regex: ^(info|debug|error)$
-      - info
-      - debug
-      - error
       helpLink: influxdb
     metrics-disabled:
       description: If true, the HTTP endpoint that exposes internal InfluxDB metrics will be inaccessible.
@@ -143,9 +140,7 @@ influxdb:
       description: Determines the type of storage used for secrets. Allowed values are bolt or vault.
       global: True
       advanced: True
-      options:
+      regex: ^(bolt|vault)$
-      - bolt
-      - vault
       helpLink: influxdb
     session-length:
       description: Number of minutes that a user login session can remain authenticated. 
@@ -265,9 +260,7 @@ influxdb:
       description: The type of data store to use for HTTP resources. Allowed values are disk or memory. Memory should not be used for production Security Onion installations.
       global: True
       advanced: True
-      options:
+      regex: ^(disk|memory)$
-      - disk
-      - memory
       helpLink: influxdb
     tls-cert: 
       description: The container path to the certificate to use for TLS encryption of the HTTP requests and responses.

salt/kafka/enabled.sls

@@ -27,7 +27,6 @@ include:
 so-kafka:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kafka:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: so-kafka
     - name: so-kafka
     - networks:

salt/kafka/soc_kafka.yaml

@@ -131,10 +131,7 @@ kafka:
       ssl_x_keystore_x_type: 
         description: The key store file format.
         title: ssl.keystore.type
-        options:
+        regex: ^(JKS|PKCS12|PEM)$
-        - JKS
-        - PKCS12
-        - PEM
         helpLink: kafka
       ssl_x_truststore_x_location:
         description: The trust store file location within the Docker container.
@@ -163,11 +160,7 @@ kafka:
       security_x_protocol:
         description: 'Broker communication protocol. Options are: SASL_SSL, PLAINTEXT, SSL, SASL_PLAINTEXT'
         title: security.protocol
-        options:
+        regex: ^(SASL_SSL|PLAINTEXT|SSL|SASL_PLAINTEXT)
-        - SASL_SSL
-        - PLAINTEXT
-        - SSL
-        - SASL_PLAINTEXT
         helpLink: kafka
       ssl_x_keystore_x_location:
         description: The key store file location within the Docker container.
@@ -181,10 +174,7 @@ kafka:
       ssl_x_keystore_x_type:
         description: The key store file format.
         title: ssl.keystore.type
-        options:
+        regex: ^(JKS|PKCS12|PEM)$
-        - JKS
-        - PKCS12
-        - PEM
         helpLink: kafka
       ssl_x_truststore_x_location:
         description: The trust store file location within the Docker container.

salt/kibana/defaults.yaml

@@ -22,7 +22,7 @@ kibana:
           - default
           - file
     migrations:
-      discardCorruptObjects: "9.3.3"
+      discardCorruptObjects: "8.18.8"
     telemetry:
       enabled: False
     xpack:

salt/kibana/enabled.sls

@@ -16,7 +16,6 @@ include:
 so-kibana:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kibana:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: kibana
     - user: kibana
     - networks:

salt/kibana/tools/sbin_jinja/so-kibana-space-defaults

@@ -9,5 +9,5 @@ SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http:
 # Disable certain Features from showing up in the Kibana UI
 echo
 echo "Setting up default Kibana Space:"
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV1","entityManager","streams","cloudConnect","slo"]} ' >> /opt/so/log/kibana/misc.log
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","entityManager"]} ' >> /opt/so/log/kibana/misc.log
 echo

salt/kratos/enabled.sls

@@ -51,7 +51,6 @@ so-kratos:
       - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
     {%   endfor %}
     {% endif %}
-    # Intentionally unless-stopped -- matches the fleet default.
     - restart_policy: unless-stopped
     - watch:
       - file: kratosschema

salt/kratos/soc_kratos.yaml

@@ -3,8 +3,8 @@ kratos:
     description: Enables or disables the Kratos authentication system. WARNING - Disabling this process will cause the grid to malfunction. Re-enabling this setting will require manual effort via SSH.
     forcedType: bool
     advanced: True
-    readonly: True
     helpLink: kratos
+
   oidc:
     enabled:
       description: Set to True to enable OIDC / Single Sign-On (SSO) to SOC. Requires a valid Security Onion license key.
@@ -21,12 +21,8 @@ kratos:
         description: "Specify the provider type. Required. Valid values are: auth0, generic, github, google, microsoft"
         global: True
         forcedType: string
-        options:
+        regex: "auth0|generic|github|google|microsoft"
-        - auth0
+        regexFailureMessage: "Valid values are: auth0, generic, github, google, microsoft"
-        - generic
-        - github
-        - google
-        - microsoft
         helpLink: oidc
       client_id: 
         description: Specify the client ID, also referenced as the application ID. Required.
@@ -47,9 +43,8 @@ kratos:
         description: The source of the subject identifier. Typically 'userinfo'. Only used when provider is 'microsoft'.
         global: True
         forcedType: string
-        options:
+        regex: me|userinfo
-        - me
+        regexFailureMessage: "Valid values are: me, userinfo"
-        - userinfo
         helpLink: oidc
       auth_url: 
         description: Provider's auth URL. Required when provider is 'generic'.
@@ -103,7 +98,7 @@ kratos:
   config:
     session:
       lifespan: 
-        description: Defines the length of a login session before it will timeout, and require a new login.
+        description: Defines the length of a login session.
         global: True
         helpLink: kratos
       whoami:

salt/logstash/defaults.yaml

@@ -26,12 +26,12 @@ logstash:
     manager:
       - so/0011_input_endgame.conf
       - so/0012_input_elastic_agent.conf.jinja
-      - so/0013_input_lumberjack_fleet.conf.jinja
+      - so/0013_input_lumberjack_fleet.conf
       - so/9999_output_redis.conf.jinja
     receiver:
       - so/0011_input_endgame.conf
       - so/0012_input_elastic_agent.conf.jinja
-      - so/0013_input_lumberjack_fleet.conf.jinja
+      - so/0013_input_lumberjack_fleet.conf
       - so/9999_output_redis.conf.jinja
     search:
       - so/0900_input_redis.conf.jinja
@@ -69,5 +69,4 @@ logstash:
     pipeline_x_batch_x_size: 125
     pipeline_x_ecs_compatibility: disabled
   dmz_nodes: []
-  latency_metrics: False
 

salt/logstash/enabled.sls

@@ -28,7 +28,6 @@ include:
 so-logstash:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-logstash:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: so-logstash
     - name: so-logstash
     - networks:

salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf.jinja

@@ -1,4 +1,3 @@
-{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
 input {
   elastic_agent {
     port => 5055
@@ -12,11 +11,6 @@ input {
   }
 }
 filter {
-  {% if LOGSTASH_MERGED.get('latency_metrics', False) %}
-  ruby {
-    code => "event.set('[_tmp][logstash_from_agent]', Time.now().utc.iso8601(3));"
-  }
-  {% endif %}
 if ![metadata] {
   mutate {
     rename => {"@metadata" => "metadata"}

salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf

@@ -0,0 +1,23 @@
+input {
+  elastic_agent {
+    port => 5056
+    tags => [ "elastic-agent", "fleet-lumberjack-input" ]
+    ssl_enabled => true
+    ssl_certificate => "/usr/share/logstash/elasticfleet-lumberjack.crt"
+    ssl_key => "/usr/share/logstash/elasticfleet-lumberjack.key"
+    ecs_compatibility => v8
+    id => "fleet-lumberjack-in"  
+    codec => "json"
+  }
+}
+
+
+filter {
+if ![metadata] {
+  mutate {
+    rename => {"@metadata" => "metadata"}
+  }
+}
+}
+
+

salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf.jinja

@@ -1,26 +0,0 @@
-{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
-input {
-  elastic_agent {
-    port => 5056
-    tags => [ "elastic-agent", "fleet-lumberjack-input" ]
-    ssl_enabled => true
-    ssl_certificate => "/usr/share/logstash/elasticfleet-lumberjack.crt"
-    ssl_key => "/usr/share/logstash/elasticfleet-lumberjack.key"
-    ecs_compatibility => v8
-    id => "fleet-lumberjack-in"
-    codec => "json"
-  }
-}
-
-filter {
-  {% if LOGSTASH_MERGED.get('latency_metrics', False) %}
-  ruby {
-    code => "event.set('[_tmp][logstash_from_fleet]', Time.now().utc.iso8601(3));"
-  }
-  {% endif %}
-  if ![metadata] {
-    mutate {
-      rename => {"@metadata" => "metadata"}
-    }
-  }
-}

salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja

@@ -1,4 +1,3 @@
-{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
 {%- set kafka_password = salt['pillar.get']('kafka:config:password') %}
 {%- set kafka_trustpass = salt['pillar.get']('kafka:config:trustpass') %}
 {%- set kafka_brokers = salt['pillar.get']('kafka:nodes', {}) %}
@@ -31,11 +30,6 @@ input {
     }
 }
 filter {
-  {% if LOGSTASH_MERGED.get('latency_metrics', False) %}
-  ruby {
-    code => "event.set('[_tmp][logstash_from_kafka]', Time.now().utc.iso8601(3));"
-  }
-  {% endif %}
   if ![metadata] {
     mutate {
       rename => { "@metadata" => "metadata" }

salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja

@@ -1,4 +1,4 @@
-{%- from 'logstash/map.jinja' import LOGSTASH_REDIS_NODES, LOGSTASH_MERGED %}
+{%- from 'logstash/map.jinja' import LOGSTASH_REDIS_NODES with context %}
 {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %}
 
 {%- for index in range(LOGSTASH_REDIS_NODES|length) %}
@@ -18,10 +18,3 @@ input {
 }
 {%   endfor %}
 {% endfor -%}
-filter {
-  {% if LOGSTASH_MERGED.get('latency_metrics', False) %}
-  ruby {
-    code => "event.set('[_tmp][logstash_from_redis]', Time.now().utc.iso8601(3));"
-  }
-  {% endif %}
-}

salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja

@@ -1,11 +1,3 @@
-{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
-{% if LOGSTASH_MERGED.get('latency_metrics', False) %}
-filter {
-  ruby {
-    code => "event.set('[_tmp][logstash_to_elasticsearch]', Time.now().utc.iso8601(3));"
-  }
-}
-{% endif %}
 output {
   if "elastic-agent" in [tags] and "so-ip-mappings" in [tags] {
     elasticsearch {

salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja

@@ -13,14 +13,7 @@ filter {
                     add_tag => "fleet-lumberjack-{{ GLOBALS.hostname }}"
           }
   }
-{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
+
-{% if LOGSTASH_MERGED.get('latency_metrics', False) %}
-filter {
-  ruby {
-    code => "event.set('[_tmp][fleet_to_logstash]', Time.now().utc.iso8601(3));"
-  }
-}
-{% endif %}
 output { 
     lumberjack { 
         codec => json 

salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja

@@ -1,17 +1,10 @@
-{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
 {%- if grains.role in ['so-heavynode', 'so-receiver'] %}
   {%- set HOST = GLOBALS.hostname %}
 {%- else %}
   {%- set HOST = GLOBALS.manager %}
 {%- endif %}
 {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %}
-{% if LOGSTASH_MERGED.get('latency_metrics', False) %}
+
-filter {
-  ruby {
-    code => "event.set('[_tmp][logstash_to_redis]', Time.now().utc.iso8601(3));"
-  }
-}
-{% endif %}
 output {
 	redis {
 		host => '{{ HOST }}'

salt/logstash/soc_logstash.yaml

@@ -86,8 +86,3 @@ logstash:
     multiline: True
     advanced: True
     forcedType: "[]string"
-  latency_metrics:
-    description: Enable latency metrics within events processed by logstash. Useful for pinpointing log ingest delay.
-    forcedType: bool
-    global: False
-    advanced: True

salt/manager/beacons.sls

@@ -1,21 +0,0 @@
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-{% from 'global/map.jinja' import GLOBALMERGED %}
-
-include:
-  - salt.minion
-
-{% if GLOBALS.is_manager and GLOBALMERGED.push.enabled %}
-salt_beacons_pushstate:
-  file.managed:
-    - name: /etc/salt/minion.d/beacons_pushstate.conf
-    - source: salt://manager/files/beacons_pushstate.conf.jinja
-    - template: jinja
-    - watch_in:
-      - service: salt_minion_service
-{% else %}
-salt_beacons_pushstate:
-  file.absent:
-    - name: /etc/salt/minion.d/beacons_pushstate.conf
-    - watch_in:
-      - service: salt_minion_service
-{% endif %}

salt/manager/files/beacons_pushstate.conf.jinja

@@ -1,41 +0,0 @@
-{% from 'global/map.jinja' import GLOBALMERGED %}
-beacons:
-  pillar_db:
-    - interval: {{ GLOBALMERGED.push.drain_interval }}
-    - disable_during_state_run: True
-  inotify:
-    - disable_during_state_run: True
-    - coalesce: True
-    - files:
-        /opt/so/saltstack/local/salt/suricata/rules:
-          mask:
-            - close_write
-            - moved_to
-            - delete
-          recurse: True
-          auto_add: True
-          exclude:
-            - '\.sw[a-z]$':
-                regex: True
-            - '~$':
-                regex: True
-            - '/4913$':
-                regex: True
-            - '/\.#':
-                regex: True
-        /opt/so/saltstack/local/salt/strelka/rules/compiled:
-          mask:
-            - close_write
-            - moved_to
-            - delete
-          recurse: True
-          auto_add: True
-          exclude:
-            - '\.sw[a-z]$':
-                regex: True
-            - '~$':
-                regex: True
-            - '/4913$':
-                regex: True
-            - '/\.#':
-                regex: True

salt/manager/init.sls

@@ -15,7 +15,6 @@ include:
   - manager.elasticsearch
   - manager.kibana
   - manager.managed_soc_annotations
-  - manager.beacons
 
 repo_log_dir:
   file.directory:
@@ -232,7 +231,6 @@ surifiltersrules:
     - user: 939
     - group: 939
 
-
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/manager/sync_es_users.sls

@@ -31,13 +31,11 @@ sync_es_users:
       - http: wait_for_kratos
       - file: so-user.lock # require so-user.lock file to be missing
 
-# we dont want this added too early in setup, so the onlyif gates on the
+# we dont want this added too early in setup, so we add the onlyif to verify 'startup_states: highstate'
-# /opt/so/state/setup-complete marker. The marker is written by
+# is in the minion config. That line is added before the final highstate during setup
-# mark_setup_complete in setup/so-functions just before the final setup
-# highstate (and by an upgrade-path state for systems set up under the old gate).
 so-user_sync:
   cron.present:
     - user: root
     - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log'
     - identifier: so-user_sync
-    - onlyif: "test -e /opt/so/state/setup-complete"
+    - onlyif: "grep -x 'startup_states: highstate' /etc/salt/minion"

salt/manager/tools/sbin/so-boot-mine-update

@@ -1,117 +0,0 @@
-#!/bin/bash
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Runs once per boot on managers (via so-boot-mine-update.service), before
-# so-boot-highstate.service. Waits for the responsive minion set to settle, pushes
-# mine.update, waits until every up minion has actually reported to the mine, then
-# warms the master's per-minion pillar cache so the mine-backed node pillars (node
-# IPs, ES/Redis/Logstash/hypervisor discovery -- some glob- and some pillar/grain-
-# targeted) are complete before the boot highstate renders them. Otherwise a node
-# that is up but not yet fully reported gets dropped from those pillars and torn
-# out of the configs they build (e.g. so-elasticsearch ExtraHosts -> container recreate).
-
-MAX_WAIT=${MINE_UPDATE_MAX_WAIT:-180}   # hard backstop only
-INTERVAL=10
-STABLE_CHECKS=3                          # up-count must hold steady this many polls
-elapsed=0
-prev=-1
-stable=0
-up=0
-
-# Wait for the *reachable* minion set to settle rather than for every accepted
-# key to report up: an operator may accept a minion's key and then intentionally
-# power off that host, so requiring up >= accepted would never be satisfied and
-# we'd always burn the full MAX_WAIT. Once the responsive count stops growing we
-# stop waiting and run mine.update against whoever is up.
-while [ "$elapsed" -lt "$MAX_WAIT" ]; do
-  up=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null \
-    | python3 -c 'import sys,json; print(len(json.load(sys.stdin)))' 2>/dev/null)
-  up=${up:-0}
-  if [ "$up" -gt 0 ] && [ "$up" -eq "$prev" ]; then
-    stable=$((stable + 1))
-    [ "$stable" -ge "$STABLE_CHECKS" ] && break
-  else
-    stable=0
-  fi
-  prev=$up
-  sleep "$INTERVAL"
-  elapsed=$((elapsed + INTERVAL))
-done
-
-echo "so-boot-mine-update: ${up} minions up (settled after ${elapsed}s); running mine.update"
-/usr/bin/salt '*' mine.update --out=txt
-
-# A node that is up but has not yet re-reported network.ip_addrs to the mine is
-# silently dropped from mine-backed pillars (elasticsearch:nodes, node_data, ...)
-# when highstate recompiles them -- which e.g. removes it from so-elasticsearch
-# ExtraHosts and forces a container recreate. After the broad mine.update above,
-# wait until every up minion actually has network.ip_addrs in the mine, re-pushing
-# mine.update to stragglers, before releasing the boot highstate. Bounded by the
-# same MAX_WAIT backstop so a slow/down node never blocks boot indefinitely.
-missing=""
-while [ "$elapsed" -lt "$MAX_WAIT" ]; do
-  up_json=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null)
-  mine_json=$(/usr/bin/salt-run mine.get '*' network.ip_addrs tgt_type=glob --out=json 2>/dev/null)
-  missing=$(printf '%s' "$up_json" | python3 -c '
-import sys, json
-up = set(json.load(sys.stdin) or [])
-mine = {k for k, v in (json.loads(sys.argv[1]) or {}).items() if v}
-print("\n".join(sorted(up - mine)))
-' "$mine_json" 2>/dev/null)
-  if [ -z "$missing" ]; then
-    echo "so-boot-mine-update: mine complete for all up minions after ${elapsed}s"
-    break
-  fi
-  echo "so-boot-mine-update: mine missing up minion(s): $(echo $missing); re-running mine.update"
-  for m in $missing; do /usr/bin/salt "$m" mine.update --out=txt; done
-  sleep "$INTERVAL"
-  elapsed=$((elapsed + INTERVAL))
-done
-[ -n "$missing" ] && echo "so-boot-mine-update: WARNING ${MAX_WAIT}s backstop hit; up minion(s) still absent from mine: $(echo $missing); highstate may drop them from configs"
-
-# The pillar/compound-targeted node pillars (elasticsearch:nodes, redis:nodes,
-# logstash:nodes, hypervisor:nodes) resolve their target against the master's
-# per-minion data cache (grains+pillar in .../minions/<id>/data.p), populated only
-# when a minion's pillar is (re)compiled -- separately from the mine. A freshly
-# booted node can be in the mine (glob/node_data sees it) yet absent from that
-# cache, so it is dropped from those pillars and from the configs they build (e.g.
-# so-elasticsearch ExtraHosts). Force a synchronous pillar refresh so the master
-# caches every up node's pillar; refresh_pillar wait=True returns only once the
-# pillar is recompiled (and thus cached for matching). Retry stragglers <= MAX_WAIT.
-echo "so-boot-mine-update: warming master pillar cache for pillar/grain-targeted node pillars"
-/usr/bin/salt '*' saltutil.refresh_pillar wait=True --out=txt
-missing=""
-while [ "$elapsed" -lt "$MAX_WAIT" ]; do
-  up_json=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null)
-  cached_json=$(/usr/bin/salt-run cache.pillar tgt='*' --out=json 2>/dev/null)
-  missing=$(printf '%s' "$up_json" | python3 -c '
-import sys, json
-up = set(json.load(sys.stdin) or [])
-cached = {k for k, v in (json.loads(sys.argv[1]) or {}).items() if v}
-print("\n".join(sorted(up - cached)))
-' "$cached_json" 2>/dev/null)
-  if [ -z "$missing" ]; then
-    echo "so-boot-mine-update: pillar cache warm for all up minions after ${elapsed}s"
-    break
-  fi
-  echo "so-boot-mine-update: pillar not yet cached for: $(echo $missing); refreshing"
-  for m in $missing; do /usr/bin/salt "$m" saltutil.refresh_pillar wait=True --out=txt; done
-  sleep "$INTERVAL"
-  elapsed=$((elapsed + INTERVAL))
-done
-[ -n "$missing" ] && echo "so-boot-mine-update: WARNING ${MAX_WAIT}s backstop hit; pillar not cached for: $(echo $missing); pillar-targeted pillars may drop them"
-
-# Log what the mine-backed pillars render so the boot-time state is inspectable.
-/usr/bin/salt-call saltutil.refresh_pillar >/dev/null 2>&1
-sleep 2
-for key in node_data elasticsearch:nodes; do
-  rendered=$(/usr/bin/salt-call --out=json pillar.get "$key" 2>/dev/null \
-    | python3 -c 'import sys,json; print(json.dumps(json.load(sys.stdin).get("local"), indent=2, sort_keys=True))' 2>/dev/null)
-  echo "so-boot-mine-update: ${key} rendered as:"
-  echo "${rendered:-null}"
-done
-exit 0

salt/manager/tools/sbin/so-detections-overrides-import

@@ -1,381 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Imports detection overrides (e.g. from so-detections-backup) into the so-detection
-# index. Reads <publicId>.<ext> files (NDJSON, one override per line) from a source
-# directory, looks up the matching detection by publicId+engine, validates each
-# override against the same rules SOC enforces, dedupes against existing overrides
-# (operational fields only), and appends new ones.
-
-import argparse
-import ipaddress
-import json
-import os
-import re
-import sys
-from datetime import datetime
-
-import requests
-from requests.auth import HTTPBasicAuth
-import urllib3
-
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
-DEFAULT_INDEX = "so-detection"
-AUTH_FILE = "/opt/so/conf/elasticsearch/curl.config"
-ES_URL = "https://localhost:9200"
-
-# Engines we know how to handle and the file extension the backup script writes.
-ENGINES = {
-    "suricata": "txt",
-}
-
-# Standard Suricata variables that ship with Security Onion. Anything else
-# referenced in an override is "custom" and the user needs to make sure it
-# exists in SOC Config before the override will function.
-BUILTIN_SURICATA_VARS = {
-    "$HOME_NET", "$EXTERNAL_NET",
-    "$HTTP_SERVERS", "$DNS_SERVERS", "$SQL_SERVERS", "$SMTP_SERVERS",
-    "$TELNET_SERVERS", "$AIM_SERVERS", "$DC_SERVERS", "$MODBUS_SERVER",
-    "$MODBUS_CLIENT", "$ENIP_CLIENT", "$ENIP_SERVER",
-    "$HTTP_PORTS", "$SHELLCODE_PORTS", "$ORACLE_PORTS", "$SSH_PORTS",
-    "$FTP_PORTS", "$FILE_DATA_PORTS",
-}
-
-VAR_PATTERN = re.compile(r"\$[A-Z_][A-Z0-9_]*")
-
-# Canonical valid values, per securityonion-soc/model/detection.go.
-SURICATA_OVERRIDE_TYPES = {"suppress", "threshold", "modify"}
-SUPPRESS_TRACKS = {"by_src", "by_dst", "by_either"}
-THRESHOLD_TRACKS = {"by_src", "by_dst", "by_both"}
-THRESHOLD_TYPES = {"limit", "threshold", "both"}
-
-STALE_WARNING = """\
-WARNING: so-detections-backup does not remove backup files when overrides are
-deleted via the Security Onion web UI. As a result, files in the source
-directory may represent overrides that were intentionally deleted and should
-NOT be re-imported.
-
-Before continuing, verify that the source directory reflects the overrides you
-actually want imported. Remove any files corresponding to overrides you previously deleted.
-"""
-
-
-def make_session(auth_file):
-    with open(auth_file, "r") as f:
-        for line in f:
-            if line.startswith("user ="):
-                creds = line.split("=", 1)[1].strip().replace('"', "")
-                user, _, password = creds.partition(":")
-                session = requests.Session()
-                session.auth = HTTPBasicAuth(user, password)
-                session.headers.update({"Content-Type": "application/json"})
-                session.verify = False
-                return session
-    raise RuntimeError(f"Could not find 'user =' line in {auth_file}")
-
-
-def find_detection(session, index, public_id, engine):
-    query = {
-        "query": {"bool": {"must": [
-            {"term": {"so_detection.publicId": public_id}},
-            {"term": {"so_detection.engine": engine}},
-        ]}},
-        "size": 2,
-    }
-    r = session.get(f"{ES_URL}/{index}/_search", json=query)
-    r.raise_for_status()
-    hits = r.json().get("hits", {}).get("hits", [])
-    if not hits:
-        return None, None, None
-    if len(hits) > 1:
-        # Shouldn't happen — publicId is unique per engine — but flag it.
-        print(f"  WARN: {len(hits)} detections matched publicId={public_id} engine={engine}; using first")
-    hit = hits[0]
-    existing = hit["_source"].get("so_detection", {}).get("overrides") or []
-    return hit["_id"], hit["_index"], existing
-
-
-def update_overrides(session, doc_index, doc_id, overrides):
-    body = {"doc": {"so_detection": {"overrides": overrides}}}
-    r = session.post(f"{ES_URL}/{doc_index}/_update/{doc_id}", json=body)
-    r.raise_for_status()
-    return r.json()
-
-
-def dedupe_key(override):
-    """Operational fields only, per Override.Equal() in detection.go.
-    Excludes timestamps and isEnabled so re-imports don't appear unique."""
-    t = override.get("type")
-    if t == "suppress":
-        return (t, override.get("track"), override.get("ip"))
-    if t == "threshold":
-        return (t, override.get("thresholdType"), override.get("track"),
-                override.get("count"), override.get("seconds"))
-    if t == "modify":
-        return (t, override.get("regex"), override.get("value"))
-
-
-def _validate_suricata_ip(ip):
-    if not ip:
-        return "ip cannot be empty"
-    if ip.startswith("$"):
-        return None
-    if ip.startswith("[") and ip.endswith("]"):
-        for part in ip[1:-1].split(","):
-            err = _validate_single_ip(part.strip())
-            if err:
-                return f"invalid IP in list: {err}"
-        return None
-    return _validate_single_ip(ip)
-
-
-def _validate_single_ip(ip):
-    try:
-        if "/" in ip:
-            ipaddress.ip_network(ip, strict=False)
-        else:
-            ipaddress.ip_address(ip)
-    except ValueError:
-        return f"invalid IP/CIDR {ip!r}"
-    return None
-
-
-def validate_override(override, engine):
-    """Mirror Override.Validate() from securityonion-soc/model/detection.go.
-    Returns None on success, an error string otherwise."""
-    t = override.get("type")
-    if not t:
-        return "override type is required"
-    if t not in SURICATA_OVERRIDE_TYPES:
-        return f"invalid type {t!r}: must be one of {sorted(SURICATA_OVERRIDE_TYPES)}"
-
-    has = {k: override.get(k) is not None for k in
-           ("regex", "value", "thresholdType", "track", "ip", "count", "seconds", "customFilter")}
-
-    if t == "suppress":
-        if not has["ip"] or not has["track"]:
-            return "suppress requires 'ip' and 'track'"
-        if any(has[k] for k in ("regex", "value", "thresholdType", "count", "seconds", "customFilter")):
-            return "suppress has unnecessary fields"
-        if override["track"] not in SUPPRESS_TRACKS:
-            return f"invalid track {override['track']!r}: must be one of {sorted(SUPPRESS_TRACKS)}"
-        return _validate_suricata_ip(override["ip"])
-
-    if t == "threshold":
-        if not all(has[k] for k in ("thresholdType", "track", "count", "seconds")):
-            return "threshold requires 'thresholdType', 'track', 'count', 'seconds'"
-        if any(has[k] for k in ("regex", "value", "customFilter")):
-            return "threshold has unnecessary fields"
-        if override["thresholdType"] not in THRESHOLD_TYPES:
-            return f"invalid thresholdType {override['thresholdType']!r}: must be one of {sorted(THRESHOLD_TYPES)}"
-        if override["track"] not in THRESHOLD_TRACKS:
-            return f"invalid track {override['track']!r}: must be one of {sorted(THRESHOLD_TRACKS)}"
-        if not isinstance(override["count"], int) or override["count"] <= 0:
-            return f"count must be a positive integer, got {override['count']!r}"
-        if not isinstance(override["seconds"], int) or override["seconds"] <= 0:
-            return f"seconds must be a positive integer, got {override['seconds']!r}"
-        return None
-
-    if t == "modify":
-        if not has["regex"] or not has["value"]:
-            return "modify requires 'regex' and 'value'"
-        if any(has[k] for k in ("thresholdType", "track", "count", "seconds", "customFilter")):
-            return "modify has unnecessary fields"
-        try:
-            re.compile(override["regex"])
-        except re.error as e:
-            return f"invalid regex: {e}"
-        return None
-
-
-def parse_overrides_file(path):
-    """Parse a file written by so-detections-backup.py: NDJSON, one override
-    per line. Returns a list of (override_dict, line_number)."""
-    overrides = []
-    with open(path, "r") as f:
-        for i, line in enumerate(f, start=1):
-            line = line.strip()
-            if not line:
-                continue
-            overrides.append((json.loads(line), i))
-    return overrides
-
-
-def describe(override):
-    """Human-readable summary of the operational fields for a given override type."""
-    t = override.get("type")
-    if t == "suppress":
-        return f"type=suppress track={override.get('track')} ip={override.get('ip')}"
-    if t == "threshold":
-        return (f"type=threshold track={override.get('track')} "
-                f"thresholdType={override.get('thresholdType')} "
-                f"count={override.get('count')} seconds={override.get('seconds')}")
-    if t == "modify":
-        return f"type=modify regex={override.get('regex')!r}"
-
-
-def collect_custom_vars(override):
-    found = set()
-    for value in override.values():
-        if isinstance(value, str):
-            for match in VAR_PATTERN.findall(value):
-                if match not in BUILTIN_SURICATA_VARS:
-                    found.add(match)
-    return found
-
-
-def parse_args():
-    p = argparse.ArgumentParser(
-        description="Import detection overrides into the so-detection index.",
-    )
-    p.add_argument("--source", "-s", required=True,
-                   help="Source directory containing <publicId>.<ext> override files.")
-    p.add_argument("--engine", "-e", default="suricata", choices=list(ENGINES.keys()),
-                   help="Detection engine (default: suricata).")
-    p.add_argument("--dry-run", "-n", action="store_true",
-                   help="Print what would happen without writing to Elasticsearch.")
-    p.add_argument("--no-import-note", action="store_true",
-                   help="Do not prepend '[Imported YYYY-MM-DD] ' to the override note.")
-    p.add_argument("--index", "-i", default=DEFAULT_INDEX,
-                   help=f"Elasticsearch index to update (default: {DEFAULT_INDEX}).")
-    return p.parse_args()
-
-
-def confirm_proceed(args):
-    """Show the stale-backup warning. Dry-run prints it and continues. Real
-    runs require the user typing 'yes' at the prompt."""
-    print(STALE_WARNING)
-    if args.dry_run:
-        print("(dry-run: no acknowledgement required)\n")
-        return True
-    answer = input("Type 'yes' to acknowledge and continue: ").strip().lower()
-    print()
-    return answer == "yes"
-
-
-def main():
-    args = parse_args()
-
-    if not os.path.isdir(args.source):
-        print(f"ERROR: source directory not found: {args.source}", file=sys.stderr)
-        sys.exit(1)
-
-    extension = ENGINES[args.engine]
-    files = sorted(f for f in os.listdir(args.source) if f.endswith(f".{extension}"))
-    if not files:
-        print(f"No *.{extension} files found in {args.source}")
-        sys.exit(0)
-
-    if not confirm_proceed(args):
-        print("Aborted.")
-        sys.exit(1)
-
-    session = make_session(AUTH_FILE)
-    today = datetime.now().strftime("%Y-%m-%d")
-    note_prefix = "" if args.no_import_note else f"[Imported {today}] "
-
-    counts = {"added": 0, "skipped_dedupe": 0, "skipped_not_found": 0, "invalid": 0, "error": 0}
-    custom_vars = set()
-
-    mode = "DRY-RUN" if args.dry_run else "IMPORT"
-    print(f"[{mode}] engine={args.engine} source={args.source} index={args.index}\n")
-
-    for filename in files:
-        public_id = os.path.splitext(filename)[0]
-        path = os.path.join(args.source, filename)
-        print(f"{public_id}:")
-
-        try:
-            new_overrides = parse_overrides_file(path)
-        except (json.JSONDecodeError, OSError) as e:
-            print(f"  ERROR: could not parse {filename}: {e}")
-            counts["error"] += 1
-            continue
-
-        if not new_overrides:
-            print("  SKIP: empty file")
-            continue
-
-        try:
-            doc_id, doc_index, existing = find_detection(session, args.index, public_id, args.engine)
-        except requests.HTTPError as e:
-            print(f"  ERROR: search failed: {e}")
-            counts["error"] += 1
-            continue
-
-        if doc_id is None:
-            print(f"  WARN: no detection found for publicId={public_id} engine={args.engine}; skipping")
-            counts["skipped_not_found"] += len(new_overrides)
-            continue
-
-        existing_keys = {dedupe_key(o) for o in existing}
-        merged = list(existing)
-        added_this_file = 0
-
-        for override, line_no in new_overrides:
-            err = validate_override(override, args.engine)
-            if err:
-                print(f"  INVALID (line {line_no}): {err}")
-                counts["invalid"] += 1
-                continue
-
-            custom_vars.update(collect_custom_vars(override))
-            key = dedupe_key(override)
-            if key in existing_keys:
-                print(f"  SKIP (line {line_no}): duplicate of existing override [{describe(override)}]")
-                counts["skipped_dedupe"] += 1
-                continue
-
-            if note_prefix:
-                override = dict(override)
-                override["note"] = note_prefix + (override.get("note") or "")
-
-            merged.append(override)
-            existing_keys.add(key)
-            added_this_file += 1
-            print(f"  ADD (line {line_no}): {describe(override)}")
-
-        if added_this_file == 0:
-            continue
-
-        if args.dry_run:
-            print(f"  DRY-RUN: would update {doc_index}/{doc_id} "
-                  f"({len(existing)} existing → {len(merged)} total)")
-            counts["added"] += added_this_file
-            continue
-
-        try:
-            update_overrides(session, doc_index, doc_id, merged)
-            print(f"  UPDATED {doc_index}/{doc_id} ({len(existing)} → {len(merged)})")
-            counts["added"] += added_this_file
-        except requests.HTTPError as e:
-            print(f"  ERROR: update failed: {e}")
-            counts["error"] += 1
-
-    print()
-    print("=" * 60)
-    print(f"Summary ({mode}):")
-    print(f"  Overrides added:           {counts['added']}")
-    print(f"  Skipped (already present): {counts['skipped_dedupe']}")
-    print(f"  Skipped (no detection):    {counts['skipped_not_found']}")
-    print(f"  Invalid (failed checks):   {counts['invalid']}")
-    print(f"  Errors:                    {counts['error']}")
-
-    if custom_vars:
-        print()
-        print("WARNING: detected custom Suricata variables in imported overrides:")
-        for v in sorted(custom_vars):
-            print(f"  {v}")
-        print("If any of these are not already defined in SOC Config (Suricata variables),")
-        print("you must add them manually before the rules will function correctly.")
-
-    sys.exit(0 if counts["error"] == 0 and counts["invalid"] == 0 else 1)
-
-
-if __name__ == "__main__":
-    main()

salt/manager/tools/sbin/so-detections-overrides-import_test.py

@@ -1,588 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-import importlib.util
-import json
-import os
-import shutil
-import sys
-import tempfile
-import unittest
-from importlib.machinery import SourceFileLoader
-from io import StringIO
-from unittest.mock import MagicMock, patch
-
-import requests
-
-# The script has no .py extension; spec_from_file_location can't auto-detect a
-# loader, so we hand it a SourceFileLoader explicitly. (load_module() is
-# deprecated in 3.14 and slated for removal in 3.15.)
-HERE = os.path.dirname(os.path.abspath(__file__))
-SCRIPT = os.path.join(HERE, "so-detections-overrides-import")
-_loader = SourceFileLoader("so_overrides_import", SCRIPT)
-_spec = importlib.util.spec_from_loader("so_overrides_import", _loader)
-soi = importlib.util.module_from_spec(_spec)
-_loader.exec_module(soi)
-
-
-class TestValidateSuppress(unittest.TestCase):
-    def test_valid(self):
-        self.assertIsNone(soi.validate_override(
-            {"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}, "suricata"))
-
-    def test_valid_var(self):
-        self.assertIsNone(soi.validate_override(
-            {"type": "suppress", "track": "by_either", "ip": "$HOME_NET"}, "suricata"))
-
-    def test_valid_cidr(self):
-        self.assertIsNone(soi.validate_override(
-            {"type": "suppress", "track": "by_dst", "ip": "10.0.0.0/8"}, "suricata"))
-
-    def test_valid_bracket_list(self):
-        self.assertIsNone(soi.validate_override(
-            {"type": "suppress", "track": "by_src", "ip": "[1.2.3.4,10.0.0.0/8]"}, "suricata"))
-
-    def test_missing_ip(self):
-        err = soi.validate_override({"type": "suppress", "track": "by_src"}, "suricata")
-        self.assertIn("requires", err)
-
-    def test_missing_track(self):
-        err = soi.validate_override({"type": "suppress", "ip": "1.2.3.4"}, "suricata")
-        self.assertIn("requires", err)
-
-    def test_invalid_track(self):
-        err = soi.validate_override(
-            {"type": "suppress", "track": "by_both", "ip": "1.2.3.4"}, "suricata")
-        self.assertIn("invalid track", err)
-
-    def test_invalid_ip(self):
-        err = soi.validate_override(
-            {"type": "suppress", "track": "by_src", "ip": "not-an-ip"}, "suricata")
-        self.assertIn("invalid IP", err)
-
-    def test_unnecessary_field(self):
-        err = soi.validate_override(
-            {"type": "suppress", "track": "by_src", "ip": "1.2.3.4", "count": 5}, "suricata")
-        self.assertIn("unnecessary fields", err)
-
-
-class TestValidateThreshold(unittest.TestCase):
-    def test_valid(self):
-        self.assertIsNone(soi.validate_override({
-            "type": "threshold", "track": "by_src",
-            "thresholdType": "limit", "count": 10, "seconds": 60,
-        }, "suricata"))
-
-    def test_valid_by_both(self):
-        self.assertIsNone(soi.validate_override({
-            "type": "threshold", "track": "by_both",
-            "thresholdType": "both", "count": 1, "seconds": 1,
-        }, "suricata"))
-
-    def test_track_by_either_invalid(self):
-        err = soi.validate_override({
-            "type": "threshold", "track": "by_either",
-            "thresholdType": "limit", "count": 10, "seconds": 60,
-        }, "suricata")
-        self.assertIn("invalid track", err)
-
-    def test_invalid_threshold_type(self):
-        err = soi.validate_override({
-            "type": "threshold", "track": "by_src",
-            "thresholdType": "bogus", "count": 10, "seconds": 60,
-        }, "suricata")
-        self.assertIn("invalid thresholdType", err)
-
-    def test_zero_count(self):
-        err = soi.validate_override({
-            "type": "threshold", "track": "by_src",
-            "thresholdType": "limit", "count": 0, "seconds": 60,
-        }, "suricata")
-        self.assertIn("count", err)
-
-    def test_negative_seconds(self):
-        err = soi.validate_override({
-            "type": "threshold", "track": "by_src",
-            "thresholdType": "limit", "count": 10, "seconds": -1,
-        }, "suricata")
-        self.assertIn("seconds", err)
-
-    def test_missing_field(self):
-        err = soi.validate_override({
-            "type": "threshold", "track": "by_src",
-            "thresholdType": "limit", "count": 10,  # missing seconds
-        }, "suricata")
-        self.assertIn("requires", err)
-
-    def test_unnecessary_field(self):
-        err = soi.validate_override({
-            "type": "threshold", "track": "by_src",
-            "thresholdType": "limit", "count": 10, "seconds": 60,
-            "regex": "foo",
-        }, "suricata")
-        self.assertIn("unnecessary fields", err)
-
-
-class TestValidateModify(unittest.TestCase):
-    def test_valid(self):
-        self.assertIsNone(soi.validate_override(
-            {"type": "modify", "regex": r"content:\"foo\"", "value": "content:bar"}, "suricata"))
-
-    def test_invalid_regex(self):
-        err = soi.validate_override(
-            {"type": "modify", "regex": "(unbalanced", "value": "x"}, "suricata")
-        self.assertIn("invalid regex", err)
-
-    def test_missing_value(self):
-        err = soi.validate_override({"type": "modify", "regex": "x"}, "suricata")
-        self.assertIn("requires", err)
-
-    def test_unnecessary_field(self):
-        err = soi.validate_override(
-            {"type": "modify", "regex": "x", "value": "y", "track": "by_src"}, "suricata")
-        self.assertIn("unnecessary fields", err)
-
-
-class TestValidateMisc(unittest.TestCase):
-    def test_unknown_type(self):
-        err = soi.validate_override({"type": "suppresss", "track": "by_src", "ip": "1.2.3.4"}, "suricata")
-        self.assertIn("invalid type", err)
-
-    def test_missing_type(self):
-        err = soi.validate_override({"track": "by_src"}, "suricata")
-        self.assertIn("type is required", err)
-
-
-class TestValidateIP(unittest.TestCase):
-    def test_plain_ipv4(self):
-        self.assertIsNone(soi._validate_suricata_ip("1.2.3.4"))
-
-    def test_plain_ipv6(self):
-        self.assertIsNone(soi._validate_suricata_ip("::1"))
-
-    def test_cidr(self):
-        self.assertIsNone(soi._validate_suricata_ip("10.0.0.0/8"))
-
-    def test_var(self):
-        self.assertIsNone(soi._validate_suricata_ip("$CONCOURSEWORKERS"))
-
-    def test_bracket_list(self):
-        self.assertIsNone(soi._validate_suricata_ip("[1.2.3.4, 10.0.0.0/8]"))
-
-    def test_bracket_list_bad_member(self):
-        err = soi._validate_suricata_ip("[1.2.3.4,nope]")
-        self.assertIn("invalid IP in list", err)
-
-    def test_empty(self):
-        self.assertIn("empty", soi._validate_suricata_ip(""))
-
-    def test_invalid(self):
-        self.assertIn("invalid", soi._validate_suricata_ip("999.999.999.999"))
-
-
-class TestDedupeKey(unittest.TestCase):
-    def test_suppress(self):
-        a = {"type": "suppress", "track": "by_src", "ip": "1.2.3.4", "count": 99}
-        b = {"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}
-        # count is irrelevant for suppress dedupe
-        self.assertEqual(soi.dedupe_key(a), soi.dedupe_key(b))
-
-    def test_suppress_differs_on_ip(self):
-        a = {"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}
-        b = {"type": "suppress", "track": "by_src", "ip": "5.6.7.8"}
-        self.assertNotEqual(soi.dedupe_key(a), soi.dedupe_key(b))
-
-    def test_threshold(self):
-        a = {"type": "threshold", "track": "by_src", "thresholdType": "limit",
-             "count": 10, "seconds": 60, "ip": "ignored"}
-        b = {"type": "threshold", "track": "by_src", "thresholdType": "limit",
-             "count": 10, "seconds": 60}
-        self.assertEqual(soi.dedupe_key(a), soi.dedupe_key(b))
-
-    def test_threshold_differs_on_count(self):
-        a = {"type": "threshold", "track": "by_src", "thresholdType": "limit",
-             "count": 10, "seconds": 60}
-        b = {"type": "threshold", "track": "by_src", "thresholdType": "limit",
-             "count": 20, "seconds": 60}
-        self.assertNotEqual(soi.dedupe_key(a), soi.dedupe_key(b))
-
-    def test_modify(self):
-        a = {"type": "modify", "regex": "x", "value": "y"}
-        b = {"type": "modify", "regex": "x", "value": "y"}
-        self.assertEqual(soi.dedupe_key(a), soi.dedupe_key(b))
-
-
-class TestDescribe(unittest.TestCase):
-    def test_suppress(self):
-        s = soi.describe({"type": "suppress", "track": "by_src", "ip": "1.2.3.4"})
-        self.assertIn("suppress", s)
-        self.assertIn("by_src", s)
-        self.assertIn("1.2.3.4", s)
-
-    def test_threshold_includes_count(self):
-        s = soi.describe({"type": "threshold", "track": "by_src",
-                          "thresholdType": "limit", "count": 10, "seconds": 60})
-        self.assertIn("count=10", s)
-        self.assertIn("seconds=60", s)
-
-    def test_modify(self):
-        s = soi.describe({"type": "modify", "regex": "foo"})
-        self.assertIn("modify", s)
-        self.assertIn("foo", s)
-
-
-class TestParseOverridesFile(unittest.TestCase):
-    def _write(self, content):
-        fd, path = tempfile.mkstemp(suffix=".txt")
-        os.close(fd)
-        with open(path, "w") as f:
-            f.write(content)
-        self.addCleanup(os.unlink, path)
-        return path
-
-    def test_single_line(self):
-        path = self._write('{"type":"suppress","track":"by_src","ip":"1.2.3.4"}')
-        result = soi.parse_overrides_file(path)
-        self.assertEqual(len(result), 1)
-        self.assertEqual(result[0][0]["type"], "suppress")
-        self.assertEqual(result[0][1], 1)
-
-    def test_ndjson(self):
-        path = self._write(
-            '{"type":"suppress","track":"by_src","ip":"1.2.3.4"}\n'
-            '{"type":"suppress","track":"by_dst","ip":"5.6.7.8"}\n'
-        )
-        result = soi.parse_overrides_file(path)
-        self.assertEqual(len(result), 2)
-        self.assertEqual(result[1][1], 2)
-
-    def test_empty(self):
-        path = self._write("")
-        self.assertEqual(soi.parse_overrides_file(path), [])
-
-    def test_blank_lines_skipped(self):
-        path = self._write('\n{"type":"suppress","track":"by_src","ip":"1.2.3.4"}\n\n')
-        result = soi.parse_overrides_file(path)
-        self.assertEqual(len(result), 1)
-        self.assertEqual(result[0][1], 2)  # line number reflects original position
-
-    def test_invalid_raises(self):
-        path = self._write("not json")
-        with self.assertRaises(json.JSONDecodeError):
-            soi.parse_overrides_file(path)
-
-
-class TestCollectCustomVars(unittest.TestCase):
-    def test_finds_custom(self):
-        v = soi.collect_custom_vars({"ip": "$CONCOURSEWORKERS"})
-        self.assertEqual(v, {"$CONCOURSEWORKERS"})
-
-    def test_filters_builtins(self):
-        v = soi.collect_custom_vars({"ip": "$HOME_NET"})
-        self.assertEqual(v, set())
-
-    def test_mixed(self):
-        v = soi.collect_custom_vars({"ip": "[$HOME_NET,$MYNET]"})
-        self.assertEqual(v, {"$MYNET"})
-
-    def test_non_string_fields_ignored(self):
-        v = soi.collect_custom_vars({"count": 10, "isEnabled": True})
-        self.assertEqual(v, set())
-
-
-class TestMakeSession(unittest.TestCase):
-    def _write(self, content):
-        fd, path = tempfile.mkstemp()
-        os.close(fd)
-        with open(path, "w") as f:
-            f.write(content)
-        self.addCleanup(os.unlink, path)
-        return path
-
-    def test_valid_auth_file(self):
-        path = self._write('user = "admin:secret"\n')
-        session = soi.make_session(path)
-        self.assertEqual(session.auth.username, "admin")
-        self.assertEqual(session.auth.password, "secret")
-        self.assertFalse(session.verify)
-
-    def test_missing_user_line(self):
-        path = self._write("# no user line here\n")
-        with self.assertRaises(RuntimeError):
-            soi.make_session(path)
-
-
-class TestFindDetection(unittest.TestCase):
-    def _session_with_response(self, payload):
-        session = MagicMock()
-        response = MagicMock()
-        response.json.return_value = payload
-        response.raise_for_status.return_value = None
-        session.get.return_value = response
-        return session
-
-    def test_found(self):
-        session = self._session_with_response({"hits": {"hits": [{
-            "_id": "abc", "_index": "so-detection",
-            "_source": {"so_detection": {"overrides": [{"type": "suppress"}]}},
-        }]}})
-        doc_id, idx, existing = soi.find_detection(session, "so-detection", "2049201", "suricata")
-        self.assertEqual(doc_id, "abc")
-        self.assertEqual(idx, "so-detection")
-        self.assertEqual(len(existing), 1)
-
-    def test_not_found(self):
-        session = self._session_with_response({"hits": {"hits": []}})
-        doc_id, idx, existing = soi.find_detection(session, "so-detection", "x", "suricata")
-        self.assertIsNone(doc_id)
-        self.assertIsNone(idx)
-        self.assertIsNone(existing)
-
-    def test_no_overrides_field(self):
-        session = self._session_with_response({"hits": {"hits": [{
-            "_id": "abc", "_index": "so-detection",
-            "_source": {"so_detection": {}},
-        }]}})
-        _, _, existing = soi.find_detection(session, "so-detection", "x", "suricata")
-        self.assertEqual(existing, [])
-
-    def test_multiple_hits_warns(self):
-        session = self._session_with_response({"hits": {"hits": [
-            {"_id": "a", "_index": "i", "_source": {"so_detection": {"overrides": []}}},
-            {"_id": "b", "_index": "i", "_source": {"so_detection": {"overrides": []}}},
-        ]}})
-        with patch("sys.stdout", new=StringIO()) as out:
-            doc_id, _, _ = soi.find_detection(session, "i", "x", "suricata")
-        self.assertEqual(doc_id, "a")
-        self.assertIn("WARN", out.getvalue())
-
-
-class TestUpdateOverrides(unittest.TestCase):
-    def test_posts_to_update_endpoint(self):
-        session = MagicMock()
-        response = MagicMock()
-        response.raise_for_status.return_value = None
-        response.json.return_value = {"result": "updated"}
-        session.post.return_value = response
-
-        result = soi.update_overrides(session, "so-detection", "abc", [{"type": "suppress"}])
-
-        self.assertEqual(result, {"result": "updated"})
-        url = session.post.call_args[0][0]
-        self.assertIn("/_update/abc", url)
-        body = session.post.call_args[1]["json"]
-        self.assertEqual(body["doc"]["so_detection"]["overrides"], [{"type": "suppress"}])
-
-
-class TestConfirmProceed(unittest.TestCase):
-    def test_dry_run_skips_prompt(self):
-        args = MagicMock(dry_run=True)
-        with patch("sys.stdout", new=StringIO()):
-            self.assertTrue(soi.confirm_proceed(args))
-
-    def test_yes_input(self):
-        args = MagicMock(dry_run=False)
-        with patch("sys.stdout", new=StringIO()):
-            with patch("builtins.input", return_value="yes"):
-                self.assertTrue(soi.confirm_proceed(args))
-
-    def test_yes_input_case_insensitive(self):
-        args = MagicMock(dry_run=False)
-        with patch("sys.stdout", new=StringIO()):
-            with patch("builtins.input", return_value="YES"):
-                self.assertTrue(soi.confirm_proceed(args))
-
-    def test_no_input_aborts(self):
-        args = MagicMock(dry_run=False)
-        with patch("sys.stdout", new=StringIO()):
-            with patch("builtins.input", return_value="no"):
-                self.assertFalse(soi.confirm_proceed(args))
-
-    def test_empty_input_aborts(self):
-        args = MagicMock(dry_run=False)
-        with patch("sys.stdout", new=StringIO()):
-            with patch("builtins.input", return_value=""):
-                self.assertFalse(soi.confirm_proceed(args))
-
-
-class TestParseArgs(unittest.TestCase):
-    def test_defaults(self):
-        with patch.object(sys, "argv", ["cmd", "--source", "/some/path"]):
-            args = soi.parse_args()
-        self.assertEqual(args.source, "/some/path")
-        self.assertEqual(args.engine, "suricata")
-        self.assertFalse(args.dry_run)
-        self.assertFalse(args.no_import_note)
-        self.assertEqual(args.index, soi.DEFAULT_INDEX)
-
-    def test_all_options(self):
-        argv = ["cmd", "-s", "/x", "-e", "suricata", "-n",
-                "--no-import-note", "-i", "alt-index"]
-        with patch.object(sys, "argv", argv):
-            args = soi.parse_args()
-        self.assertEqual(args.source, "/x")
-        self.assertTrue(args.dry_run)
-        self.assertTrue(args.no_import_note)
-        self.assertEqual(args.index, "alt-index")
-
-
-class TestMain(unittest.TestCase):
-    def setUp(self):
-        self.tmpdir = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, self.tmpdir, ignore_errors=True)
-        # Stub make_session so tests don't need /opt/so/conf/elasticsearch/curl.config.
-        p = patch.object(soi, "make_session", return_value=MagicMock())
-        p.start()
-        self.addCleanup(p.stop)
-
-    def _write_file(self, public_id, overrides, ext="txt"):
-        """Write an NDJSON override file. Entries may be dicts or raw strings (for malformed input)."""
-        path = os.path.join(self.tmpdir, f"{public_id}.{ext}")
-        with open(path, "w") as f:
-            for o in overrides:
-                f.write(o if isinstance(o, str) else json.dumps(o))
-                f.write("\n")
-        return path
-
-    def _run_main(self, *extra_argv, input_response="yes"):
-        """Run main() with stdout/stderr captured and input mocked. Returns (stdout, stderr, exit_code)."""
-        argv = ["cmd", "--source", self.tmpdir, *extra_argv]
-        out, err = StringIO(), StringIO()
-        with patch.object(sys, "argv", argv), \
-                patch("sys.stdout", new=out), \
-                patch("sys.stderr", new=err), \
-                patch("builtins.input", return_value=input_response):
-            with self.assertRaises(SystemExit) as cm:
-                soi.main()
-        return out.getvalue(), err.getvalue(), cm.exception.code
-
-    def test_source_dir_missing(self):
-        argv = ["cmd", "--source", "/no/such/path/here"]
-        err = StringIO()
-        with patch.object(sys, "argv", argv), patch("sys.stderr", new=err):
-            with self.assertRaises(SystemExit) as cm:
-                soi.main()
-        self.assertEqual(cm.exception.code, 1)
-        self.assertIn("source directory not found", err.getvalue())
-
-    def test_no_files_found(self):
-        out, _, code = self._run_main()
-        self.assertEqual(code, 0)
-        self.assertIn("No *.txt files found", out)
-
-    def test_user_aborts(self):
-        self._write_file("1001", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}])
-        out, _, code = self._run_main(input_response="no")
-        self.assertEqual(code, 1)
-        self.assertIn("Aborted", out)
-
-    def test_parse_error_increments_error(self):
-        # Malformed JSON line — parse_overrides_file raises JSONDecodeError.
-        self._write_file("1002", ["not json"])
-        out, _, code = self._run_main("--dry-run")
-        self.assertEqual(code, 1)  # invalid+error → non-zero
-        self.assertIn("could not parse", out)
-        self.assertIn("Errors:                    1", out)
-
-    def test_empty_file_skipped(self):
-        # Blank lines only — parse_overrides_file returns []; main reports "empty file" and continues.
-        path = os.path.join(self.tmpdir, "1003.txt")
-        with open(path, "w") as f:
-            f.write("\n\n")
-        out, _, code = self._run_main("--dry-run")
-        self.assertEqual(code, 0)
-        self.assertIn("empty file", out)
-
-    @patch.object(soi, "find_detection")
-    def test_search_http_error(self, mock_find):
-        mock_find.side_effect = requests.HTTPError("boom")
-        self._write_file("1004", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}])
-        out, _, code = self._run_main("--dry-run")
-        self.assertEqual(code, 1)
-        self.assertIn("search failed", out)
-
-    @patch.object(soi, "find_detection")
-    def test_no_detection_found(self, mock_find):
-        mock_find.return_value = (None, None, None)
-        self._write_file("1005", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}])
-        out, _, code = self._run_main("--dry-run")
-        self.assertEqual(code, 0)
-        self.assertIn("no detection found", out)
-        self.assertIn("Skipped (no detection):    1", out)
-
-    @patch.object(soi, "find_detection")
-    def test_all_duplicates_no_update(self, mock_find):
-        existing = [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}]
-        mock_find.return_value = ("doc1", "so-detection", existing)
-        self._write_file("1006", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}])
-        out, _, code = self._run_main("--dry-run")
-        self.assertEqual(code, 0)
-        self.assertIn("SKIP", out)
-        self.assertNotIn("DRY-RUN: would update", out)  # added_this_file == 0 branch
-
-    @patch.object(soi, "update_overrides")
-    @patch.object(soi, "find_detection")
-    def test_happy_path_full(self, mock_find, mock_update):
-        # Exercises: ADD, dedupe SKIP, INVALID, note prefix, UPDATE, custom-vars warning, exit=1 (invalid present)
-        existing = [{"type": "suppress", "track": "by_src", "ip": "9.9.9.9"}]
-        mock_find.return_value = ("doc1", "so-detection", existing)
-        mock_update.return_value = {"result": "updated"}
-        self._write_file("1007", [
-            {"type": "suppress", "track": "by_src", "ip": "1.2.3.4"},                # ADD
-            {"type": "suppress", "track": "by_src", "ip": "9.9.9.9"},                # SKIP (dupe of existing)
-            {"type": "suppress", "track": "bogus",  "ip": "1.2.3.4"},                # INVALID
-            {"type": "suppress", "track": "by_src", "ip": "$CONCOURSEWORKERS"},      # ADD + custom var
-        ])
-        out, _, code = self._run_main()
-        self.assertEqual(code, 1)  # one invalid -> non-zero
-
-        mock_update.assert_called_once()
-        merged = mock_update.call_args[0][3]
-        self.assertEqual(len(merged), 3)  # 1 existing + 2 new
-        new_notes = [o.get("note", "") for o in merged if o.get("ip") in ("1.2.3.4", "$CONCOURSEWORKERS")]
-        self.assertTrue(all(n.startswith("[Imported ") for n in new_notes))
-
-        self.assertIn("ADD", out)
-        self.assertIn("SKIP", out)
-        self.assertIn("INVALID", out)
-        self.assertIn("UPDATED", out)
-        self.assertIn("$CONCOURSEWORKERS", out)
-
-    @patch.object(soi, "update_overrides")
-    @patch.object(soi, "find_detection")
-    def test_no_import_note_preserves_note(self, mock_find, mock_update):
-        mock_find.return_value = ("doc1", "so-detection", [])
-        mock_update.return_value = {"result": "updated"}
-        self._write_file("1008", [
-            {"type": "suppress", "track": "by_src", "ip": "1.2.3.4", "note": "original"},
-        ])
-        _, _, code = self._run_main("--no-import-note")
-        self.assertEqual(code, 0)
-        merged = mock_update.call_args[0][3]
-        self.assertEqual(merged[0]["note"], "original")  # no prefix applied
-
-    @patch.object(soi, "find_detection")
-    def test_dry_run_skips_update(self, mock_find):
-        mock_find.return_value = ("doc1", "so-detection", [])
-        self._write_file("1009", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}])
-        with patch.object(soi, "update_overrides") as mock_update:
-            out, _, code = self._run_main("--dry-run")
-        self.assertEqual(code, 0)
-        mock_update.assert_not_called()
-        self.assertIn("DRY-RUN: would update", out)
-
-    @patch.object(soi, "update_overrides")
-    @patch.object(soi, "find_detection")
-    def test_update_http_error(self, mock_find, mock_update):
-        mock_find.return_value = ("doc1", "so-detection", [])
-        mock_update.side_effect = requests.HTTPError("nope")
-        self._write_file("1010", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}])
-        out, _, code = self._run_main()
-        self.assertEqual(code, 1)
-        self.assertIn("update failed", out)
-
-
-if __name__ == "__main__":
-    unittest.main()

salt/manager/tools/sbin/so-minion

@@ -133,7 +133,7 @@ function getinstallinfo() {
 		return 1
 	fi
 	
-	while read -r var; do export "$var"; done <<< "$INSTALLVARS"
+	export $(echo "$INSTALLVARS" | xargs)
 	if [ $? -ne 0 ]; then
 		log "ERROR" "Failed to source install variables"
 		return 1
@@ -281,39 +281,6 @@ function deleteMinionFiles () {
 	fi
 }
 
-# Remove this minion's postgres Telegraf credential from the shared creds
-# pillar and drop the matching role in Postgres. Always returns 0 so a dead
-# or unreachable so-postgres doesn't block minion deletion — in that case we
-# log a warning and leave the role behind for manual cleanup.
-function remove_postgres_telegraf_from_minion() {
-	local MINION_SAFE
-	MINION_SAFE=$(echo "$MINION_ID" | tr '.-' '__' | tr '[:upper:]' '[:lower:]')
-	local PG_USER="so_telegraf_${MINION_SAFE}"
-
-	log "INFO" "Removing postgres telegraf cred for $MINION_ID"
-
-	so-telegraf-cred remove "$MINION_ID" >/dev/null 2>&1 || true
-
-	if docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^so-postgres$'; then
-		if ! docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf >/dev/null 2>&1 <<EOSQL
-DO \$\$
-BEGIN
-    IF EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '$PG_USER') THEN
-        EXECUTE format('REASSIGN OWNED BY %I TO so_telegraf', '$PG_USER');
-        EXECUTE format('DROP OWNED BY %I', '$PG_USER');
-        EXECUTE format('DROP ROLE %I', '$PG_USER');
-    END IF;
-END
-\$\$;
-EOSQL
-		then
-			log "WARN" "Failed to drop postgres role $PG_USER; pillar entry was removed — drop manually if the role persists"
-		fi
-	else
-		log "WARN" "so-postgres container is not running; skipping DB role cleanup for $PG_USER"
-	fi
-}
-
 # Create the minion file
 function ensure_socore_ownership() {
 	log "INFO" "Setting socore ownership on minion files"
@@ -575,17 +542,6 @@ function add_telegraf_to_minion() {
         log "ERROR" "Failed to add telegraf configuration to $PILLARFILE"
         return 1
     fi
-
-    # Provision the per-minion postgres Telegraf credential in the shared
-    # telegraf/creds.sls pillar. so-telegraf-cred is the only writer; it
-    # generates a password on first add and is a no-op on re-add so the cred
-    # is stable across repeated so-minion runs. postgres.telegraf_users on the
-    # manager creates/updates the DB role from the same pillar.
-    so-telegraf-cred add "$MINION_ID"
-    if [ $? -ne 0 ]; then
-        log "ERROR" "Failed to provision postgres telegraf cred for $MINION_ID"
-        return 1
-    fi
 }
 
 function add_influxdb_to_minion() {
@@ -1113,7 +1069,6 @@ case "$OPERATION" in
 
 	"delete")
 		log "INFO" "Removing minion $MINION_ID"
-		remove_postgres_telegraf_from_minion
 		deleteMinionFiles || {
 			log "ERROR" "Failed to delete minion files for $MINION_ID"
 			exit 1

salt/manager/tools/sbin/so-push-drainer

@@ -1,232 +0,0 @@
-#!/opt/saltstack/salt/bin/python3
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-"""
-so-push-drainer
-===============
-
-Scheduled drainer for the active-push feature. Runs on the manager every
-drain_interval seconds (default 15) via a salt schedule in salt/schedule.sls.
-
-For each intent file under /opt/so/state/push_pending/*.json whose last_touch
-is older than debounce_seconds, this script:
-  * concatenates the actions lists from every ready intent
-  * dedupes by (state or __highstate__, tgt, tgt_type)
-  * dispatches a single `salt-run state.orchestrate orch.push_batch --async`
-    with the deduped actions list passed as pillar kwargs
-  * deletes the contributed intent files on successful dispatch
-
-Reactor sls files (push_suricata, push_strelka, push_pillar) write intents
-but never dispatch directly -- see plan
-/home/mreeves/.claude/plans/goofy-marinating-hummingbird.md for the full design.
-"""
-
-import fcntl
-import glob
-import json
-import logging
-import logging.handlers
-import os
-import subprocess
-import sys
-import time
-
-import salt.client
-
-PENDING_DIR = '/opt/so/state/push_pending'
-LOCK_FILE = os.path.join(PENDING_DIR, '.lock')
-LOG_FILE = '/opt/so/log/salt/so-push-drainer.log'
-
-HIGHSTATE_SENTINEL = '__highstate__'
-
-
-def _make_logger():
-    logger = logging.getLogger('so-push-drainer')
-    logger.setLevel(logging.INFO)
-    if not logger.handlers:
-        os.makedirs(os.path.dirname(LOG_FILE), exist_ok=True)
-        handler = logging.handlers.RotatingFileHandler(
-            LOG_FILE, maxBytes=5 * 1024 * 1024, backupCount=3,
-        )
-        handler.setFormatter(logging.Formatter(
-            '%(asctime)s | %(levelname)s | %(message)s',
-        ))
-        logger.addHandler(handler)
-    return logger
-
-
-def _load_push_cfg():
-    """Read the global:push pillar subtree via salt-call. Returns a dict."""
-    caller = salt.client.Caller()
-    cfg = caller.cmd('pillar.get', 'global:push', {})
-    return cfg if isinstance(cfg, dict) else {}
-
-
-def _read_intent(path, log):
-    try:
-        with open(path, 'r') as f:
-            return json.load(f)
-    except (IOError, ValueError) as exc:
-        log.warning('cannot read intent %s: %s', path, exc)
-        return None
-    except Exception:
-        log.exception('unexpected error reading %s', path)
-        return None
-
-
-def _dedupe_actions(actions):
-    seen = set()
-    deduped = []
-    for action in actions:
-        if not isinstance(action, dict):
-            continue
-        state_key = HIGHSTATE_SENTINEL if action.get('highstate') else action.get('state')
-        tgt = action.get('tgt')
-        tgt_type = action.get('tgt_type', 'compound')
-        if not state_key or not tgt:
-            continue
-        key = (state_key, tgt, tgt_type)
-        if key in seen:
-            continue
-        seen.add(key)
-        deduped.append(action)
-    return deduped
-
-
-def _dispatch(actions, log):
-    pillar_arg = json.dumps({'actions': actions})
-    cmd = [
-        'salt-run',
-        'state.orchestrate',
-        'orch.push_batch',
-        'pillar={}'.format(pillar_arg),
-        '--async',
-    ]
-    log.info('dispatching: %s', ' '.join(cmd[:3]) + ' pillar=<{} actions>'.format(len(actions)))
-    try:
-        result = subprocess.run(
-            cmd, check=True, capture_output=True, text=True, timeout=60,
-        )
-    except subprocess.CalledProcessError as exc:
-        log.error('dispatch failed (rc=%s): stdout=%s stderr=%s',
-                  exc.returncode, exc.stdout, exc.stderr)
-        return False
-    except subprocess.TimeoutExpired:
-        log.error('dispatch timed out after 60s')
-        return False
-    except Exception:
-        log.exception('dispatch raised')
-        return False
-    log.info('dispatch accepted: %s', (result.stdout or '').strip())
-    return True
-
-
-def main():
-    log = _make_logger()
-
-    if not os.path.isdir(PENDING_DIR):
-        # Nothing to do; reactors create the dir on first use.
-        return 0
-
-    try:
-        push = _load_push_cfg()
-    except Exception:
-        log.exception('failed to read global:push pillar; aborting drain pass')
-        return 1
-
-    if not push.get('enabled', True):
-        log.debug('push disabled; exiting')
-        return 0
-
-    debounce_seconds = int(push.get('debounce_seconds', 30))
-
-    os.makedirs(PENDING_DIR, exist_ok=True)
-    lock_fd = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR, 0o644)
-    try:
-        fcntl.flock(lock_fd, fcntl.LOCK_EX)
-
-        intent_files = [
-            p for p in sorted(glob.glob(os.path.join(PENDING_DIR, '*.json')))
-            if os.path.basename(p) != '.lock'
-        ]
-        if not intent_files:
-            return 0
-
-        now = time.time()
-        ready = []
-        skipped = 0
-        broken = []
-        for path in intent_files:
-            intent = _read_intent(path, log)
-            if not isinstance(intent, dict):
-                broken.append(path)
-                continue
-            last_touch = intent.get('last_touch', 0)
-            if now - last_touch < debounce_seconds:
-                skipped += 1
-                continue
-            ready.append((path, intent))
-
-        for path in broken:
-            try:
-                os.unlink(path)
-            except OSError:
-                pass
-
-        if not ready:
-            if skipped:
-                log.debug('no ready intents (%d still in debounce window)', skipped)
-            return 0
-
-        combined_actions = []
-        oldest_first_touch = now
-        all_paths = []
-        for path, intent in ready:
-            combined_actions.extend(intent.get('actions', []) or [])
-            first = intent.get('first_touch', now)
-            if first < oldest_first_touch:
-                oldest_first_touch = first
-            all_paths.extend(intent.get('paths', []) or [])
-
-        deduped = _dedupe_actions(combined_actions)
-        if not deduped:
-            log.warning('%d intent(s) had no usable actions; clearing', len(ready))
-            for path, _ in ready:
-                try:
-                    os.unlink(path)
-                except OSError:
-                    pass
-            return 0
-
-        debounce_duration = now - oldest_first_touch
-        log.info(
-            'draining %d intent(s): %d action(s) after dedupe (raw=%d), '
-            'debounce_duration=%.1fs, paths=%s',
-            len(ready), len(deduped), len(combined_actions),
-            debounce_duration, all_paths[:20],
-        )
-
-        if not _dispatch(deduped, log):
-            log.warning('dispatch failed; leaving intent files in place for retry')
-            return 1
-
-        for path, _ in ready:
-            try:
-                os.unlink(path)
-            except OSError:
-                log.exception('failed to remove drained intent %s', path)
-
-        return 0
-    finally:
-        try:
-            fcntl.flock(lock_fd, fcntl.LOCK_UN)
-        finally:
-            os.close(lock_fd)
-
-
-if __name__ == '__main__':
-    sys.exit(main())