set default DLM retention 90d

respect elasticfleet enable_auto_configuration setting for so-elastic…

.github/DISCUSSION_TEMPLATE/3-0.yml

@@ -11,6 +11,7 @@ body:
         -
         - 3.0.0
         - 3.1.0
+        - 3.2.0
         - Other (please provide detail below)
     validations:
       required: true

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 3.0.0-20260331 ISO image released on 2026/03/31
+### 3.1.0-20260528 ISO image released on 2026/05/28
 
 
 ### Download and Verify
 
-3.0.0-20260331 ISO image:  
+3.1.0-20260528 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
+https://download.securityonion.net/file/securityonion/securityonion-3.1.0-20260528.iso
  
-MD5: ECD318A1662A6FDE0EF213F5A9BD4B07  
+MD5: 9D6FF58DEEE24089D722C73169765B3E  
-SHA1: E55BE314440CCF3392DC0B06BC5E270B43176D9C  
+SHA1: 2B8B816B6CEC3B7F96B3C5E040EBF502DD2C412F  
-SHA256: 7FC47405E335CBE5C2B6C51FE7AC60248F35CBE504907B8B5A33822B23F8F4D5  
+SHA256: 62FAB57E247C843D6A04F0796D8162C732B65D82FC3E4A59D087135B9FD32912  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.1.0-20260528.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.1.0-20260528.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-3.1.0-20260528.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-3.0.0-20260331.iso.sig securityonion-3.0.0-20260331.iso
+gpg --verify securityonion-3.1.0-20260528.iso.sig securityonion-3.1.0-20260528.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Mon 30 Mar 2026 06:22:14 PM EDT using RSA key ID FE507013
+gpg: Signature made Wed 27 May 2026 03:03:59 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

HOTFIX

@@ -0,0 +1 @@
+

VERSION

@@ -1 +1 @@
-3.1.0
+3.2.0

pillar/elasticsearch/index_templates.sls

@@ -1,2 +0,0 @@
-elasticsearch:
-  index_settings:

pillar/telegraf/creds.sls

@@ -0,0 +1,12 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+# Per-minion Telegraf Postgres credentials. so-telegraf-cred on the manager is
+# the single writer; it mutates /opt/so/saltstack/local/pillar/telegraf/creds.sls
+# under flock. Pillar_roots order (local before default) means the populated
+# copy shadows this default on any real grid; this file exists so the pillar
+# key is always defined on fresh installs and when no minions have creds yet.
+telegraf:
+  postgres_creds: {}

pillar/top.sls

@@ -17,6 +17,7 @@ base:
     - sensoroni.adv_sensoroni
     - telegraf.soc_telegraf
     - telegraf.adv_telegraf
+    - telegraf.creds
     - versionlock.soc_versionlock
     - versionlock.adv_versionlock
     - soc.license
@@ -38,6 +39,9 @@ base:
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
+    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
+    - postgres.auth
+    {% endif %}
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
     - kibana.secrets
     {% endif %}
@@ -60,6 +64,8 @@ base:
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
+    - postgres.soc_postgres
+    - postgres.adv_postgres
     - elasticsearch.nodes
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
@@ -97,10 +103,12 @@ base:
     - node_data.ips
     - secrets
     - healthcheck.eval
-    - elasticsearch.index_templates
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
+    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
+    - postgres.auth
+    {% endif %}
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
     - kibana.secrets
     {% endif %}
@@ -126,6 +134,8 @@ base:
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
+    - postgres.soc_postgres
+    - postgres.adv_postgres
     - backup.soc_backup
     - backup.adv_backup
     - zeek.soc_zeek
@@ -142,10 +152,12 @@ base:
     - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
-    - elasticsearch.index_templates
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
+    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
+    - postgres.auth
+    {% endif %}
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
     - kibana.secrets
     {% endif %}
@@ -160,6 +172,8 @@ base:
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
+    - postgres.soc_postgres
+    - postgres.adv_postgres
     - elasticsearch.nodes
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
@@ -256,10 +270,12 @@ base:
   '*_import':
     - node_data.ips
     - secrets
-    - elasticsearch.index_templates
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
+    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
+    - postgres.auth
+    {% endif %}
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
     - kibana.secrets
     {% endif %}
@@ -285,6 +301,8 @@ base:
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
+    - postgres.soc_postgres
+    - postgres.adv_postgres
     - zeek.soc_zeek
     - zeek.adv_zeek
     - bpf.soc_bpf

salt/allowed_states.map.jinja

@@ -29,10 +29,14 @@
     'manager',
     'nginx',
     'influxdb',
+    'postgres',
+    'postgres.auth',
     'soc',
     'kratos',
     'hydra',
     'elasticfleet',
+    'elasticfleet.manager',
+    'elasticsearch.cluster',
     'elastic-fleet-package-registry',
     'utility'
 ] %}
@@ -77,7 +81,7 @@
         ),
         'so-heavynode': (
             sensor_states +
-            ['elasticagent', 'elasticsearch', 'logstash', 'redis', 'nginx']
+            ['elasticagent', 'elasticsearch', 'elasticsearch.cluster', 'logstash', 'redis', 'nginx']
         ),
         'so-idh': (
             ['idh']

salt/backup/config_backup.sls

@@ -32,3 +32,4 @@ so_config_backup:
     - daymonth: '*'
     - month: '*'
     - dayweek: '*'
+

salt/backup/tools/sbin/so-config-backup.jinja

@@ -25,9 +25,11 @@ if [ ! -f $BACKUPFILE ]; then
   # Create empty backup file
   tar -cf $BACKUPFILE -T /dev/null
 
-  # Loop through all paths defined in global.sls, and append them to backup file
+  # Loop through all paths defined in global.sls, and append them to backup file if they exist
   {%- for LOCATION in BACKUPLOCATIONS %}
+  if [[ -d {{ LOCATION }} || -f {{ LOCATION }} ]]; then
     tar -rf $BACKUPFILE "${EXCLUSIONS[@]}" {{ LOCATION }}
+  fi
   {%- endfor %}
 
 fi

salt/ca/files/signing_policies.conf

@@ -54,6 +54,20 @@ x509_signing_policies:
     - extendedKeyUsage: serverAuth
     - days_valid: 820
     - copypath: /etc/pki/issued_certs/
+  postgres:
+    - minions: '*'
+    - signing_private_key: /etc/pki/ca.key
+    - signing_cert: /etc/pki/ca.crt
+    - C: US
+    - ST: Utah
+    - L: Salt Lake City
+    - basicConstraints: "critical CA:false"
+    - keyUsage: "critical keyEncipherment"
+    - subjectKeyIdentifier: hash
+    - authorityKeyIdentifier: keyid,issuer:always
+    - extendedKeyUsage: serverAuth
+    - days_valid: 820
+    - copypath: /etc/pki/issued_certs/
   elasticfleet:
     - minions: '*'
     - signing_private_key: /etc/pki/ca.key

salt/common/tools/sbin/so-common

@@ -142,6 +142,11 @@ check_elastic_license() {
 	fi  
 }
 
+check_elasticsearch_responsive() {
+    retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" ||
+        fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot."
+}
+
 check_salt_master_status() {
 	local count=0
     local attempts="${1:- 10}"

salt/common/tools/sbin/so-image-common

@@ -31,6 +31,7 @@ container_list() {
       "so-hydra"
       "so-nginx"
       "so-pcaptools"
+      "so-postgres"
       "so-soc"
       "so-suricata"
       "so-telegraf"
@@ -55,6 +56,7 @@ container_list() {
       "so-logstash"
       "so-nginx"
       "so-pcaptools"
+      "so-postgres"
       "so-redis"
       "so-soc"
       "so-strelka-backend"
@@ -186,8 +188,27 @@ update_docker_containers() {
         if [ -z "$HOSTNAME" ]; then
           HOSTNAME=$(hostname)
         fi
-        docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$image $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 
+        docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$image $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 || {
-        docker push $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 
+          echo "Unable to tag $image" >> "$LOG_FILE" 2>&1
+          exit 1
+        }
+        # Push to the embedded registry via a registry-to-registry copy. Avoids
+        # `docker push`, which on Docker 29.x with the containerd image store
+        # represents freshly-pulled images as an index whose layer content
+        # isn't reachable through the push path. The local `docker tag` above
+        # is preserved so so-image-pull's `:5000` existence check still works.
+        # Pin to the digest already gpg-verified above so we copy exactly the
+        # bytes we approved.
+        local VERIFIED_REF
+        VERIFIED_REF=$(echo "$DOCKERINSPECT" | jq -r ".[0].RepoDigests[] | select(. | contains(\"$CONTAINER_REGISTRY\"))" | head -n 1)
+        if [ -z "$VERIFIED_REF" ] || [ "$VERIFIED_REF" = "null" ]; then
+          echo "Unable to determine verified digest for $image" >> "$LOG_FILE" 2>&1
+          exit 1
+        fi
+        docker buildx imagetools create --tag $HOSTNAME:5000/$IMAGEREPO/$image "$VERIFIED_REF" >> "$LOG_FILE" 2>&1 || {
+          echo "Unable to copy $image to embedded registry" >> "$LOG_FILE" 2>&1
+          exit 1
+        }
       fi
     else
       echo "There is a problem downloading the $image image. Details: " >> "$LOG_FILE" 2>&1 

salt/common/tools/sbin/so-log-check

@@ -165,6 +165,8 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading component template"  # false positive (elasticsearch index or template names contain 'error')
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading composable template" # false positive (elasticsearch composable template names contain 'error')
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error while parsing document for index \[.ds-logs-kratos-so-.*object mapping for \[file\]" # false positive (mapping error occuring BEFORE kratos index has rolled over in 2.4.210)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|No such container"            # false positive (telegraf trying to run stats on an old container)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|passwords do not match"       # false positive (automated hydra test)
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
@@ -227,7 +229,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|from NIC checksum offloading" # zeek reporter.log
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal"           # docker container getting recycled
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-(tychon|aws_billing|microsoft_defender_endpoint).*user so_kibana lacks the required permissions \[logs-\1" # Known issue with 3 integrations using kibana_system role vs creating unique api creds with proper permissions.
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-(tychon|aws_billing|microsoft_defender_endpoint|armis|o365_metrics|microsoft_sentinel|snyk|cyera|island_browser).*user so_kibana lacks the required permissions \[(logs|metrics)-\1" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user. This error should not be seen on fresh ES 9.3.3 installs or after SO 3.1.0 with soups addition of check_transform_health_and_reauthorize()
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown"             # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1
 fi
 

salt/common/tools/sbin_jinja/so-raid-status

@@ -9,7 +9,7 @@
 
 . /usr/sbin/so-common
 
-software_raid=("SOSMN" "SOSMN-DE02" "SOSSNNV" "SOSSNNV-DE02" "SOS10k-DE02" "SOS10KNV" "SOS10KNV-DE02" "SOS10KNV-DE02" "SOS2000-DE02" "SOS-GOFAST-LT-DE02" "SOS-GOFAST-MD-DE02" "SOS-GOFAST-HV-DE02")
+software_raid=("SOSMN" "SOSMN-DE02" "SOSSNNV" "SOSSNNV-DE02" "SOS10k-DE02" "SOS10KNV" "SOS10KNV-DE02" "SOS10KNV-DE02" "SOS2000-DE02" "SOS-GOFAST-LT-DE02" "SOS-GOFAST-MD-DE02" "SOS-GOFAST-HV-DE02" "HVGUEST")
 hardware_raid=("SOS1000" "SOS1000F" "SOSSN7200" "SOS5000" "SOS4000")
 
 {%- if salt['grains.get']('sosmodel', '') %}
@@ -87,6 +87,11 @@ check_boss_raid() {
 }
 
 check_software_raid() {
+  if [[ ! -f /proc/mdstat ]]; then
+    SWRAID=0
+    return
+  fi
+
   SWRC=$(grep "_" /proc/mdstat)
   if [[ -n $SWRC ]]; then
       # RAID is failed in some way
@@ -107,8 +112,10 @@ if [[ "$is_hwraid" == "true" ]]; then
 fi
 if [[ "$is_softwareraid" == "true" ]]; then
 	check_software_raid
+  if [ "$model" != "HVGUEST" ]; then
     check_boss_raid
   fi
+fi
 
 sum=$(($SWRAID + $BOSSRAID + $HWRAID))
 

salt/docker/defaults.yaml

@@ -237,3 +237,11 @@ docker:
       extra_hosts: []
       extra_env: []
       ulimits: []
+    'so-postgres':
+      final_octet: 47
+      port_bindings:
+        - 0.0.0.0:5432:5432
+      custom_bind_mounts: []
+      extra_hosts: []
+      extra_env: []
+      ulimits: []

salt/elastic-fleet-package-registry/enabled.sls

@@ -51,6 +51,16 @@ so-elastic-fleet-package-registry:
       - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
     {%   endfor %}
     {% endif %}
+
+wait_for_so-elastic-fleet-package-registry:
+  http.wait_for_successful_query:
+    - name: "http://localhost:8080/health"
+    - status: 200
+    - wait_for: 300
+    - request_interval: 15
+    - require:
+      - docker_container: so-elastic-fleet-package-registry
+
 delete_so-elastic-fleet-package-registry_so-status.disabled:
   file.uncomment:
     - name: /opt/so/conf/so-status/so-status.conf

salt/elasticfleet/content-defaults.map.jinja

@@ -0,0 +1,122 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+  or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
+  this file except in compliance with the Elastic License 2.0. #}
+
+
+{% import_json '/opt/so/state/esfleet_content_package_components.json' as ADDON_CONTENT_PACKAGE_COMPONENTS %}
+{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
+{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+
+{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
+{% set ADDON_CONTENT_INTEGRATION_DEFAULTS = {} %}
+
+{% for pkg in ADDON_CONTENT_PACKAGE_COMPONENTS %}
+{%   if pkg.name in CORE_ESFLEET_PACKAGES %}
+{#     skip core content packages #}
+{%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
+{#     generate defaults for each content package #}
+{%     if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0%}
+{%       for pattern in pkg.dataStreams %}
+{#         in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config)
+             title: generic is an artifact of that and is not in use #}
+{%         if pattern.title == "generic" %}
+{%           continue %}
+{%         endif %}
+{%         if "metrics-" in pattern.name %}
+{%           set integration_type = "metrics-" %}
+{%         elif "logs-" in pattern.name %}
+{%           set integration_type = "logs-" %}
+{%         else %}
+{%           set integration_type = "" %}
+{%         endif %}
+{#         on content integrations the component name is user defined at the time it is added to an agent policy #}
+{%         set component_name = pattern.title %}
+{%         set index_pattern = pattern.name %}
+{#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
+{%           set component_name_x = component_name.replace(".","_x_") %}
+{#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
+{%           set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %}
+{#           Default integration settings #}
+{%           set integration_defaults = {
+               "index_sorting": false,
+               "index_template": {
+                 "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
+                 "data_stream": {
+                   "allow_custom_routing": false,
+                   "hidden": false
+                 },
+                 "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
+                 "index_patterns": [index_pattern],
+                 "priority": 501,
+                 "template": {
+                   "settings": {
+                     "index": {
+                       "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
+                         "number_of_replicas": 0
+                     }
+                   }
+                 }
+               },
+               "policy": {
+                 "phases": {
+                   "cold": {
+                     "actions": {
+                        "allocate":{
+                          "number_of_replicas": ""
+                        },
+                       "set_priority": {"priority": 0}
+                     },
+                     "min_age": "60d"
+                   },
+                   "delete": {
+                     "actions": {
+                       "delete": {}
+                     },
+                     "min_age": "365d"
+                   },
+                   "hot": {
+                     "actions": {
+                       "rollover": {
+                         "max_age": "30d",
+                         "max_primary_shard_size": "50gb"
+                       },
+                       "forcemerge":{
+                         "max_num_segments": ""
+                       },
+                       "shrink":{
+                         "max_primary_shard_size": "",
+                         "method": "COUNT",
+                         "number_of_shards": ""
+                       },
+                       "set_priority": {"priority": 100}
+                      },
+                      "min_age": "0ms"
+                   },
+                   "warm": {
+                     "actions": {
+                        "allocate": {
+                          "number_of_replicas": ""
+                        },
+                        "forcemerge": {
+                          "max_num_segments": ""
+                        },
+                        "shrink":{
+                          "max_primary_shard_size": "",
+                          "method": "COUNT",
+                          "number_of_shards": ""
+                        },
+                       "set_priority": {"priority": 50}
+                     },
+                     "min_age": "30d"
+                   }
+                 }
+               }
+             } %}
+
+
+{%         do ADDON_CONTENT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
+{%       endfor %}
+{%     else %}
+{%     endif %}
+{%   endif %}
+{% endfor %}

salt/elasticfleet/defaults.yaml

@@ -1,5 +1,6 @@
 elasticfleet:
   enabled: False
+  patch_version: 9.3.3+build202604082258 # Elastic Agent specific patch release.
   enable_manager_output: True
   config:
     server:

salt/elasticfleet/enabled.sls

@@ -17,65 +17,19 @@ include:
   - logstash.ssl
   - elasticfleet.config
   - elasticfleet.sostatus
+{%- if GLOBALS.role != "so-fleet" %}
+  - elasticfleet.manager
+{%- endif %}
 
-{% if grains.role not in ['so-fleet'] %}
+{% if GLOBALS.role != "so-fleet" %}
 # Wait for Elasticsearch to be ready - no reason to try running Elastic Fleet server if ES is not ready
 wait_for_elasticsearch_elasticfleet:
   cmd.run:
     - name: so-elasticsearch-wait
 {% endif %}
 
-# If enabled, automatically update Fleet Logstash Outputs
+{% if GLOBALS.role == "so-fleet" %}
-{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval', 'so-fleet'] %}
-so-elastic-fleet-auto-configure-logstash-outputs:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-outputs-update
-    - retry:
-        attempts: 4
-        interval: 30
-
-{# Separate from above in order to catch elasticfleet-logstash.crt changes and force update to fleet output policy #}
-so-elastic-fleet-auto-configure-logstash-outputs-force:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-outputs-update --certs
-    - retry:
-        attempts: 4
-        interval: 30
-    - onchanges:
-        - x509: etc_elasticfleet_logstash_crt
-        - x509: elasticfleet_kafka_crt
-{% endif %}
-
-# If enabled, automatically update Fleet Server URLs & ES Connection
-{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-fleet'] %}
-so-elastic-fleet-auto-configure-server-urls:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-urls-update
-    - retry:
-        attempts: 4
-        interval: 30
-{% endif %}
-
-# Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
-{% if grains.role not in ['so-fleet'] %}
-so-elastic-fleet-auto-configure-elasticsearch-urls:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-es-url-update
-    - retry:
-        attempts: 4
-        interval: 30
-
-so-elastic-fleet-auto-configure-artifact-urls:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-artifacts-url-update
-    - retry:
-        attempts: 4
-        interval: 30
-
-{% endif %}
-
 # Sync Elastic Agent artifacts to Fleet Node
-{% if grains.role in ['so-fleet'] %}
 elasticagent_syncartifacts:
   file.recurse:
     - name: /nsm/elastic-fleet/artifacts/beats
@@ -149,57 +103,6 @@ so-elastic-fleet:
       - x509: etc_elasticfleet_crt
 {%   endif %}
 
-{%  if GLOBALS.role != "so-fleet" %}
-so-elastic-fleet-package-statefile:
-  file.managed:
-    - name: /opt/so/state/elastic_fleet_packages.txt
-    - contents: {{ELASTICFLEETMERGED.packages}}
-
-so-elastic-fleet-package-upgrade:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-package-upgrade
-    - retry:
-        attempts: 3
-        interval: 10
-    - onchanges:
-      - file: /opt/so/state/elastic_fleet_packages.txt
-
-so-elastic-fleet-integrations:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-integration-policy-load
-    - retry:
-        attempts: 3
-        interval: 10
-
-so-elastic-agent-grid-upgrade:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-agent-grid-upgrade
-    - retry:
-        attempts: 12
-        interval: 5
-
-so-elastic-fleet-integration-upgrade:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-integration-upgrade
-    - retry:
-        attempts: 3
-        interval: 10
-
-{# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
-so-elastic-fleet-addon-integrations:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
-
-{%   if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
-so-elastic-defend-manage-filters-file-watch:
-  cmd.run:
-    - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log
-    - onchanges:
-      - file: elasticdefendcustom
-      - file: elasticdefenddisabled
-{%    endif %}
-{%  endif %}
-
 delete_so-elastic-fleet_so-status.disabled:
   file.uncomment:
     - name: /opt/so/conf/so-status/so-status.conf

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json

@@ -9,16 +9,22 @@
   "namespace": "so",
   "description": "Zeek Import logs",
   "policy_id": "so-grid-nodes_general",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/zeek/logs/*.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "import",
             "pipeline": "",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -34,7 +40,8 @@
             "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json

@@ -15,19 +15,25 @@
     "version": ""
   },
   "name": "kratos-logs",
+  "namespace": "so",
   "description": "Kratos logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/kratos/kratos.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "kratos",
             "pipeline": "kratos",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -48,10 +54,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json

@@ -9,16 +9,22 @@
   "namespace": "so",
   "description": "Zeek logs",
   "policy_id": "so-grid-nodes_general",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/zeek/logs/current/*.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "zeek",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"],
@@ -30,10 +36,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -5,7 +5,7 @@
   "package": {
     "name": "endpoint",
     "title": "Elastic Defend",
-    "version": "9.0.2",
+    "version": "9.3.0",
     "requires_root": true
   },
   "enabled": true,

salt/elasticfleet/files/integrations/grid-nodes_general/elastic-agent-monitor.json

@@ -6,21 +6,23 @@
   "name": "agent-monitor",
   "namespace": "",
   "description": "",
+  "policy_id": "so-grid-nodes_general",
   "policy_ids": [
     "so-grid-nodes_general"
   ],
-  "output_id": null,
   "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/agents/agent-monitor.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "agentmonitor",
             "pipeline": "elasticagent.monitor",
             "parsers": "",
@@ -34,15 +36,16 @@
             "ignore_older": "72h",
             "clean_inactive": -1,
             "harvester_limit": 0,
-            "fingerprint": true,
+            "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": 64,
+            "file_identity_native": true,
-            "file_identity_native": false,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
-          }
+            "delete_enabled": false
           }
         }
       }
     }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "hydra-logs",
+  "namespace": "so",
   "description": "Hydra logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/hydra/hydra.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "hydra",
             "pipeline": "hydra",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -34,10 +40,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "idh-logs",
+  "namespace": "so",
   "description": "IDH integration",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/idh/opencanary.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "idh",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -31,10 +37,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -4,26 +4,32 @@
     "version": ""
   },
   "name": "import-evtx-logs",
+  "namespace": "so",
   "description": "Import Windows EVTX logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/evtx/*.json"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "import",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": [
               "\\.gz$"
             ],
             "include_files": [],
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.6.1\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.1.2\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.6.1\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.6.1\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.1.2\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.15.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.15.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.15.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
             ],
@@ -33,10 +39,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "import-suricata-logs",
+  "namespace": "so",
   "description": "Import Suricata logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/suricata/eve*.json"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "import",
             "pipeline": "suricata.common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -32,10 +38,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json

@@ -4,14 +4,18 @@
     "version": ""
   },
   "name": "rita-logs",
+  "namespace": "so",
   "description": "RITA Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
@@ -19,6 +23,8 @@
               "/nsm/rita/exploded-dns.csv",
               "/nsm/rita/long-connections.csv"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "rita",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": [
@@ -33,10 +39,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "so-ip-mappings",
+  "namespace": "so",
   "description": "IP Description mappings",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/custom-mappings/ip-descriptions.csv"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "hostnamemappings",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": [
@@ -32,10 +38,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "soc-auth-sync-logs",
+  "namespace": "so",
   "description": "Security Onion - Elastic Auth Sync - Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/sync.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -31,10 +37,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json

@@ -4,20 +4,26 @@
     "version": ""
   },
   "name": "soc-detections-logs",
+  "namespace": "so",
   "description": "Security Onion Console - Detections Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/detections_runtime-status_sigma.log",
               "/opt/so/log/soc/detections_runtime-status_yara.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -35,10 +41,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "soc-salt-relay-logs",
+  "namespace": "so",
   "description": "Security Onion - Salt Relay - Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/salt-relay.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -33,10 +39,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "soc-sensoroni-logs",
+  "namespace": "so",
   "description": "Security Onion - Sensoroni - Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/sensoroni/sensoroni.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -31,10 +37,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "soc-server-logs",
+  "namespace": "so",
   "description": "Security Onion Console Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/sensoroni-server.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -33,10 +39,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "strelka-logs",
+  "namespace": "so",
   "description": "Strelka Logs",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/strelka/log/strelka.log"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "strelka",
             "pipeline": "strelka.file",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -31,10 +37,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json

@@ -4,19 +4,25 @@
     "version": ""
   },
   "name": "suricata-logs",
+  "namespace": "so",
   "description": "Suricata integration",
   "policy_id": "so-grid-nodes_general",
-  "namespace": "so",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.generic": {
+        "filestream.filestream": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/suricata/eve*.json"
             ],
+            "compression_gzip": false,
+            "use_logs_stream": false,
             "data_stream.dataset": "suricata",
             "pipeline": "suricata.common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -31,10 +37,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
-            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": []
+            "include_lines": [],
+            "delete_enabled": false
           }
         }
       }

salt/elasticfleet/input-defaults.map.jinja

@@ -0,0 +1,121 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+  or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
+  this file except in compliance with the Elastic License 2.0. #}
+
+
+{% import_json '/opt/so/state/esfleet_input_package_components.json' as ADDON_INPUT_PACKAGE_COMPONENTS %}
+{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
+{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+
+{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
+{% set ADDON_INPUT_INTEGRATION_DEFAULTS = {} %}
+
+{% for pkg in ADDON_INPUT_PACKAGE_COMPONENTS %}
+{%   if pkg.name in CORE_ESFLEET_PACKAGES %}
+{#     skip core input packages #}
+{%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
+{#     generate defaults for each input package #}
+{%     if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %}
+{%       for pattern in pkg.dataStreams %}
+{#         in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config)
+             title: generic is an artifact of that and is not in use #}
+{%         if pattern.title == "generic" %}
+{%           continue %}
+{%         endif %}
+{%         if "metrics-" in pattern.name %}
+{%           set integration_type = "metrics-" %}
+{%         elif "logs-" in pattern.name %}
+{%           set integration_type = "logs-" %}
+{%         else %}
+{%           set integration_type = "" %}
+{%         endif %}
+{#         on input integrations the component name is user defined at the time it is added to an agent policy #}
+{%         set component_name = pattern.title %}
+{%         set index_pattern = pattern.name %}
+{#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
+{%           set component_name_x = component_name.replace(".","_x_") %}
+{#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
+{%           set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %}
+{#           Default integration settings #}
+{%           set integration_defaults = {
+               "index_sorting": false,
+               "index_template": {
+                 "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
+                 "data_stream": {
+                   "allow_custom_routing": false,
+                   "hidden": false
+                 },
+                 "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
+                 "index_patterns": [index_pattern],
+                 "priority": 501,
+                 "template": {
+                   "settings": {
+                     "index": {
+                       "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
+                         "number_of_replicas": 0
+                     }
+                   }
+                 }
+               },
+               "policy": {
+                 "phases": {
+                   "cold": {
+                     "actions": {
+                        "allocate":{
+                          "number_of_replicas": ""
+                        },
+                       "set_priority": {"priority": 0}
+                     },
+                     "min_age": "60d"
+                   },
+                   "delete": {
+                     "actions": {
+                       "delete": {}
+                     },
+                     "min_age": "365d"
+                   },
+                   "hot": {
+                     "actions": {
+                       "rollover": {
+                         "max_age": "30d",
+                         "max_primary_shard_size": "50gb"
+                       },
+                       "forcemerge":{
+                         "max_num_segments": ""
+                       },
+                       "shrink":{
+                         "max_primary_shard_size": "",
+                         "method": "COUNT",
+                         "number_of_shards": ""
+                       },
+                       "set_priority": {"priority": 100}
+                      },
+                      "min_age": "0ms"
+                   },
+                   "warm": {
+                     "actions": {
+                        "allocate": {
+                          "number_of_replicas": ""
+                        },
+                        "forcemerge": {
+                          "max_num_segments": ""
+                        },
+                        "shrink":{
+                          "max_primary_shard_size": "",
+                          "method": "COUNT",
+                          "number_of_shards": ""
+                        },
+                       "set_priority": {"priority": 50}
+                     },
+                     "min_age": "30d"
+                   }
+                 }
+               }
+             } %}
+
+
+{%         do ADDON_INPUT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
+{%       endfor %}
+{%     endif %}
+{%   endif %}
+{% endfor %}

salt/elasticfleet/integration-defaults.map.jinja

@@ -59,8 +59,8 @@
 {#     skip core integrations #}
 {%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
 {#     generate defaults for each integration #}
-{%     if pkg.es_index_patterns is defined and pkg.es_index_patterns is not none %}
+{%     if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %}
-{%       for pattern in pkg.es_index_patterns %}
+{%       for pattern in pkg.dataStreams %}
 {%         if "metrics-" in pattern.name %}
 {%           set integration_type = "metrics-" %}
 {%         elif "logs-" in pattern.name %}
@@ -75,44 +75,27 @@
 {%         if component_name in WEIRD_INTEGRATIONS %}
 {%           set component_name = WEIRD_INTEGRATIONS[component_name] %}
 {%         endif %}
-
-{#         create duplicate of component_name, so we can split generics from @custom component templates in the index template below and overwrite the default @package when needed
-           eg. having to replace unifiedlogs.generic@package with filestream.generic@package, but keep the ability to customize unifiedlogs.generic@custom and its ILM policy #}
-{%         set custom_component_name = component_name %}
-
-{#         duplicate integration_type to assist with sometimes needing to overwrite component templates with 'logs-filestream.generic@package' (there is no metrics-filestream.generic@package) #}
-{%         set generic_integration_type = integration_type %}
-
 {#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
 {%           set component_name_x = component_name.replace(".","_x_") %}
 {#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
 {%           set integration_key = "so-" ~ integration_type ~ component_name_x %}
 
-{#         if its a .generic template make sure that a .generic@package for the integration exists. Else default to logs-filestream.generic@package #}
-{%         if ".generic" in component_name and integration_type ~ component_name ~ "@package" not in INSTALLED_COMPONENT_TEMPLATES %}
-{#           these generic templates by default are directed to index_pattern of 'logs-generic-*', overwrite that here to point to eg gcp_pubsub.generic-* #}
-{%           set index_pattern = integration_type ~ component_name ~ "-*" %}
-{#           includes use of .generic component template, but it doesn't exist in installed component templates. Redirect it to filestream.generic@package #}
-{%           set component_name = "filestream.generic" %}
-{%           set generic_integration_type = "logs-" %}
-{%         endif %}
-
 {#           Default integration settings #}
 {%           set integration_defaults = {
                "index_sorting": false,
                "index_template": {
-                 "composed_of": [generic_integration_type ~ component_name ~ "@package", integration_type ~ custom_component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
+                 "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
                  "data_stream": {
                    "allow_custom_routing": false,
                    "hidden": false
                  },
-                 "ignore_missing_component_templates": [integration_type ~ custom_component_name ~ "@custom"],
+                 "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
                  "index_patterns": [index_pattern],
                  "priority": 501,
                  "template": {
                    "settings": {
                      "index": {
-                       "lifecycle": {"name": "so-" ~ integration_type ~ custom_component_name ~ "-logs"},
+                       "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
                          "number_of_replicas": 0
                      }
                    }

salt/elasticfleet/manager.sls

@@ -0,0 +1,103 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls in allowed_states %}
+{%   from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+
+include:
+  - elasticfleet.config
+
+# If enabled, automatically update Fleet Logstash Outputs
+{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration %}
+{%   if grains.role not in ['so-import', 'so-eval']%}
+so-elastic-fleet-auto-configure-logstash-outputs:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-outputs-update
+    - retry:
+        attempts: 4
+        interval: 30
+{%   endif %}
+
+# If enabled, automatically update Fleet Server URLs & ES Connection
+so-elastic-fleet-auto-configure-server-urls:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-urls-update
+    - retry:
+        attempts: 4
+        interval: 30
+{% endif %}
+
+# Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
+so-elastic-fleet-auto-configure-elasticsearch-urls:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-es-url-update
+    - retry:
+        attempts: 4
+        interval: 30
+
+so-elastic-fleet-auto-configure-artifact-urls:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-artifacts-url-update
+    - retry:
+        attempts: 4
+        interval: 30
+
+so-elastic-fleet-package-statefile:
+  file.managed:
+    - name: /opt/so/state/elastic_fleet_packages.txt
+    - contents: {{ELASTICFLEETMERGED.packages}}
+
+so-elastic-fleet-package-upgrade:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-package-upgrade
+    - retry:
+        attempts: 3
+        interval: 10
+    - onchanges:
+      - file: /opt/so/state/elastic_fleet_packages.txt
+
+so-elastic-fleet-integrations:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-integration-policy-load
+    - retry:
+        attempts: 3
+        interval: 10
+
+so-elastic-agent-grid-upgrade:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-agent-grid-upgrade
+    - retry:
+        attempts: 12
+        interval: 5
+
+so-elastic-fleet-integration-upgrade:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-integration-upgrade
+    - retry:
+        attempts: 3
+        interval: 10
+
+{# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
+so-elastic-fleet-addon-integrations:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
+
+{% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
+so-elastic-defend-manage-filters-file-watch:
+  cmd.run:
+    - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log
+    - onchanges:
+      - file: elasticdefendcustom
+      - file: elasticdefenddisabled
+{% endif %}
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/elasticfleet/tools/sbin/so-elastic-fleet-common

@@ -135,9 +135,33 @@ elastic_fleet_bulk_package_install() {
     fi
 }
 
-elastic_fleet_installed_packages() {
+elastic_fleet_get_package_list_by_type() {
-    if ! fleet_api "epm/packages/installed?perPage=500"; then
+    if ! output=$(fleet_api "epm/packages"); then
         return 1
+    else
+        is_integration=$(jq '[.items[] | select(.type=="integration") | .name ]' <<< "$output")
+        is_input=$(jq '[.items[] | select(.type=="input") | .name ]' <<< "$output")
+        is_content=$(jq '[.items[] | select(.type=="content") | .name ]' <<< "$output")
+        jq -n --argjson is_integration "${is_integration:-[]}" \
+              --argjson is_input "${is_input:-[]}" \
+              --argjson is_content "${is_content:-[]}" \
+              '{"integration": $is_integration,"input": $is_input, "content": $is_content}'
+    fi
+}
+elastic_fleet_installed_packages_components() {
+    package_type=${1,,}
+    if [[ "$package_type" != "integration" && "$package_type" != "input" && "$package_type" != "content" ]]; then
+        echo "Error: Invalid package type ${package_type}. Valid types are 'integration', 'input', or 'content'."
+        return 1
+    fi
+
+    packages_by_type=$(elastic_fleet_get_package_list_by_type)
+    packages=$(jq --arg package_type "$package_type" '.[$package_type]' <<< "$packages_by_type")
+
+    if ! output=$(fleet_api "epm/packages/installed?perPage=500"); then
+        return 1
+    else
+        jq -c --argjson packages "$packages" '[.items[] | select(.name | IN($packages[])) | {name: .name, dataStreams: .dataStreams}]' <<< "$output"
     fi
 }
 
@@ -216,7 +240,7 @@ elastic_fleet_policy_create() {
         --arg DESC "$DESC" \
         --arg TIMEOUT $TIMEOUT \
         --arg FLEETSERVER "$FLEETSERVER" \
-            '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER}'
+            '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER,"advanced_settings":{"agent_logging_level": "warning"}}'
         )
     # Create Fleet Policy
     if ! fleet_api "agent_policies" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade

@@ -5,7 +5,13 @@
 # this file except in compliance with the Elastic License 2.0.
 
 . /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common
 {%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
+{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+{# Optionally override Elasticsearch version for Elastic Agent patch releases #}
+{%- if ELASTICFLEETDEFAULTS.elasticfleet.patch_version is defined %}
+{%-   do ELASTICSEARCHDEFAULTS.elasticsearch.update({'version': ELASTICFLEETDEFAULTS.elasticfleet.patch_version}) %}
+{%- endif %}
 
 # Only run on Managers
 if ! is_manager_node; then
@@ -14,11 +20,8 @@ if ! is_manager_node; then
 fi
 
 # Get current list of Grid Node Agents that need to be upgraded
-RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=NOT%20agent.version%3A%20{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}%20AND%20policy_id%3A%20so-grid-nodes_%2A&showInactive=false&getStatusSummary=true" --retry 3 --retry-delay 30 --fail 2>/dev/null)
+if ! RAW_JSON=$(fleet_api "agents?perPage=20&page=1&kuery=NOT%20agent.version%3A%20{{ELASTICSEARCHDEFAULTS.elasticsearch.version | urlencode }}%20AND%20policy_id%3A%20so-grid-nodes_%2A&showInactive=false&getStatusSummary=true" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
 
-# Check to make sure that the server responded with good data - else, bail from script
-CHECKSUM=$(jq -r '.page' <<< "$RAW_JSON")
-if [ "$CHECKSUM" -ne 1 ]; then
     printf "Failed to query for current Grid Agents...\n"
     exit 1
 fi
@@ -31,10 +34,12 @@ if [ "$OUTDATED_LIST" != '[]' ]; then
    printf "Initiating upgrades for $AGENTNUMBERS Agents to Elastic {{ELASTICSEARCHDEFAULTS.elasticsearch.version}}...\n\n"
 
    # Generate updated JSON payload
-   JSON_STRING=$(jq -n --arg ELASTICVERSION {{ELASTICSEARCHDEFAULTS.elasticsearch.version}} --arg UPDATELIST $OUTDATED_LIST '{"version": $ELASTICVERSION,"agents": $UPDATELIST }')
+   JSON_STRING=$(jq -n --arg ELASTICVERSION "{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}" --argjson UPDATELIST "$OUTDATED_LIST" '{"version": $ELASTICVERSION,"agents": $UPDATELIST }')
 
    # Update Node Agents
-   curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "http://localhost:5601/api/fleet/agents/bulk_upgrade" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+   if ! fleet_api "agents/bulk_upgrade" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
+       printf "Failed to initiate Agent upgrades...\n"
+   fi
 else
     printf "No Agents need updates... Exiting\n\n"
     exit 0

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load

@@ -18,7 +18,9 @@ INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json
 BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
 BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
 BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json
-PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
+INTEGRATION_PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
+INPUT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_input_package_components.json
+CONTENT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_content_package_components.json
 COMPONENT_TEMPLATES=/opt/so/state/esfleet_component_templates.json
 
 PENDING_UPDATE=false
@@ -179,10 +181,13 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
         else
             echo "Elastic integrations don't appear to need installation/updating..."
         fi
-        # Write out file for generating index/component/ilm templates
+        # Write out file for generating index/component/ilm templates, keeping each package type separate
-        if latest_installed_package_list=$(elastic_fleet_installed_packages); then
+        for package_type in "INTEGRATION" "INPUT" "CONTENT"; do
-            echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
+            if latest_installed_package_list=$(elastic_fleet_installed_packages_components "$package_type"); then
+                outfile="${package_type}_PACKAGE_COMPONENTS"
+                echo $latest_installed_package_list > "${!outfile}"
             fi
+        done
         if retry 3 1 "so-elasticsearch-query / --fail --output /dev/null"; then
             # Refresh installed component template list
             latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.')

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update

@@ -235,6 +235,16 @@ function update_kafka_outputs() {
 
 {% endif %}
 
+# Compare the current Elastic Fleet certificate against what is on disk
+POLICY_CERT_SHA=$(jq -r '.item.ssl.certificate' <<< $RAW_JSON | openssl x509 -noout -sha256 -fingerprint)
+DISK_CERT_SHA=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt -noout -sha256 -fingerprint)
+
+if [[ "$POLICY_CERT_SHA" != "$DISK_CERT_SHA" ]]; then
+    printf "Certificate on disk doesn't match certificate in policy - forcing update\n"
+    UPDATE_CERTS=true
+    FORCE_UPDATE=true
+fi
+
 # Sort & hash the new list of Logstash Outputs
 NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}")
 NEW_HASH=$(sha256sum <<< "$NEW_LIST_JSON" | awk '{print $1}')

salt/elasticsearch/cluster.sls

@@ -0,0 +1,193 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
+{%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %}
+{%   if GLOBALS.role != 'so-heavynode' %}
+{%     from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS, ADDON_INDICES %}
+{%   endif %}
+
+include:
+  - elasticsearch.enabled
+
+escomponenttemplates:
+  file.recurse:
+    - name: /opt/so/conf/elasticsearch/templates/component
+    - source: salt://elasticsearch/templates/component
+    - user: 930
+    - group: 939
+    - clean: True
+    - onchanges_in:
+      - file: so-elasticsearch-templates-reload
+    - show_changes: False
+
+# Clean up legacy and non-SO managed templates from the elasticsearch/templates/index/ directory
+so_index_template_dir:
+  file.directory:
+    - name: /opt/so/conf/elasticsearch/templates/index
+    - clean: True
+    {%- if SO_MANAGED_INDICES %}
+    - require:
+      {%- for index in SO_MANAGED_INDICES %}
+      - file: so_index_template_{{index}}
+      {%- endfor %}
+    {%- endif %}
+
+{%  if GLOBALS.role != "so-heavynode" %}
+# Clean up legacy and non-SO managed templates from the elasticsearch/templates/addon-index/ directory
+addon_index_template_dir:
+  file.directory:
+    - name: /opt/so/conf/elasticsearch/templates/addon-index
+    - clean: True
+    {%- if ADDON_INDICES %}
+    - require:
+      {%- for index in ADDON_INDICES %}
+      - file: addon_index_template_{{index}}
+      {%- endfor %}
+    {%- endif %}
+{%  endif %}
+
+# Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml)
+#   These index templates are for the core SO datasets and are always required
+{%  for index, settings in ES_INDEX_SETTINGS.items() %}
+{%    if settings.index_template is defined %}
+so_index_template_{{index}}:
+  file.managed:
+    - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json
+    - source: salt://elasticsearch/base-template.json.jinja
+    - defaults:
+        TEMPLATE_CONFIG: {{ settings.index_template }}
+    - template: jinja
+    - onchanges_in:
+      - file: so-elasticsearch-templates-reload
+{%    endif %}
+{%  endfor %}
+
+{%  if GLOBALS.role != "so-heavynode" %}
+# Auto-generate optional index templates for integration | input | content packages
+#   These index templates are not used by default (until user adds package to an agent policy).
+#   Pre-configured with standard defaults, and incorporated into SOC configuration for user customization.
+{%    for index,settings in ALL_ADDON_SETTINGS.items() %}
+{%      if settings.index_template is defined %}
+addon_index_template_{{index}}:
+  file.managed:
+    - name: /opt/so/conf/elasticsearch/templates/addon-index/{{ index }}-template.json
+    - source: salt://elasticsearch/base-template.json.jinja
+    - defaults:
+        TEMPLATE_CONFIG: {{ settings.index_template }}
+    - template: jinja
+    - show_changes: False
+    - onchanges_in:
+      - file: addon-elasticsearch-templates-reload
+{%       endif %}
+{%     endfor %}
+{%   endif %}
+
+{%  if GLOBALS.role in GLOBALS.manager_roles %}
+so-es-cluster-settings:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-cluster-settings
+    - cwd: /opt/so
+    - template: jinja
+    - require:
+      - docker_container: so-elasticsearch
+      - file: elasticsearch_sbin_jinja
+      - http: wait_for_so-elasticsearch
+{%  endif %}
+
+# heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)
+so-elasticsearch-ilm-policy-load:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-ilm-policy-load
+    - cwd: /opt/so
+    - require:
+      - docker_container: so-elasticsearch
+      - file: so-elasticsearch-ilm-policy-load-script
+    - onchanges:
+      - file: so-elasticsearch-ilm-policy-load-script
+
+so-elasticsearch-templates-reload:
+  file.absent:
+    - name: /opt/so/state/estemplates.txt
+
+addon-elasticsearch-templates-reload:
+  file.absent:
+    - name: /opt/so/state/addon_estemplates.txt
+
+# so-elasticsearch-templates-load will have its first successful run during the 'so-elastic-fleet-setup' script
+so-elasticsearch-templates:
+  cmd.run:
+{%-  if GLOBALS.role == "so-heavynode" %}
+    - name: /usr/sbin/so-elasticsearch-templates-load --heavynode
+{%-  else %}
+    - name: /usr/sbin/so-elasticsearch-templates-load
+{%-  endif %}
+    - cwd: /opt/so
+    - template: jinja
+    - require:
+      - docker_container: so-elasticsearch
+      - file: elasticsearch_sbin_jinja
+
+so-elasticsearch-dlm-apply:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-dlm-apply
+    - cwd: /opt/so
+    - require:
+      - docker_container: so-elasticsearch
+      - file: elasticsearch_sbin_jinja
+      - cmd: so-elasticsearch-templates
+    - retry:
+        attempts: 3
+        interval: 10
+
+so-elasticsearch-pipelines:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }}
+    - require:
+      - docker_container: so-elasticsearch
+      - file: so-elasticsearch-pipelines-script
+
+so-elasticsearch-roles-load:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-roles-load
+    - cwd: /opt/so
+    - template: jinja
+    - require:
+      - docker_container: so-elasticsearch
+      - file: elasticsearch_sbin_jinja
+
+{%  if grains.role in ['so-managersearch', 'so-manager', 'so-managerhype'] %}
+{%    set ap = "absent" %}
+{%  endif %}
+{%  if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %}
+{%    if ELASTICSEARCHMERGED.index_clean %}
+{%      set ap = "present" %}
+{%    else %}
+{%      set ap = "absent" %}
+{%    endif %}
+{%  endif %}
+{%  if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
+so-elasticsearch-indices-delete:
+  cron.{{ap}}:
+    - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1
+    - identifier: so-elasticsearch-indices-delete
+    - user: root
+    - minute: '*/5'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+{%  endif %}
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/elasticsearch/config.sls

@@ -66,6 +66,8 @@ so-elasticsearch-ilm-policy-load-script:
     - group: 939
     - mode: 754
     - template: jinja
+    - defaults:
+        GLOBALS: {{ GLOBALS }}
     - show_changes: False
 
 so-elasticsearch-pipelines-script:
@@ -91,6 +93,13 @@ estemplatedir:
     - group: 939
     - makedirs: True
 
+esaddontemplatedir:
+  file.directory:
+    - name: /opt/so/conf/elasticsearch/templates/addon-index
+    - user: 930
+    - group: 939
+    - makedirs: True
+
 esrolesdir:
   file.directory:
     - name: /opt/so/conf/elasticsearch/roles

salt/elasticsearch/defaults.yaml

@@ -1,7 +1,8 @@
 elasticsearch:
   enabled: false
-  version: 9.0.8
+  version: 9.3.3
   index_clean: true
+  data_retention_method: DLM
   vm:
     max_map_count: 1048576
   config:
@@ -18,9 +19,18 @@ elasticsearch:
               flood_stage: 90%
               high: 85%
               low: 80%
+    # don't want to set retention here since it will make ES restart with every update +
+    #   potentially case where we could unintentially fall back to retention 7d and cause data loss
+    # data_streams:
+    #   lifecycle:
+    #     retention:
+    #       default: 7d
     indices:
       id_field_data:
         enabled: false
+    # index:
+      # lifecycle:
+        # prefer_ilm: true
     logger:
       org:
         elasticsearch:
@@ -63,6 +73,9 @@ elasticsearch:
             verification_mode: none
   index_settings:
     global_overrides:
+    # Tie this into cluster setting for data_streams.lifecycle.retention.default
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         template:
           settings:
@@ -143,6 +156,8 @@ elasticsearch:
                 order: desc
     so-common:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -304,6 +319,8 @@ elasticsearch:
               number_of_shards: 1
     so-assistant-chat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: ""
       index_template:
         composed_of:
         - assistant-chat-mappings
@@ -344,6 +361,8 @@ elasticsearch:
             min_age: 0ms
     so-assistant-session:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: ""
       index_template:
         composed_of:
         - assistant-session-mappings
@@ -497,6 +516,8 @@ elasticsearch:
             min_age: 30d
     so-idh:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -605,6 +626,8 @@ elasticsearch:
             min_age: 30d
     so-import:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -787,6 +810,8 @@ elasticsearch:
             min_age: 0ms
     so-kismet:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - kismet-mappings
@@ -836,6 +861,8 @@ elasticsearch:
             min_age: 30d
     so-kratos:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -904,6 +931,8 @@ elasticsearch:
             min_age: 30d
     so-hydra:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -1049,6 +1078,8 @@ elasticsearch:
             min_age: 0ms
     so-logs:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - so-data-streams-mappings
@@ -1129,6 +1160,8 @@ elasticsearch:
             min_age: 30d
     so-logs-detections_x_alerts:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - so-data-streams-mappings
@@ -1192,6 +1225,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1307,6 +1342,8 @@ elasticsearch:
             min_age: 30d
     so-elastic-agent-monitor:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1369,6 +1406,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_apm_server:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.apm_server@package
@@ -1433,6 +1472,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_auditbeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.auditbeat@package
@@ -1497,6 +1538,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_cloudbeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.cloudbeat@package
@@ -1561,6 +1604,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_endpoint_security:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1620,6 +1665,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_filebeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1679,6 +1726,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_fleet_server:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1735,6 +1784,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_heartbeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.heartbeat@package
@@ -1799,6 +1850,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_metricbeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1858,6 +1911,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_osquerybeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1917,6 +1972,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_packetbeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.packetbeat@package
@@ -1981,6 +2038,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elasticsearch_x_server:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-elasticsearch.server@package
@@ -2045,6 +2104,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_actions:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - .logs-endpoint.actions@package
@@ -2104,6 +2165,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_action_x_responses:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - .logs-endpoint.action.responses@package
@@ -2163,6 +2226,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_alerts:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.alerts@package
@@ -2222,6 +2287,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_diagnostic_x_collection:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - .logs-endpoint.diagnostic.collection@package
@@ -2297,6 +2364,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_api:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.api@package
@@ -2356,6 +2425,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_file:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.file@package
@@ -2415,6 +2486,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_library:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.library@package
@@ -2474,6 +2547,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_network:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.network@package
@@ -2533,6 +2608,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_process:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.process@package
@@ -2592,6 +2669,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_registry:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.registry@package
@@ -2651,6 +2730,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_security:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.security@package
@@ -2710,6 +2791,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_heartbeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - .logs-endpoint.heartbeat@package
@@ -2769,6 +2852,8 @@ elasticsearch:
             min_age: 30d
     so-logs-http_endpoint_x_generic:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-http_endpoint.generic@package
@@ -2817,6 +2902,8 @@ elasticsearch:
             min_age: 30d
     so-logs-httpjson_x_generic:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-httpjson.generic@package
@@ -2882,6 +2969,8 @@ elasticsearch:
               number_of_replicas: 0
     so-logs-osquery-manager_x_action_x_responses:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         _meta:
           managed: true
@@ -2953,6 +3042,8 @@ elasticsearch:
               number_of_replicas: 0
     so-logs-osquery-manager_x_result:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         _meta:
           managed: true
@@ -3005,6 +3096,8 @@ elasticsearch:
             min_age: 30d
     so-logs-soc:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -3113,6 +3206,8 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_application:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3162,6 +3257,8 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_auth:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3211,6 +3308,8 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_security:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3260,6 +3359,8 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_syslog:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3309,6 +3410,8 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_system:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3358,6 +3461,8 @@ elasticsearch:
             min_age: 30d
     so-logs-windows_x_forwarded:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-windows.forwarded@package
@@ -3405,6 +3510,8 @@ elasticsearch:
             min_age: 30d
     so-logs-windows_x_powershell:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-windows.powershell@package
@@ -3452,6 +3559,8 @@ elasticsearch:
             min_age: 30d
     so-logs-windows_x_powershell_operational:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-windows.powershell_operational@package
@@ -3499,6 +3608,8 @@ elasticsearch:
             min_age: 30d
     so-logs-windows_x_sysmon_operational:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-windows.sysmon_operational@package
@@ -3546,6 +3657,8 @@ elasticsearch:
             min_age: 30d
     so-logs-winlog_x_winlog:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-winlog.winlog@package
@@ -3594,6 +3707,8 @@ elasticsearch:
             min_age: 30d
     so-logstash:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -3709,6 +3824,8 @@ elasticsearch:
             min_age: 30d
     so-metrics-endpoint_x_metadata:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - metrics-endpoint.metadata@package
@@ -3756,6 +3873,8 @@ elasticsearch:
             min_age: 30d
     so-metrics-endpoint_x_metrics:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - metrics-endpoint.metrics@package
@@ -3803,6 +3922,8 @@ elasticsearch:
             min_age: 30d
     so-metrics-endpoint_x_policy:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - metrics-endpoint.policy@package
@@ -3850,6 +3971,8 @@ elasticsearch:
             min_age: 30d
     so-metrics-fleet_server_x_agent_status:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - metrics@tsdb-settings
@@ -3874,6 +3997,8 @@ elasticsearch:
               number_of_replicas: 0
     so-metrics-fleet_server_x_agent_versions:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - metrics@tsdb-settings
@@ -3898,6 +4023,8 @@ elasticsearch:
               number_of_replicas: 0
     so-redis:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4013,6 +4140,8 @@ elasticsearch:
             min_age: 30d
     so-strelka:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4130,6 +4259,8 @@ elasticsearch:
             min_age: 30d
     so-suricata:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4246,6 +4377,8 @@ elasticsearch:
             min_age: 30d
     so-suricata_x_alerts:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4362,6 +4495,8 @@ elasticsearch:
             min_age: 30d
     so-syslog:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4478,6 +4613,8 @@ elasticsearch:
             min_age: 30d
     so-zeek:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings

salt/elasticsearch/enabled.sls

@@ -10,8 +10,6 @@
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
-{%   set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
-{%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
 
 include:
   - ca
@@ -19,6 +17,9 @@ include:
   - elasticsearch.ssl
   - elasticsearch.config
   - elasticsearch.sostatus
+{%- if GLOBALS.role != "so-searchnode" %}
+  - elasticsearch.cluster
+{%- endif%}
 
 so-elasticsearch:
   docker_container.running:
@@ -101,134 +102,24 @@ so-elasticsearch:
       - cmd: auth_users_roles_inode
       - cmd: auth_users_inode
 
+wait_for_so-elasticsearch:
+  http.wait_for_successful_query:
+    - name: "https://localhost:9200/"
+    - username: 'so_elastic'
+    - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}'
+    - ssl: True
+    - verify_ssl: False
+    - status: 200
+    - wait_for: 300
+    - request_interval: 15
+    - require:
+      - docker_container: so-elasticsearch
+
 delete_so-elasticsearch_so-status.disabled:
   file.uncomment:
     - name: /opt/so/conf/so-status/so-status.conf
     - regex: ^so-elasticsearch$
 
-{%   if GLOBALS.role != "so-searchnode" %}
-escomponenttemplates:
-  file.recurse:
-    - name: /opt/so/conf/elasticsearch/templates/component
-    - source: salt://elasticsearch/templates/component
-    - user: 930
-    - group: 939
-    - clean: True
-    - onchanges_in:
-      - file: so-elasticsearch-templates-reload
-    - show_changes: False
-      
-# Auto-generate templates from defaults file
-{%     for index, settings in ES_INDEX_SETTINGS.items() %}
- {%      if settings.index_template is defined %}
-es_index_template_{{index}}:
-  file.managed:
-    - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json
-    - source: salt://elasticsearch/base-template.json.jinja
-    - defaults:
-      TEMPLATE_CONFIG: {{ settings.index_template }}
-    - template: jinja
-    - show_changes: False
-    - onchanges_in:
-      - file: so-elasticsearch-templates-reload
-{%       endif %}
-{%     endfor %}
-
-{%     if TEMPLATES %}
-# Sync custom templates to /opt/so/conf/elasticsearch/templates
-{%       for TEMPLATE in TEMPLATES %}
-es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
-  file.managed:
-    - source: salt://elasticsearch/templates/index/{{TEMPLATE}}
-{%         if 'jinja' in TEMPLATE.split('.')[-1] %}
-    - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}}
-    - template: jinja
-{%         else %}
-    - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1]}}
-{%         endif %}
-    - user: 930
-    - group: 939
-    - show_changes: False
-    - onchanges_in:
-      - file: so-elasticsearch-templates-reload
-{%       endfor %}
-{%     endif %}
-
-{%     if GLOBALS.role in GLOBALS.manager_roles %}
-so-es-cluster-settings:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-cluster-settings
-    - cwd: /opt/so
-    - template: jinja
-    - require:
-      - docker_container: so-elasticsearch
-      - file: elasticsearch_sbin_jinja
-{%     endif %}
-
-so-elasticsearch-ilm-policy-load:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-ilm-policy-load
-    - cwd: /opt/so
-    - require:
-      - docker_container: so-elasticsearch
-      - file: so-elasticsearch-ilm-policy-load-script
-    - onchanges:
-      - file: so-elasticsearch-ilm-policy-load-script
-
-so-elasticsearch-templates-reload:
-  file.absent:
-    - name: /opt/so/state/estemplates.txt
-
-so-elasticsearch-templates:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-templates-load
-    - cwd: /opt/so
-    - template: jinja
-    - require:
-      - docker_container: so-elasticsearch
-      - file: elasticsearch_sbin_jinja
-
-so-elasticsearch-pipelines:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }}
-    - require:
-      - docker_container: so-elasticsearch
-      - file: so-elasticsearch-pipelines-script
-
-so-elasticsearch-roles-load:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-roles-load
-    - cwd: /opt/so
-    - template: jinja
-    - require:
-      - docker_container: so-elasticsearch
-      - file: elasticsearch_sbin_jinja
-
-{%     if grains.role in ['so-managersearch', 'so-manager', 'so-managerhype'] %}
-{%       set ap = "absent" %}
-{%     endif %}
-{%     if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %}
-{%       if ELASTICSEARCHMERGED.index_clean %}
-{%         set ap = "present" %}
-{%       else %}
-{%         set ap = "absent" %}
-{%       endif %}
-{%     endif %}  
-{%     if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
-so-elasticsearch-indices-delete:
-  cron.{{ap}}:
-    - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1
-    - identifier: so-elasticsearch-indices-delete
-    - user: root
-    - minute: '*/5'
-    - hour: '*'
-    - daymonth: '*'
-    - month: '*'
-    - dayweek: '*'
-{%     endif %}
-
-{%   endif %}
-
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/elasticsearch/files/ingest/common

@@ -63,7 +63,8 @@
     { "set":             { "if": "ctx.event?.dataset != null && !ctx.event.dataset.contains('.')", "field": "event.dataset", "value": "{{event.module}}.{{event.dataset}}" } },
     { "split":           { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "dataset_tag_temp" } },
     { "append":          { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}"  } },
-    { "grok":            { "if": "ctx.http?.response?.status_code != null", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long} %{GREEDYDATA}"]} },
+    { "grok":            { "if": "ctx.http?.response?.status_code instanceof String", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long}(?:\\s+%{GREEDYDATA})?"], "ignore_failure": true } },
+    { "convert":         { "if": "ctx.http?.response?.status_code != null && !(ctx.http.response.status_code instanceof Number)", "field": "http.response.status_code", "type": "long", "ignore_failure": true } },
     { "set":             { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
     { "remove":          { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } },
     { "pipeline": { "name": "global@custom", "ignore_missing_pipeline": true, "description": "[Fleet] Global pipeline for all data streams" } }

salt/elasticsearch/files/ingest/global@custom

@@ -177,12 +177,84 @@
                 "description": "Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"
             }
         },
+        {
+            "script": {
+                "description": "Snapshot event.ingested into _tmp.event_ingested_pre_fleet before .fleet_final_pipeline-1 overwrites it with ES ingest time",
+                "lang": "painless",
+                "if": "ctx.event?.ingested != null && ctx.event?.created == null",
+                "ignore_failure": true,
+                "source": "ctx.putIfAbsent('_tmp', [:]); ctx._tmp.event_ingested_pre_fleet = ctx.event.ingested;"
+            }
+        },
         {
             "pipeline": {
                 "name": ".fleet_final_pipeline-1",
                 "ignore_missing_pipeline": true
             }
         },
+        {
+            "script": {
+                "description": "Calculate time from Elastic Agent to Logstash.",
+                "lang": "painless",
+                "if": "ctx._tmp?.logstash_from_agent != null",
+                "ignore_failure": true,
+                "source": "ZonedDateTime start = ctx._tmp.event_ingested_pre_fleet != null ? ZonedDateTime.parse(ctx._tmp.event_ingested_pre_fleet) : ZonedDateTime.parse(ctx['@timestamp']); ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_elasticagent_to_logstash = ChronoUnit.SECONDS.between(start, ZonedDateTime.parse(ctx._tmp.logstash_from_agent));"
+            }
+        },
+        {
+            "script": {
+                "description": "Calculate time from Logstash to Redis",
+                "lang": "painless",
+                "if": "ctx._tmp?.logstash_from_agent != null && ctx._tmp?.logstash_to_redis != null",
+                "ignore_failure": true,
+                "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_logstash_to_redis = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_from_agent), ZonedDateTime.parse(ctx._tmp.logstash_to_redis));"
+            }
+        },
+        {
+            "script": {
+                "description": "Calculate time message spends in redis queue (logstash delay in pulling event).",
+                "lang": "painless",
+                "if": "ctx._tmp?.logstash_to_redis != null && ctx._tmp?.logstash_from_redis != null",
+                "ignore_failure": true,
+                "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_redis_to_logstash = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_to_redis), ZonedDateTime.parse(ctx._tmp.logstash_from_redis));"
+            }
+        },
+        {
+            "script": {
+                "description": "Calculate time from Logstash to Elasticsearch (after read from Redis).",
+                "lang": "painless",
+                "if": "ctx._tmp?.logstash_from_redis != null",
+                "ignore_failure": true,
+                "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_logstash_to_elasticsearch = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_from_redis), metadata().now);"
+            }
+        },
+        {
+            "script": {
+                "description": "Calculate time from Elastic Agent to Kafka.",
+                "lang": "painless",
+                "if": "ctx._tmp?.logstash_from_kafka != null && ctx._tmp?.logstash_from_agent == null",
+                "ignore_failure": true,
+                "source": "ZonedDateTime start = ctx._tmp.event_ingested_pre_fleet != null ? ZonedDateTime.parse(ctx._tmp.event_ingested_pre_fleet) : ZonedDateTime.parse(ctx['@timestamp']); ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_elasticagent_to_kafka = ChronoUnit.SECONDS.between(start, ZonedDateTime.parse(ctx._tmp.logstash_from_kafka));"
+            }
+        },
+        {
+            "script": {
+                "description": "Calculate time message spends in Kafka queue (logstash delay in pulling event).",
+                "lang": "painless",
+                "if": "ctx._tmp?.logstash_from_kafka != null && ctx.metadata?.kafka?.timestamp != null && ctx._tmp?.logstash_from_agent == null",
+                "ignore_failure": true,
+                "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_kafka_queue = ChronoUnit.SECONDS.between(ZonedDateTime.ofInstant(Instant.ofEpochMilli(Long.parseLong(ctx.metadata.kafka.timestamp.toString())), ZoneId.of('UTC')), ZonedDateTime.parse(ctx._tmp.logstash_from_kafka));"
+            }
+        },
+        {
+            "script": {
+                "description": "Calculate time from Logstash to Elasticsearch (after read from Kafka).",
+                "lang": "painless",
+                "if": "ctx._tmp?.logstash_from_kafka != null && ctx._tmp?.logstash_from_agent == null",
+                "ignore_failure": true,
+                "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_kafka_to_elasticsearch = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_from_kafka), metadata().now);"
+            }
+        },
         {
             "remove": {
                 "field": "event.agent_id_status",
@@ -202,7 +274,8 @@
                     "event.dataset_temp",
                     "dataset_tag_temp",
                     "module_temp",
-                    "datastream_dataset_temp"
+                    "datastream_dataset_temp",
+                    "_tmp"
                 ],
                 "ignore_missing": true,
                 "ignore_failure": true

salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2

@@ -10,24 +10,28 @@
   "processors": [
     {
       "set": {
+        "tag": "set_ecs_version_f5923549",
         "field": "ecs.version",
         "value": "8.17.0"
       }
     },
     {
       "set": {
+        "tag": "set_observer_vendor_ad9d35cc",
         "field": "observer.vendor",
         "value": "netgate"
       }
     },
     {
       "set": {
+        "tag": "set_observer_type_5dddf3ba",
         "field": "observer.type",
         "value": "firewall"
       }
     },
     {
       "rename": {
+        "tag": "rename_message_to_event_original_56a77271",
         "field": "message",
         "target_field": "event.original",
         "ignore_missing": true,
@@ -36,12 +40,14 @@
     },
     {
       "set": {
+        "tag": "set_event_kind_de80643c",
         "field": "event.kind",
         "value": "event"
       }
     },
     {
       "set": {
+        "tag": "set_event_timezone_4ca44cac",
         "field": "event.timezone",
         "value": "{{{_tmp.tz_offset}}}",
         "if": "ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local'"
@@ -49,6 +55,7 @@
     },
     {
       "grok": {
+        "tag": "grok_event_original_27d9c8c7",
         "description": "Parse syslog header",
         "field": "event.original",
         "patterns": [
@@ -72,6 +79,7 @@
     },
     {
       "date": {
+        "tag": "date__tmp_timestamp8601_to_timestamp_6ac9d3ce",
         "if": "ctx._tmp.timestamp8601 != null",
         "field": "_tmp.timestamp8601",
         "target_field": "@timestamp",
@@ -82,6 +90,7 @@
     },
     {
       "date": {
+        "tag": "date__tmp_timestamp_to_timestamp_f21e536e",
         "if": "ctx.event?.timezone != null && ctx._tmp?.timestamp != null",
         "field": "_tmp.timestamp",
         "target_field": "@timestamp",
@@ -95,6 +104,7 @@
     },
     {
       "grok": {
+        "tag": "grok_process_name_cef3d489",
         "description": "Set Event Provider",
         "field": "process.name",
         "patterns": [
@@ -107,71 +117,83 @@
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-firewall",
+        "tag": "pipeline_e16851a7",
+        "name": "logs-pfsense.log-1.25.2-firewall",
         "if": "ctx.event.provider == 'filterlog'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-openvpn",
+        "tag": "pipeline_828590b5",
+        "name": "logs-pfsense.log-1.25.2-openvpn",
         "if": "ctx.event.provider == 'openvpn'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-ipsec",
+        "tag": "pipeline_9d37039c",
+        "name": "logs-pfsense.log-1.25.2-ipsec",
         "if": "ctx.event.provider == 'charon'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-dhcp",
+        "tag": "pipeline_ad56bbca",
-        "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)"
+        "name": "logs-pfsense.log-1.25.2-dhcp",
+        "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\", \"dnsmasq-dhcp\"].contains(ctx.event.provider)"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-unbound",
+        "tag": "pipeline_dd85553d",
+        "name": "logs-pfsense.log-1.25.2-unbound",
         "if": "ctx.event.provider == 'unbound'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-haproxy",
+        "tag": "pipeline_720ed255",
+        "name": "logs-pfsense.log-1.25.2-haproxy",
         "if": "ctx.event.provider == 'haproxy'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-php-fpm",
+        "tag": "pipeline_456beba5",
+        "name": "logs-pfsense.log-1.25.2-php-fpm",
         "if": "ctx.event.provider == 'php-fpm'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-squid",
+        "tag": "pipeline_a0d89375",
+        "name": "logs-pfsense.log-1.25.2-squid",
         "if": "ctx.event.provider == 'squid'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-snort",
+        "tag": "pipeline_c2f1ed55",
+        "name": "logs-pfsense.log-1.25.2-snort",
         "if": "ctx.event.provider == 'snort'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.23.1-suricata",
+        "tag":"pipeline_33db1c9e",
+        "name": "logs-pfsense.log-1.25.2-suricata",
         "if": "ctx.event.provider == 'suricata'"
       }
     },
     {
       "drop": {
-        "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)"
+        "tag": "drop_9d7c46f8",
+        "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dnsmasq-dhcp\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)"
       }
     },
     {
       "append": {
+        "tag": "append_event_category_4780a983",
         "field": "event.category",
         "value": "network",
         "if": "ctx.network != null"
@@ -179,6 +201,7 @@
     },
     {
       "convert": {
+        "tag": "convert_source_address_to_source_ip_f5632a20",
         "field": "source.address",
         "target_field": "source.ip",
         "type": "ip",
@@ -188,6 +211,7 @@
     },
     {
       "convert": {
+        "tag": "convert_destination_address_to_destination_ip_f1388f0c",
         "field": "destination.address",
         "target_field": "destination.ip",
         "type": "ip",
@@ -197,6 +221,7 @@
     },
     {
       "set": {
+        "tag": "set_network_type_1f1d940a",
         "field": "network.type",
         "value": "ipv6",
         "if": "ctx.source?.ip != null && ctx.source.ip.contains(\":\")"
@@ -204,6 +229,7 @@
     },
     {
       "set": {
+        "tag": "set_network_type_69deca38",
         "field": "network.type",
         "value": "ipv4",
         "if": "ctx.source?.ip != null && ctx.source.ip.contains(\".\")"
@@ -211,6 +237,7 @@
     },
     {
       "geoip": {
+        "tag": "geoip_source_ip_to_source_geo_da2e41b2",
         "field": "source.ip",
         "target_field": "source.geo",
         "ignore_missing": true
@@ -218,6 +245,7 @@
     },
     {
       "geoip": {
+        "tag": "geoip_destination_ip_to_destination_geo_ab5e2968",
         "field": "destination.ip",
         "target_field": "destination.geo",
         "ignore_missing": true
@@ -225,6 +253,7 @@
     },
     {
       "geoip": {
+        "tag": "geoip_source_ip_to_source_as_28d69883",
         "ignore_missing": true,
         "database_file": "GeoLite2-ASN.mmdb",
         "field": "source.ip",
@@ -237,6 +266,7 @@
     },
     {
       "geoip": {
+        "tag": "geoip_destination_ip_to_destination_as_8a007787",
         "database_file": "GeoLite2-ASN.mmdb",
         "field": "destination.ip",
         "target_field": "destination.as",
@@ -249,6 +279,7 @@
     },
     {
       "rename": {
+        "tag": "rename_source_as_asn_to_source_as_number_a917047d",
         "field": "source.as.asn",
         "target_field": "source.as.number",
         "ignore_missing": true
@@ -256,6 +287,7 @@
     },
     {
       "rename": {
+        "tag": "rename_source_as_organization_name_to_source_as_organization_name_f1362d0b",
         "field": "source.as.organization_name",
         "target_field": "source.as.organization.name",
         "ignore_missing": true
@@ -263,6 +295,7 @@
     },
     {
       "rename": {
+        "tag": "rename_destination_as_asn_to_destination_as_number_3b459fcd",
         "field": "destination.as.asn",
         "target_field": "destination.as.number",
         "ignore_missing": true
@@ -270,6 +303,7 @@
     },
     {
       "rename": {
+        "tag": "rename_destination_as_organization_name_to_destination_as_organization_name_814bd459",
         "field": "destination.as.organization_name",
         "target_field": "destination.as.organization.name",
         "ignore_missing": true
@@ -277,12 +311,14 @@
     },
     {
       "community_id": {
+        "tag": "community_id_d2308e7a",
         "target_field": "network.community_id",
         "ignore_failure": true
       }
     },
     {
       "grok": {
+        "tag": "grok_observer_ingress_interface_name_968018d3",
         "field": "observer.ingress.interface.name",
         "patterns": [
           "%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}"
@@ -293,6 +329,7 @@
     },
     {
       "set": {
+        "tag": "set_network_vlan_id_efd4d96a",
         "field": "network.vlan.id",
         "copy_from": "observer.ingress.vlan.id",
         "ignore_empty_value": true
@@ -300,6 +337,7 @@
     },
     {
       "append": {
+        "tag": "append_related_ip_c1a6356b",
         "field": "related.ip",
         "value": "{{{destination.ip}}}",
         "allow_duplicates": false,
@@ -308,6 +346,7 @@
     },
     {
       "append": {
+        "tag": "append_related_ip_8121c591",
         "field": "related.ip",
         "value": "{{{source.ip}}}",
         "allow_duplicates": false,
@@ -316,6 +355,7 @@
     },
     {
       "append": {
+        "tag": "append_related_ip_53b62ed8",
         "field": "related.ip",
         "value": "{{{source.nat.ip}}}",
         "allow_duplicates": false,
@@ -324,6 +364,7 @@
     },
     {
       "append": {
+        "tag": "append_related_hosts_6f162628",
         "field": "related.hosts",
         "value": "{{{destination.domain}}}",
         "if": "ctx.destination?.domain != null"
@@ -331,6 +372,7 @@
     },
     {
       "append": {
+        "tag": "append_related_user_c036eec2",
         "field": "related.user",
         "value": "{{{user.name}}}",
         "if": "ctx.user?.name != null"
@@ -338,6 +380,7 @@
     },
     {
       "set": {
+        "tag": "set_network_direction_cb1e3125",
         "field": "network.direction",
         "value": "{{{network.direction}}}bound",
         "if": "ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/"
@@ -345,6 +388,7 @@
     },
     {
       "remove": {
+        "tag": "remove_a82e20f2",
         "field": [
           "_tmp"
         ],
@@ -353,11 +397,21 @@
     },
     {
       "script": {
+        "tag": "script_a7f2c062",
         "lang": "painless",
         "description": "This script processor iterates over the whole document to remove fields with null values.",
         "source": "void handleMap(Map map) {\n  for (def x : map.values()) {\n    if (x instanceof Map) {\n        handleMap(x);\n    } else if (x instanceof List) {\n        handleList(x);\n    }\n  }\n  map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n  for (def x : list) {\n      if (x instanceof Map) {\n          handleMap(x);\n      } else if (x instanceof List) {\n          handleList(x);\n      }\n  }\n}\nhandleMap(ctx);\n"
       }
     },
+    {
+      "append": {
+        "tag": "append_preserve_original_event_on_error",
+        "field": "tags",
+        "value": "preserve_original_event",
+        "allow_duplicates": false,
+        "if": "ctx.error?.message != null"
+      }
+    },
     {
       "pipeline": {
         "name": "global@custom",
@@ -405,7 +459,14 @@
     {
       "append": {
         "field": "error.message",
-        "value": "{{{ _ingest.on_failure_message }}}"
+        "value": "Processor '{{{ _ingest.on_failure_processor_type }}}' {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}'"
+      }
+    },
+    {
+      "append": {
+        "field": "tags",
+        "value": "preserve_original_event",
+        "allow_duplicates": false
       }
     }
   ]

salt/elasticsearch/files/ingest/zeek.ja4d

@@ -0,0 +1,71 @@
+{
+    "description": "zeek.ja4d",
+    "processors": [
+        {
+            "set": {
+                "field": "event.dataset",
+                "value": "ja4d"
+            }
+        },
+        {
+            "remove": {
+                "field": [
+                    "host"
+                ],
+                "ignore_failure": true
+            }
+        },
+        {
+            "json": {
+                "field": "message",
+                "target_field": "message2",
+                "ignore_failure": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.ja4d",
+                "target_field": "hash.ja4d",
+                "ignore_missing": true,
+                "if": "ctx?.message2?.ja4d != null && ctx.message2.ja4d.length() > 0"
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.client_mac",
+                "target_field": "host.mac",
+                "ignore_missing": true,
+                "if": "ctx?.message2?.client_mac != null && ctx.message2.client_mac.length() > 0"
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.hostname",
+                "target_field": "host.hostname",
+                "ignore_missing": true,
+                "if": "ctx?.message2?.hostname != null && ctx.message2.hostname.length() > 0"
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.requested_ip",
+                "target_field": "dhcp.requested_address",
+                "ignore_missing": true,
+                "if": "ctx?.message2?.requested_ip != null && ctx.message2.requested_ip.length() > 0"
+            }
+        },
+        {
+            "rename": {
+                "field": "message2.vendor_class_id",
+                "target_field": "zeek.ja4d.vendor_class_id",
+                "ignore_missing": true,
+                "if": "ctx?.message2?.vendor_class_id != null && ctx.message2.vendor_class_id.length() > 0"
+            }
+        },
+        {
+            "pipeline": {
+                "name": "zeek.common"
+            }
+        }
+    ]
+}

salt/elasticsearch/files/log4j2.properties

@@ -45,3 +45,7 @@ appender.rolling_json.strategy.action.condition.nested_condition.age = 1D
 rootLogger.level = info
 rootLogger.appenderRef.rolling.ref = rolling
 rootLogger.appenderRef.rolling_json.ref = rolling_json
+
+# Suppress NotEntitledException WARNs (ES 9.3.3 bug)
+logger.entitlement_security.name = org.elasticsearch.entitlement.runtime.policy.PolicyManager.x-pack-security.org.elasticsearch.security.org.elasticsearch.xpack.security
+logger.entitlement_security.level = error

salt/elasticsearch/soc_elasticsearch.yaml

@@ -4,6 +4,13 @@ elasticsearch:
     forcedType: bool
     advanced: True
     helpLink: elasticsearch
+  data_retention_method:
+    description: Method for data retention. Options are ILM or DLM. For single node deployments and most distributed grid users, DLM will be the recommended option for simplified management. Those with more complex use cases may prefer ILM. The latter allows for more granular control, but requires more management overhead.
+    options:
+    - ILM
+    - DLM
+    forcedType: string
+    global: True
   version:
     description: "This specifies the version of the following containers: so-elastic-fleet-package-registry, so-elastic-agent, so-elastic-fleet, so-kibana, so-logstash and so-elasticsearch. Modifying this value in the Elasticsearch defaults.yaml will result in catastrophic grid failure."
     readonly: True
@@ -13,7 +20,7 @@ elasticsearch:
     description: Specify the memory heap size in (m)egabytes for Elasticsearch.
     helpLink: elasticsearch
   index_clean:
-    description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, indices will only be deleted by the age defined in the ILM settings. This setting only applies to EVAL, STANDALONE, and HEAVY NODE installations. Other installations can only use ILM settings.
+    description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, data is retained by the configured lifecycle settings. This setting only applies to EVAL, STANDALONE, and HEAVY NODE installations. Other installations use lifecycle settings only.
     forcedType: bool
     helpLink: elasticsearch
   vm:
@@ -139,6 +146,21 @@ elasticsearch:
     custom010: *pipelines
   index_settings:
     global_overrides:
+      data_stream_lifecycle:
+        data_retention:
+          description: |
+            The retention period for all data streams. Retention does not define the period that the data will be removed, but the minimum time period they will be kept.
+
+            Use a number followed by a time unit, such as 7d. Leave blank for indefinite retention where supported.
+
+            Configured retention period also affects the frequency of rolling over data streams.
+              - If retention is less than or equal to 1 day, max_age will be 1 hour
+              - If retention is less than or equal to 14 days, max_age will be 1 day
+              - If retention is less than or equal to 90 days, max_age will be 7 days
+              - If retention is greater than 90 days, max_age will be 30 days
+          forcedType: string
+          regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$
+          regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d.
       index_template:
         template:
           settings:
@@ -311,13 +333,28 @@ elasticsearch:
               forcedType: string
               global: True
               helpLink: elasticsearch
-    so-logs: &indexSettings
+    so-logs: &dataStreamSettings
       index_sorting:
         description: Sorts the index by event time, at the cost of additional processing resource consumption.
         forcedType: bool
         global: True
         advanced: True
         helpLink: elasticsearch
+      data_stream_lifecycle:
+        data_retention:
+          description: |
+            The retention period for this data stream. Retention does not define the period that the data will be removed, but the minimum time period it will be kept.
+
+            Use a number followed by a time unit, such as 7d. Leave blank for indefinite retention where supported.
+
+            Configured retention period also affects the frequency of rolling over this data stream.
+              - If retention is less than or equal to 1 day, max_age will be 1 hour
+              - If retention is less than or equal to 14 days, max_age will be 1 day
+              - If retention is less than or equal to 90 days, max_age will be 7 days
+              - If retention is greater than 90 days, max_age will be 30 days
+          forcedType: string
+          regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$
+          regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d.
       index_template:
         index_patterns:
           description: Patterns for matching multiple indices or tables.
@@ -335,6 +372,14 @@ elasticsearch:
                 global: True
                 advanced: True
                 helpLink: elasticsearch
+              auto_expand_replicas:
+                description: Automatically expand the number of replicas based on the number of data nodes in the cluster. This can help ensure high availability as the cluster scales up or down.
+                forcedType: string
+                regex: "^(0-[1-9]|1-[2-9]|2-[3-9]|3-[4-9]|4-[5-9]|5-[6-9]|6-[7-9]|7-[89]|8-9|[0-9]-all|false)$"
+                regexFailureMessage: Must be in the format of "x-y" where x is minimum number of replicas and y is maximum number of replicas, or "0-all" to specify a minimum of 0 and no maximum, or "false" to disable automatic replica expansion.
+                global: True
+                advanced: True
+                helpLink: elasticsearch
               mapping:
                 total_fields:
                   limit:
@@ -596,65 +641,349 @@ elasticsearch:
             global: True
             advanced: True
             helpLink: elasticsearch
-    so-logs-system_x_auth: *indexSettings
+    so-logs-system_x_auth: *dataStreamSettings
-    so-logs-system_x_syslog: *indexSettings
+    so-logs-system_x_syslog: *dataStreamSettings
-    so-logs-system_x_system: *indexSettings
+    so-logs-system_x_system: *dataStreamSettings
-    so-logs-system_x_application: *indexSettings
+    so-logs-system_x_application: *dataStreamSettings
-    so-logs-system_x_security: *indexSettings
+    so-logs-system_x_security: *dataStreamSettings
-    so-logs-windows_x_forwarded: *indexSettings
+    so-logs-windows_x_forwarded: *dataStreamSettings
-    so-logs-windows_x_powershell: *indexSettings
+    so-logs-windows_x_powershell: *dataStreamSettings
-    so-logs-windows_x_powershell_operational: *indexSettings
+    so-logs-windows_x_powershell_operational: *dataStreamSettings
-    so-logs-windows_x_sysmon_operational: *indexSettings
+    so-logs-windows_x_sysmon_operational: *dataStreamSettings
-    so-logs-winlog_x_winlog: *indexSettings
+    so-logs-winlog_x_winlog: *dataStreamSettings
-    so-logs-detections_x_alerts: *indexSettings
+    so-logs-detections_x_alerts: *dataStreamSettings
-    so-logs-http_endpoint_x_generic: *indexSettings
+    so-logs-http_endpoint_x_generic: *dataStreamSettings
-    so-logs-httpjson_x_generic: *indexSettings
+    so-logs-httpjson_x_generic: *dataStreamSettings
-    so-logs-osquery-manager-actions: *indexSettings
+    so-logs-osquery-manager-actions: *dataStreamSettings
-    so-logs-osquery-manager-action_x_responses: *indexSettings
+    so-logs-osquery-manager-action_x_responses: *dataStreamSettings
-    so-logs-osquery-manager_x_action_x_responses: *indexSettings
+    so-logs-osquery-manager_x_action_x_responses: *dataStreamSettings
-    so-logs-osquery-manager_x_result: *indexSettings
+    so-logs-osquery-manager_x_result: *dataStreamSettings
-    so-logs-elastic_agent_x_apm_server: *indexSettings
+    so-logs-elastic_agent_x_apm_server: *dataStreamSettings
-    so-logs-elastic_agent_x_auditbeat: *indexSettings
+    so-logs-elastic_agent_x_auditbeat: *dataStreamSettings
-    so-logs-elastic_agent_x_cloudbeat: *indexSettings
+    so-logs-elastic_agent_x_cloudbeat: *dataStreamSettings
-    so-logs-elastic_agent_x_endpoint_security: *indexSettings
+    so-logs-elastic_agent_x_endpoint_security: *dataStreamSettings
-    so-logs-endpoint_x_alerts: *indexSettings
+    so-logs-endpoint_x_alerts: *dataStreamSettings
-    so-logs-endpoint_x_events_x_api: *indexSettings
+    so-logs-endpoint_x_events_x_api: *dataStreamSettings
-    so-logs-endpoint_x_events_x_file: *indexSettings
+    so-logs-endpoint_x_events_x_file: *dataStreamSettings
-    so-logs-endpoint_x_events_x_library: *indexSettings
+    so-logs-endpoint_x_events_x_library: *dataStreamSettings
-    so-logs-endpoint_x_events_x_network: *indexSettings
+    so-logs-endpoint_x_events_x_network: *dataStreamSettings
-    so-logs-endpoint_x_events_x_process: *indexSettings
+    so-logs-endpoint_x_events_x_process: *dataStreamSettings
-    so-logs-endpoint_x_events_x_registry: *indexSettings
+    so-logs-endpoint_x_events_x_registry: *dataStreamSettings
-    so-logs-endpoint_x_events_x_security: *indexSettings
+    so-logs-endpoint_x_events_x_security: *dataStreamSettings
-    so-logs-elastic_agent_x_filebeat: *indexSettings
+    so-logs-elastic_agent_x_filebeat: *dataStreamSettings
-    so-logs-elastic_agent_x_fleet_server: *indexSettings
+    so-logs-elastic_agent_x_fleet_server: *dataStreamSettings
-    so-logs-elastic_agent_x_heartbeat: *indexSettings
+    so-logs-elastic_agent_x_heartbeat: *dataStreamSettings
-    so-logs-elastic_agent: *indexSettings
+    so-logs-elastic_agent: *dataStreamSettings
-    so-logs-elastic_agent_x_metricbeat: *indexSettings
+    so-logs-elastic_agent_x_metricbeat: *dataStreamSettings
-    so-logs-elastic_agent_x_osquerybeat: *indexSettings
+    so-logs-elastic_agent_x_osquerybeat: *dataStreamSettings
-    so-logs-elastic_agent_x_packetbeat: *indexSettings
+    so-logs-elastic_agent_x_packetbeat: *dataStreamSettings
-    so-logs-elasticsearch_x_server: *indexSettings
+    so-logs-elasticsearch_x_server: *dataStreamSettings
-    so-metrics-endpoint_x_metadata: *indexSettings
+    so-metrics-endpoint_x_metadata: *dataStreamSettings
-    so-metrics-endpoint_x_metrics: *indexSettings
+    so-metrics-endpoint_x_metrics: *dataStreamSettings
-    so-metrics-endpoint_x_policy: *indexSettings
+    so-metrics-endpoint_x_policy: *dataStreamSettings
-    so-metrics-nginx_x_stubstatus: *indexSettings
+    so-metrics-nginx_x_stubstatus: *dataStreamSettings
-    so-metrics-vsphere_x_datastore: *indexSettings
+    so-metrics-vsphere_x_datastore: *dataStreamSettings
-    so-metrics-vsphere_x_host: *indexSettings
+    so-metrics-vsphere_x_host: *dataStreamSettings
-    so-metrics-vsphere_x_virtualmachine: *indexSettings
+    so-metrics-vsphere_x_virtualmachine: *dataStreamSettings
-    so-case: *indexSettings
+    so-common: *dataStreamSettings
-    so-common: *indexSettings
+    so-endgame: *dataStreamSettings
-    so-endgame: *indexSettings
+    so-idh: *dataStreamSettings
-    so-idh: *indexSettings
+    so-suricata: *dataStreamSettings
-    so-suricata: *indexSettings
+    so-suricata_x_alerts: *dataStreamSettings
-    so-suricata_x_alerts: *indexSettings
+    so-import: *dataStreamSettings
-    so-import: *indexSettings
+    so-kratos: *dataStreamSettings
-    so-kratos: *indexSettings
+    so-hydra: *dataStreamSettings
-    so-hydra: *indexSettings
+    so-kismet: *dataStreamSettings
-    so-kismet: *indexSettings
+    so-logstash: *dataStreamSettings
-    so-logstash: *indexSettings
+    so-redis: *dataStreamSettings
-    so-redis: *indexSettings
+    so-strelka: *dataStreamSettings
-    so-strelka: *indexSettings
+    so-syslog: *dataStreamSettings
-    so-syslog: *indexSettings
+    so-zeek: *dataStreamSettings
-    so-zeek: *indexSettings
+    # Managed SOC integration annotations are inserted below this line. Referencing '*dataStreamSettings'
+    so-case: &indexSettings
+      index_sorting:
+        description: Sorts the index by event time, at the cost of additional processing resource consumption.
+        forcedType: bool
+        global: True
+        advanced: True
+        helpLink: elasticsearch
+      index_template:
+        index_patterns:
+          description: Patterns for matching multiple indices or tables.
+          forcedType: "[]string"
+          multiline: True
+          global: True
+          advanced: True
+          helpLink: elasticsearch
+        template:
+          settings:
+            index:
+              number_of_replicas:
+                description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs.
+                forcedType: int
+                global: True
+                advanced: True
+                helpLink: elasticsearch
+              auto_expand_replicas:
+                description: Automatically expand the number of replicas based on the number of data nodes in the cluster. This can help ensure high availability as the cluster scales up or down.
+                forcedType: string
+                regex: "^(0-[1-9]|1-[2-9]|2-[3-9]|3-[4-9]|4-[5-9]|5-[6-9]|6-[7-9]|7-[89]|8-9|[0-9]-all|false)$"
+                regexFailureMessage: Must be in the format of "x-y" where x is minimum number of replicas and y is maximum number of replicas, or "0-all" to specify a minimum of 0 and no maximum, or "false" to disable automatic replica expansion.
+                global: True
+                advanced: True
+                helpLink: elasticsearch
+              mapping:
+                total_fields:
+                  limit:
+                    description: Max number of fields that can exist on a single index. Larger values will consume more resources.
+                    global: True
+                    advanced: True
+                    helpLink: elasticsearch
+              refresh_interval:
+                description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation.
+                global: True
+                advanced: True
+                helpLink: elasticsearch
+              number_of_shards:
+                description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs.
+                global: True
+                advanced: True
+                helpLink: elasticsearch
+              sort:
+                field:
+                  description: The field to sort by. Must set index_sorting to True.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+                order:
+                  description: The order to sort by. Must set index_sorting to True.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+          mappings:
+            _meta:
+              package:
+                name:
+                  description: Meta settings for the mapping.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+              managed_by:
+                  description: Meta settings for the mapping.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+              managed:
+                  description: Meta settings for the mapping.
+                  forcedType: bool
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+        composed_of:
+          description: The index template is composed of these component templates.
+          forcedType: "[]string"
+          global: True
+          advanced: True
+          helpLink: elasticsearch
+        priority:
+          description: The priority of the index template.
+          forcedType: int
+          global: True
+          advanced: True
+          helpLink: elasticsearch
+      policy:
+        phases:
+          hot:
+            min_age:
+              description: Minimum age of index. This determines when the index should be moved to the hot tier.
+              global: True
+              advanced: True
+              helpLink: elasticsearch
+            actions:
+              set_priority:
+                priority:
+                  description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
+                  forcedType: int
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+              rollover:
+                max_age:
+                  description: Maximum age of index.  Once an index reaches this limit, it will be rolled over into a new index.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+                max_primary_shard_size:
+                  description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+              shrink:
+                method:
+                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
+                  options:
+                  - COUNT
+                  - SIZE
+                  global: True
+                  advanced: True
+                  forcedType: string
+                number_of_shards:
+                  title: shard count
+                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                max_primary_shard_size:
+                  title: max shard size
+                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
+                  regex: ^[0-9]+(?:gb|tb|pb)$
+                  global: True
+                  forcedType: string
+                  advanced: True
+                allow_write_after_shrink:
+                  description: Allow writes after shrink.
+                  global: True
+                  forcedType: bool
+                  default: False
+                  advanced: True
+              forcemerge:
+                max_num_segments:
+                  description: Reduce the number of segments in each index shard and clean up deleted documents.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                index_codec:
+                  title: compression
+                  description: Use higher compression for stored fields at the cost of slower performance.
+                  forcedType: bool
+                  global: True
+                  default: False
+                  advanced: True
+          warm:
+            min_age:
+              description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
+              regex: ^[0-9]{1,5}d$
+              forcedType: string
+              global: True
+              advanced: True
+              helpLink: elasticsearch
+            actions:
+              set_priority:
+                priority:
+                  description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
+                  forcedType: int
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+              rollover:
+                max_age:
+                  description: Maximum age of index.  Once an index reaches this limit, it will be rolled over into a new index.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+                max_primary_shard_size:
+                  description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+              shrink:
+                method:
+                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
+                  options:
+                  - COUNT
+                  - SIZE
+                  global: True
+                  advanced: True
+                number_of_shards:
+                  title: shard count
+                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                max_primary_shard_size:
+                  title: max shard size
+                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
+                  regex: ^[0-9]+(?:gb|tb|pb)$
+                  global: True
+                  forcedType: string
+                  advanced: True
+                allow_write_after_shrink:
+                  description: Allow writes after shrink.
+                  global: True
+                  forcedType: bool
+                  default: False
+                  advanced: True
+              forcemerge:
+                max_num_segments:
+                  description: Reduce the number of segments in each index shard and clean up deleted documents.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                index_codec:
+                  title: compression
+                  description: Use higher compression for stored fields at the cost of slower performance.
+                  forcedType: bool
+                  global: True
+                  default: False
+                  advanced: True
+              allocate:
+                number_of_replicas:
+                  description: Set the number of replicas. Remains the same as the previous phase by default.
+                  forcedType: int
+                  global: True
+                  advanced: True
+          cold:
+            min_age:
+              description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
+              regex: ^[0-9]{1,5}d$
+              forcedType: string
+              global: True
+              advanced: True
+              helpLink: elasticsearch
+            actions:
+              set_priority:
+                priority:
+                  description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
+                  forcedType: int
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+              allocate:
+                number_of_replicas:
+                  description: Set the number of replicas. Remains the same as the previous phase by default.
+                  forcedType: int
+                  global: True
+                  advanced: True
+          delete:
+            min_age:
+              description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.
+              regex: ^[0-9]{1,5}d$
+              forcedType: string
+              global: True
+              advanced: True
+              helpLink: elasticsearch
+        _meta:
+          package:
+            name:
+              description: Meta settings for the mapping.
+              global: True
+              advanced: True
+              helpLink: elasticsearch
+          managed_by:
+            description: Meta settings for the mapping.
+            global: True
+            advanced: True
+            helpLink: elasticsearch
+          managed:
+            description: Meta settings for the mapping.
+            forcedType: bool
+            global: True
+            advanced: True
+            helpLink: elasticsearch
+    sos-backup: *indexSettings
+    so-detection: *indexSettings
+    so-assistant-chat: *indexSettings
+    so-assistant-session: *indexSettings
     so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings
       index_sorting:
         description: Sorts the index by event time, at the cost of additional processing resource consumption.

salt/elasticsearch/template.map.jinja

@@ -5,6 +5,7 @@
 
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
 {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %}
+{% set DATA_RETENTION_METHOD = salt['pillar.get']('elasticsearch:data_retention_method', ELASTICSEARCHDEFAULTS.elasticsearch.get('data_retention_method', 'ILM')) %}
 
 {% set PILLAR_GLOBAL_OVERRIDES = {} %}
 {% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings', {}) %}
@@ -14,16 +15,43 @@
 
 {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %}
 
+{% set ALL_ADDON_INTEGRATION_DEFAULTS = {} %}
+{% set ALL_ADDON_SETTINGS_ORIG = {} %}
+{% set ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES = {} %}
+{% set ALL_ADDON_SETTINGS = {} %}
 {# start generation of integration default index_settings #}
-{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') and salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %}
+{% if salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %}
-{%   set check_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %}
+{#   import integration type defaults #}
-{%   if check_package_components.size > 1 %}
+{%   if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') %}
+{%     set check_integration_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %}
+{%     if check_integration_package_components.size > 1 %}
 {%       from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}
-{%    for index, settings in ADDON_INTEGRATION_DEFAULTS.items() %}
+{%       do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INTEGRATION_DEFAULTS) %}
-{%      do ES_INDEX_SETTINGS_ORIG.update({index: settings}) %}
-{%    endfor %}
 {%     endif %}
 {%   endif %}
+
+{#   import input type defaults #}
+{%   if salt['file.file_exists']('/opt/so/state/esfleet_input_package_components.json') %}
+{%     set check_input_package_components = salt['file.stats']('/opt/so/state/esfleet_input_package_components.json') %}
+{%     if check_input_package_components.size > 1 %}
+{%       from 'elasticfleet/input-defaults.map.jinja' import ADDON_INPUT_INTEGRATION_DEFAULTS %}
+{%       do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INPUT_INTEGRATION_DEFAULTS) %}
+{%     endif %}
+{%   endif %}
+
+{#   import content type defaults #}
+{%   if salt['file.file_exists']('/opt/so/state/esfleet_content_package_components.json') %}
+{%     set check_content_package_components = salt['file.stats']('/opt/so/state/esfleet_content_package_components.json') %}
+{%     if check_content_package_components.size > 1 %}
+{%       from 'elasticfleet/content-defaults.map.jinja' import ADDON_CONTENT_INTEGRATION_DEFAULTS %}
+{%       do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_CONTENT_INTEGRATION_DEFAULTS) %}
+{%     endif %}
+{%   endif %}
+
+{%   for index, settings in ALL_ADDON_INTEGRATION_DEFAULTS.items() %}
+{%     do ALL_ADDON_SETTINGS_ORIG.update({index: settings}) %}
+{%   endfor %}
+{% endif %}
 {# end generation of integration default index_settings #}
 
 {% set ES_INDEX_SETTINGS_GLOBAL_OVERRIDES = {} %}
@@ -31,25 +59,43 @@
 {%   do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}
 {% endfor %}
 
+{% if ALL_ADDON_SETTINGS_ORIG.keys() | length > 0 %}
+{%   for index in ALL_ADDON_SETTINGS_ORIG.keys() %}
+{%     do ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ALL_ADDON_SETTINGS_ORIG[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}
+{#     Explicitly excluding addon indices from ES_INDEX_SETTINGS_ORIG
+         When manager.soc_managed_annotations runs, new entries are added to the salt/elasticsearch/defaults.yaml file to support 'revert to default' functionality.
+         Subsequent map renders will then incorrectly include 'integration X' in 'ES_INDEX_SETTINGS_ORIG' due to being in the defaults.yaml file. #}
+{%     if index in ES_INDEX_SETTINGS_ORIG.keys() %}
+{%       do ES_INDEX_SETTINGS_ORIG.pop(index) %}
+{%     endif %}
+{%   endfor %}
+{% endif %}
+
 {% set ES_INDEX_SETTINGS = {} %}
-{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
+{% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS, EXCLUDE_INDICES=[]) %}
-{% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %}
+
+{% do GLOBAL_OVERRIDES.update(salt['defaults.merge'](GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
+{% for index, settings in GLOBAL_OVERRIDES.items() %}
+
+{%   if index in EXCLUDE_INDICES %}
+{%     continue %}
+{%   endif %}
 
 {#   prevent this action from being performed on custom defined indices. #}
 {#   the custom defined index is not present in either of the dictionaries and fails to reder. #}
-{%   if index in ES_INDEX_SETTINGS_ORIG and index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES %}
+{%   if index in DEFINED_SETTINGS and index in GLOBAL_OVERRIDES %}
 
 {#     dont merge policy from the global_overrides if policy isn't defined in the original index settingss #}
 {#     this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #}
-{%     if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
+{%     if not DEFINED_SETTINGS[index].policy is defined and GLOBAL_OVERRIDES[index].policy is defined %}
-{%       do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %}
+{%       do GLOBAL_OVERRIDES[index].pop('policy') %}
 {%     endif %}
 
 {#     this prevents and index from inderiting a policy phase from global overrides if it wasnt defined in the defaults. #}
-{%     if ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
+{%     if GLOBAL_OVERRIDES[index].policy is defined %}
-{%       for phase in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.copy() %}
+{%       for phase in GLOBAL_OVERRIDES[index].policy.phases.copy() %}
-{%         if ES_INDEX_SETTINGS_ORIG[index].policy.phases[phase] is not defined %}
+{%         if DEFINED_SETTINGS[index].policy.phases[phase] is not defined %}
-{%           do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %}
+{%           do GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %}
 {%         endif %}
 {%       endfor %}
 {%     endif %}
@@ -60,6 +106,17 @@
 {%     if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %}
 {%       do settings.index_template.template.settings.index.pop('sort') %}
 {%     endif %}
+{%     if DATA_RETENTION_METHOD == 'DLM' and settings.index_template.data_stream is defined and settings.data_stream_lifecycle is defined %}
+{%       if settings.data_stream_lifecycle.data_retention is defined and settings.data_stream_lifecycle.data_retention %}
+{%         do settings.index_template.template.update({'lifecycle': {'data_retention': settings.data_stream_lifecycle.data_retention}}) %}
+{%       else %}
+{%         do settings.index_template.template.update({'lifecycle': {}}) %}
+{%       endif %}
+{%       if settings.index_template.template.settings.index.lifecycle is not defined %}
+{%         do settings.index_template.template.settings.index.update({'lifecycle': {}}) %}
+{%       endif %}
+{%       do settings.index_template.template.settings.index.lifecycle.update({'prefer_ilm': false}) %}
+{%     endif %}
 {%   endif %}
 
 {# advanced ilm actions #}
@@ -111,5 +168,23 @@
 {%     endfor %}
 {%   endif %}
 
-{%   do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %}
+{%   do FINAL_INDEX_SETTINGS.update({index | replace("_x_", "."): GLOBAL_OVERRIDES[index]}) %}
+{% endfor %}
+{% endmacro %}
+
+{# Exclude addon integrations from final ES_INDEX_SETTINGS #}
+{{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS, ALL_ADDON_SETTINGS_ORIG.keys() | list ) }}
+
+{# Exclude SO managed indices, otherwise ALL_ADDON_SETTINGS will include pillar values
+  of core integrations without merging defaults, resulting in an overlapping, but bad index template being generated. #}
+{{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS, ES_INDEX_SETTINGS_ORIG.keys() | list ) }}
+
+{% set SO_MANAGED_INDICES = [] %}
+{% for index, settings in ES_INDEX_SETTINGS.items() %}
+{%   do SO_MANAGED_INDICES.append(index) %}
+{% endfor %}
+
+{% set ADDON_INDICES = [] %}
+{% for index, settings in ALL_ADDON_SETTINGS.items() %}
+{%   do ADDON_INDICES.append(index) %}
 {% endfor %}

salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list

@@ -6,8 +6,19 @@
 # Elastic License 2.0.
 
 . /usr/sbin/so-common
-if [ "$1" == "" ]; then
+
-    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template | jq '.component_templates[] |.name'| sort
+if [[ -z "$1" ]]; then
+    if output=$(so-elasticsearch-query "_component_template" --retry 3 --retry-delay 1 --fail); then
+        jq '[.component_templates[] | .name] | sort' <<< "$output"
     else
-    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template/$1 | jq
+        echo "Failed to retrieve component templates from Elasticsearch."
+        exit 1
+    fi
+else
+    if output=$(so-elasticsearch-query "_component_template/$1" --retry 3 --retry-delay 1 --fail); then
+        jq <<< "$output"
+    else
+        echo "Failed to retrieve component template '$1' from Elasticsearch."
+        exit 1
+    fi
 fi

salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load

@@ -0,0 +1,268 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+SO_STATEFILE_SUCCESS=/opt/so/state/estemplates.txt
+ADDON_STATEFILE_SUCCESS=/opt/so/state/addon_estemplates.txt
+ELASTICSEARCH_TEMPLATES_DIR="/opt/so/conf/elasticsearch/templates"
+SO_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/index"
+ADDON_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/addon-index"
+SO_LOAD_FAILURES=0
+ADDON_LOAD_FAILURES=0
+SO_LOAD_FAILURES_NAMES=()
+ADDON_LOAD_FAILURES_NAMES=()
+IS_HEAVYNODE="false"
+FORCE="false"
+VERBOSE="false"
+SHOULD_EXIT_ON_FAILURE="true"
+
+# If soup is running, ignore errors
+pgrep soup >/dev/null && SHOULD_EXIT_ON_FAILURE="false"
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+    --heavynode)
+        IS_HEAVYNODE="true"
+        ;;
+    --force)
+        FORCE="true"
+        ;;
+    --verbose)
+        VERBOSE="true"
+        ;;
+    *)
+        echo "Usage: $0 [options]"
+        echo "Options:"
+        echo "  --heavynode     Only loads index templates specific to heavynodes"
+        echo "  --force     Force reload all templates regardless of statefiles (default: false)"
+        echo "  --verbose     Enable verbose output"
+        exit 1
+        ;;
+    esac
+    shift
+done
+
+load_template() {
+    local uri="$1"
+    local file="$2"
+
+    echo "Loading template file $file"
+    if ! output=$(retry 3 3 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"); then
+        echo "$output"
+
+        return 1
+
+    elif [[ "$VERBOSE" == "true" ]]; then
+        echo "$output"
+    fi
+
+}
+
+check_required_component_template_exists() {
+    local required
+    local missing
+    local file=$1
+
+    required=$(jq '[((.composed_of //[]) - (.ignore_missing_component_templates // []))[]]' "$file")
+    missing=$(jq -n --argjson required "$required" --argjson component_templates "$component_templates" '(($required) - ($component_templates))')
+
+    if [[ $(jq length <<<"$missing") -gt 0 ]]; then
+
+        return 1
+    fi
+}
+
+check_heavynode_compatiable_index_template() {
+    # The only templates that are relevant to heavynodes are from datasets defined in elasticagent/files/elastic-agent.yml.jinja.
+    # Heavynodes do not have fleet server packages installed and do not support elastic agents reporting directly to them.
+    local -A heavynode_index_templates=(
+        ["so-import"]=1
+        ["so-syslog"]=1
+        ["so-logs-soc"]=1
+        ["so-suricata"]=1
+        ["so-suricata.alerts"]=1
+        ["so-zeek"]=1
+        ["so-strelka"]=1
+    )
+
+    local template_name="$1"
+
+    if [[ ! -v heavynode_index_templates["$template_name"] ]]; then
+
+        return 1
+    fi
+
+}
+
+load_component_templates() {
+    local printed_name="$1"
+    local pattern="${ELASTICSEARCH_TEMPLATES_DIR}/component/$2"
+    local append_mappings="${3:-"false"}"
+
+    echo -e "\nLoading $printed_name component templates...\n"
+
+    if ! compgen -G "${pattern}/*.json" > /dev/null; then
+        echo "No $printed_name component templates found in ${pattern}, skipping."
+        return
+    fi
+
+    for component in "$pattern"/*.json; do
+        tmpl_name=$(basename "${component%.json}")
+
+        if [[ "$append_mappings" == "true" ]]; then
+            # avoid duplicating "-mappings" if it already exists in the component template filename
+            tmpl_name="${tmpl_name%-mappings}-mappings"
+        fi
+
+        if ! load_template "_component_template/${tmpl_name}" "$component"; then
+            SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
+            SO_LOAD_FAILURES_NAMES+=("$component")
+        fi
+    done
+}
+
+index_templates_exist() {
+    local templates_dir="$1"
+
+    if [[ ! -d "$templates_dir" ]]; then
+        return 1
+    fi
+
+    compgen -G "${templates_dir}/*.json" > /dev/null
+}
+
+should_load_addon_templates() {
+    if [[ "$IS_HEAVYNODE" == "true" ]]; then
+        return 1
+    fi
+
+    # Skip statefile checks when forcing template load
+    if [[ "$FORCE" != "true" ]]; then
+        if [[ ! -f "$SO_STATEFILE_SUCCESS" || -f "$ADDON_STATEFILE_SUCCESS" ]]; then
+            return 1
+        fi
+    fi
+
+    index_templates_exist "$ADDON_TEMPLATES_DIR"
+}
+
+if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]] && index_templates_exist "$SO_TEMPLATES_DIR"; then
+    check_elasticsearch_responsive
+
+    if [[ "$IS_HEAVYNODE" == "false" ]]; then
+        # TODO: Better way to check if fleet server is installed vs checking for Elastic Defend component template.
+        fleet_check="logs-endpoint.alerts@package"
+        if ! so-elasticsearch-query "_component_template/$fleet_check" --output /dev/null --retry 5 --retry-delay 3 --fail; then
+            # This check prevents so-elasticsearch-templates-load from running before so-elastic-fleet-setup has run.
+            echo -e "\nPackage $fleet_check not yet installed. Fleet Server may not be fully configured yet."
+            # Fleet Server is required because some SO index templates depend on components installed via
+            #  specific integrations eg Elastic Defend. These are components that we do not manually create / manage
+            #  via /opt/so/saltstack/salt/elasticsearch/templates/component/
+
+            exit 0
+        fi
+    fi
+
+    # load_component_templates "Name" "directory" "append '-mappings'?"
+    load_component_templates "ECS" "ecs" "true"
+    load_component_templates "Elastic Agent" "elastic-agent"
+    load_component_templates "Security Onion" "so"
+
+    component_templates=$(so-elasticsearch-component-templates-list)
+    echo -e "Loading Security Onion index templates...\n"
+    for so_idx_tmpl in "${SO_TEMPLATES_DIR}"/*.json; do
+        tmpl_name=$(basename "${so_idx_tmpl%-template.json}")
+
+        if [[ "$IS_HEAVYNODE" == "true" ]]; then
+            # TODO: Better way to load only heavynode specific templates
+            if ! check_heavynode_compatiable_index_template "$tmpl_name"; then
+                if [[ "$VERBOSE" == "true" ]]; then
+                    echo "Skipping over $so_idx_tmpl, template is not a heavynode specific index template."
+                fi
+
+                continue
+            fi
+        fi
+
+        if check_required_component_template_exists "$so_idx_tmpl"; then
+            if ! load_template "_index_template/$tmpl_name" "$so_idx_tmpl"; then
+                SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
+                SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl")
+            fi
+        else
+            echo "Skipping over $so_idx_tmpl due to missing required component template(s)."
+            SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
+            SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl")
+
+            continue
+        fi
+    done
+
+    if [[ $SO_LOAD_FAILURES -eq 0 ]]; then
+        echo "All Security Onion core templates loaded successfully."
+
+        touch "$SO_STATEFILE_SUCCESS"
+    else
+        echo "Encountered $SO_LOAD_FAILURES failure(s) loading templates:"
+        for failed_template in "${SO_LOAD_FAILURES_NAMES[@]}"; do
+            echo "  - $failed_template"
+        done
+        if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then
+            fail "Failed to load all Security Onion core templates successfully."
+        fi
+    fi
+elif ! index_templates_exist "$SO_TEMPLATES_DIR"; then
+    echo "No Security Onion core index templates found in ${SO_TEMPLATES_DIR}, skipping."
+elif [[ -f "$SO_STATEFILE_SUCCESS" ]]; then
+    echo "Security Onion core templates already loaded"
+fi
+
+# Start loading addon templates
+if should_load_addon_templates; then
+
+    check_elasticsearch_responsive
+
+    echo -e "\nLoading addon integration index templates...\n"
+    component_templates=$(so-elasticsearch-component-templates-list)
+
+    for addon_idx_tmpl in "${ADDON_TEMPLATES_DIR}"/*.json; do
+        tmpl_name=$(basename "${addon_idx_tmpl%-template.json}")
+
+        if check_required_component_template_exists "$addon_idx_tmpl"; then
+            if ! load_template "_index_template/${tmpl_name}" "$addon_idx_tmpl"; then
+                ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1))
+                ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl")
+            fi
+        else
+            echo "Skipping over $addon_idx_tmpl due to missing required component template(s)."
+            ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1))
+            ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl")
+
+            continue
+        fi
+    done
+
+    if [[ $ADDON_LOAD_FAILURES -eq 0 ]]; then
+        echo "All addon integration templates loaded successfully."
+
+        touch "$ADDON_STATEFILE_SUCCESS"
+    else
+        echo "Encountered $ADDON_LOAD_FAILURES failure(s) loading addon integration templates:"
+        for failed_template in "${ADDON_LOAD_FAILURES_NAMES[@]}"; do
+            echo "  - $failed_template"
+        done
+        if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then
+            fail "Failed to load all addon integration templates successfully."
+        fi
+    fi
+
+elif [[ ! -f "$SO_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" ]]; then
+    echo "Skipping loading addon integration templates until Security Onion core templates have been loaded."
+
+elif [[ -f "$ADDON_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" && "$FORCE" == "false" ]]; then
+    echo "Addon integration templates already loaded"
+fi

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply

@@ -0,0 +1,178 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+{%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
+
+{%- set DATA_RETENTION_METHOD = salt['pillar.get']('elasticsearch:data_retention_method', ELASTICSEARCHDEFAULTS.elasticsearch.get('data_retention_method', 'ILM')) %}
+
+ELASTICSEARCH_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR:-/opt/so/conf/elasticsearch/templates}"
+TEMPLATE_DIRS=(
+    "${ELASTICSEARCH_TEMPLATES_DIR}/index"
+    "${ELASTICSEARCH_TEMPLATES_DIR}/addon-index"
+)
+DATA_RETENTION_METHOD=$(cat <<'EOF'
+{{ DATA_RETENTION_METHOD }}
+EOF
+)
+DLM_FAILURES=0
+DLM_FAILURE_NAMES=()
+
+if [[ "$DATA_RETENTION_METHOD" != "DLM" && "$DATA_RETENTION_METHOD" != "ILM" ]]; then
+    echo "Unsupported data retention method $DATA_RETENTION_METHOD. Expected DLM or ILM."
+    exit 1
+fi
+
+validate_template_file() {
+    local template_file="$1"
+
+    if ! jq -e 'type == "object" and (.data_stream == null or (.data_stream | type == "object")) and (.template.lifecycle == null or (.template.lifecycle | type == "object")) and (.template.lifecycle.data_retention == null or (.template.lifecycle.data_retention | type == "string"))' >/dev/null 2>&1 "$template_file"; then
+        echo "Invalid index template JSON: $template_file"
+        return 1
+    fi
+}
+
+is_data_stream_template() {
+    jq -e '.data_stream | type == "object"' >/dev/null 2>&1 "$1"
+}
+
+has_data_stream_lifecycle() {
+    jq -e '.template.lifecycle | type == "object"' >/dev/null 2>&1 "$1"
+}
+
+get_data_retention() {
+    jq -r '.template.lifecycle.data_retention // ""' "$1"
+}
+
+find_template_file() {
+    local template="$1"
+    local template_dir
+    local template_file
+
+    for template_dir in "${TEMPLATE_DIRS[@]}"; do
+        template_file="${template_dir}/${template}-template.json"
+
+        if [[ -f "$template_file" ]]; then
+            echo "$template_file"
+            return 0
+        fi
+    done
+
+    return 1
+}
+
+set_data_stream_lifecycle() {
+    local data_stream="$1"
+    local data_retention="$2"
+    local body
+    local output
+
+    if [[ -n "$data_retention" ]]; then
+        if jq -e --arg data_stream "$data_stream" --arg data_retention "$data_retention" '.data_streams[]? | select(.name == $data_stream and .lifecycle.enabled == true and .lifecycle.data_retention == $data_retention)' >/dev/null 2>&1 <<< "$data_streams"; then
+            echo "DLM lifecycle already set for $data_stream with data_retention $data_retention, skipping."
+            return 0
+        fi
+    elif jq -e --arg data_stream "$data_stream" '.data_streams[]? | select(.name == $data_stream and .lifecycle.enabled == true and (.lifecycle.data_retention == null))' >/dev/null 2>&1 <<< "$data_streams"; then
+        echo "DLM lifecycle already set for $data_stream with indefinite retention, skipping."
+        return 0
+    fi
+
+    if [[ -n "$data_retention" ]]; then
+        body=$(jq -cn --arg data_retention "$data_retention" '{data_retention: $data_retention}')
+    else
+        # Setting indefinite retention
+        body='{}'
+    fi
+
+    if ! output=$(so-elasticsearch-query "_data_stream/${data_stream}/_lifecycle" -XPUT -d "$body" --retry 3 --retry-delay 5 --fail); then
+        echo "Failed to set data stream lifecycle for $data_stream."
+        echo "$output"
+        return 1
+    fi
+
+    if [[ -n "$data_retention" ]]; then
+        echo "Set DLM lifecycle for $data_stream with data_retention $data_retention."
+    else
+        echo "Set DLM lifecycle for $data_stream with indefinite retention."
+    fi
+}
+
+disable_data_stream_lifecycle() {
+    local data_stream="$1"
+    local body='{"enabled":false}'
+    local output
+
+    if ! jq -e --arg data_stream "$data_stream" '.data_streams[]? | select(.name == $data_stream and .lifecycle != null and .lifecycle.enabled != false)' >/dev/null 2>&1 <<< "$data_streams"; then
+        # No action needed
+        return 0
+    fi
+
+    if ! output=$(so-elasticsearch-query "_data_stream/${data_stream}/_lifecycle" -XPUT -d "$body" --retry 3 --retry-delay 5 --fail); then
+        echo "Failed to disable data stream lifecycle for $data_stream."
+        echo "$output"
+        return 1
+    fi
+
+    echo "Disabled DLM lifecycle for $data_stream."
+}
+
+process_data_stream() {
+    local data_stream="$1"
+    local data_retention="$2"
+
+    if [[ "$DATA_RETENTION_METHOD" == "DLM" ]]; then
+        set_data_stream_lifecycle "$data_stream" "$data_retention"
+    else
+        disable_data_stream_lifecycle "$data_stream"
+    fi
+}
+
+check_elasticsearch_responsive
+
+if ! data_streams=$(so-elasticsearch-query "_data_stream?format=json" --retry 3 --retry-delay 5 --fail); then
+    echo "Failed to retrieve data streams."
+    exit 1
+fi
+
+while read -r data_stream_config; do
+    data_stream=$(jq -r '.name' <<< "$data_stream_config")
+    template=$(jq -r '.template' <<< "$data_stream_config")
+
+    if ! template_file=$(find_template_file "$template"); then
+        echo "Skipping $data_stream: index template file not found for $template."
+        continue
+    fi
+
+    validate_template_file "$template_file" || exit 1
+
+    if ! is_data_stream_template "$template_file"; then
+        echo "Skipping $data_stream: $template_file is not a data stream template."
+        continue
+    fi
+
+    if [[ "$DATA_RETENTION_METHOD" == "DLM" ]] && ! has_data_stream_lifecycle "$template_file"; then
+        echo "Skipping $data_stream: $template_file does not define data stream lifecycle."
+        continue
+    fi
+
+    data_retention=$(get_data_retention "$template_file")
+
+    if ! process_data_stream "$data_stream" "$data_retention"; then
+        DLM_FAILURES=$((DLM_FAILURES + 1))
+        DLM_FAILURE_NAMES+=("$data_stream")
+    fi
+done < <(jq -c '.data_streams[]' <<< "$data_streams")
+
+if [[ $DLM_FAILURES -eq 0 ]]; then
+    echo "Data stream lifecycle updates completed successfully."
+else
+    echo "Encountered $DLM_FAILURES failure(s) updating data stream lifecycle:"
+    for failed_data_stream in "${DLM_FAILURE_NAMES[@]}"; do
+        echo "  - $failed_data_stream"
+    done
+    exit 1
+fi

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load

@@ -7,6 +7,9 @@
 . /usr/sbin/so-common
 
 {%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
+{%- if GLOBALS.role != "so-heavynode" %}
+{%-   from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
+{%- endif %}
 
 {%- for index, settings in ES_INDEX_SETTINGS.items() %}
 {%-   if settings.policy is defined %}
@@ -33,3 +36,13 @@
 {%-   endif %}
 {%- endfor %}
 echo
+{%- if GLOBALS.role != "so-heavynode" %}
+{%-   for index, settings in ALL_ADDON_SETTINGS.items() %}
+{%-     if settings.policy is defined %}
+  echo
+  echo "Setting up {{ index }}-logs policy..."
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+  echo
+{%-     endif %}
+{%-   endfor %}
+{%- endif %}

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load

@@ -1,165 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
-{%  from 'vars/globals.map.jinja' import GLOBALS %}
-
-STATE_FILE_INITIAL=/opt/so/state/estemplates_initial_load_attempt.txt
-STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt
-
-if [[ -f $STATE_FILE_INITIAL ]]; then
-  # The initial template load has already run. As this is a subsequent load, all dependencies should 
-  # already be satisified. Therefore, immediately exit/abort this script upon any template load failure
-  # since this is an unrecoverable failure.
-  should_exit_on_failure=1
-else
-  # This is the initial template load, and there likely are some components not yet setup in Elasticsearch.
-  # Therefore load as many templates as possible at this time and if an error occurs proceed to the next 
-  # template. But if at least one template fails to load do not mark the templates as having been loaded.
-  # This will allow the next load to resume the load of the templates that failed to load initially.
-  should_exit_on_failure=0
-  echo "This is the initial template load"
-fi
-
-# If soup is running, ignore errors
-pgrep soup > /dev/null && should_exit_on_failure=0
-
-load_failures=0
-
-load_template() {
-  uri=$1
-  file=$2
-
-  echo "Loading template file $i" 
-  if ! retry 3 1 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"; then
-    if [[ $should_exit_on_failure -eq 1 ]]; then
-      fail "Could not load template file: $file"
-    else
-      load_failures=$((load_failures+1))
-      echo "Incremented load failure counter: $load_failures"
-    fi
-  fi
-}
-
-if [ ! -f $STATE_FILE_SUCCESS ]; then
-  echo "State file $STATE_FILE_SUCCESS not found. Running so-elasticsearch-templates-load."
-
-  . /usr/sbin/so-common
-
-  {% if GLOBALS.role != 'so-heavynode' %}
-  if [ -f /usr/sbin/so-elastic-fleet-common ]; then
-    . /usr/sbin/so-elastic-fleet-common
-  fi
-  {% endif %}
-
-  default_conf_dir=/opt/so/conf
-
-  # Define a default directory to load pipelines from
-  ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/"
-
-  {% if GLOBALS.role == 'so-heavynode' %}
-  file="/opt/so/conf/elasticsearch/templates/index/so-common-template.json"
-  {% else %}
-  file="/usr/sbin/so-elastic-fleet-common"
-  {% endif %}
-
-  if [ -f "$file" ]; then
-    # Wait for ElasticSearch to initialize
-    echo -n "Waiting for ElasticSearch..."
-    retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
-    {% if GLOBALS.role != 'so-heavynode' %}
-    TEMPLATE="logs-endpoint.alerts@package"
-    INSTALLED=$(so-elasticsearch-query _component_template/$TEMPLATE | jq -r .component_templates[0].name)
-    if [ "$INSTALLED" != "$TEMPLATE" ]; then
-      echo
-      echo "Packages not yet installed."
-      echo
-      exit 0
-    fi
-    {% endif %}
-
-    touch $STATE_FILE_INITIAL
-
-    cd ${ELASTICSEARCH_TEMPLATES}/component/ecs
-
-    echo "Loading ECS component templates..."
-    for i in *; do 
-      TEMPLATE=$(echo $i | cut -d '.' -f1) 
-      load_template "_component_template/${TEMPLATE}-mappings" "$i"
-    done
-    echo
-
-    cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent
-
-    echo "Loading Elastic Agent component templates..."
-    {% if GLOBALS.role == 'so-heavynode' %}
-    component_pattern="so-*"
-    {% else %}
-    component_pattern="*"
-    {% endif %}
-    for i in $component_pattern; do 
-      TEMPLATE=${i::-5} 
-      load_template "_component_template/$TEMPLATE" "$i"
-    done
-    echo
-	    
-    # Load SO-specific component templates
-    cd ${ELASTICSEARCH_TEMPLATES}/component/so
-
-    echo "Loading Security Onion component templates..."
-    for i in *; do 
-      TEMPLATE=$(echo $i | cut -d '.' -f1); 
-      load_template "_component_template/$TEMPLATE" "$i"
-    done
-    echo
-
-    # Load SO index templates
-    cd ${ELASTICSEARCH_TEMPLATES}/index
-
-    echo "Loading Security Onion index templates..."
-    shopt -s extglob
-    {% if GLOBALS.role == 'so-heavynode' %}
-    pattern="!(*1password*|*aws*|*azure*|*cloudflare*|*elastic_agent*|*fim*|*github*|*google*|*osquery*|*system*|*windows*|*endpoint*|*elasticsearch*|*generic*|*fleet_server*|*soc*)"
-    {% else %}
-    pattern="*"
-    {% endif %}
-    # Index templates will be skipped if the following conditions are met:
-    # 1. The template is part of the "so-logs-" template group
-    # 2. The template name does not correlate to at least one existing component template
-    # In this situation, the script will treat the skipped template as a temporary failure
-    # and allow the templates to be loaded again on the next run or highstate, whichever 
-    # comes first.
-    COMPONENT_LIST=$(so-elasticsearch-component-templates-list)
-    for i in $pattern; do
-      TEMPLATE=${i::-14}
-      COMPONENT_PATTERN=${TEMPLATE:3}
-      MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery")
-      if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ \.generic|logs-winlog\.winlog ]]; then
-        load_failures=$((load_failures+1))
-        echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures"
-      else
-        load_template "_index_template/$TEMPLATE" "$i"
-      fi
-    done
-  else
-    {% if GLOBALS.role == 'so-heavynode' %}
-    echo "Common template does not exist. Exiting..."
-    {% else %}
-    echo "Elastic Fleet not configured. Exiting..."
-    {% endif %}
-    exit 0
-  fi
-  
-  cd - >/dev/null
-
-  if [[ $load_failures -eq 0 ]]; then
-    echo "All templates loaded successfully"
-    touch $STATE_FILE_SUCCESS
-  else
-    echo "Encountered $load_failures templates that were unable to load, likely due to missing dependencies that will be available later; will retry on next highstate"
-  fi
-else
-  echo "Templates already loaded"
-fi

salt/firewall/containers.map.jinja

@@ -11,6 +11,7 @@
      'so-kratos',
      'so-hydra',
      'so-nginx',
+     'so-postgres',
      'so-redis',
      'so-soc',
      'so-strelka-coordinator',
@@ -34,6 +35,7 @@
      'so-hydra',
      'so-logstash',
      'so-nginx',
+     'so-postgres',
      'so-redis',
      'so-soc',
      'so-strelka-coordinator',
@@ -77,6 +79,7 @@
      'so-kratos',
      'so-hydra',
      'so-nginx',
+     'so-postgres',
      'so-soc'
 ] %}
 

salt/firewall/defaults.yaml

@@ -98,6 +98,10 @@ firewall:
       tcp:
         - 8086
       udp: []
+    postgres:
+      tcp:
+        - 5432
+      udp: []
     kafka_controller:
       tcp:
         - 9093
@@ -193,6 +197,7 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
+                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - localrules
@@ -379,6 +384,7 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
+                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - docker_registry
@@ -392,6 +398,7 @@ firewall:
                 - elasticsearch_rest
                 - docker_registry
                 - influxdb
+                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -404,6 +411,7 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
+                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -421,6 +429,7 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
+                - postgres
                 - sensoroni
             searchnode:
               portgroups:
@@ -431,6 +440,7 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
+                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -444,6 +454,7 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
+                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -453,6 +464,7 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
+                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -486,6 +498,7 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
+                - postgres
                 - sensoroni
                 - yum
                 - elastic_agent_control
@@ -496,6 +509,7 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
+                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -590,6 +604,7 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
+                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - docker_registry
@@ -603,6 +618,7 @@ firewall:
                 - elasticsearch_rest
                 - docker_registry
                 - influxdb
+                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -615,6 +631,7 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
+                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -632,6 +649,7 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
+                - postgres
                 - sensoroni
             searchnode:
               portgroups:
@@ -642,6 +660,7 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
+                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -655,6 +674,7 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
+                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -664,6 +684,7 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
+                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -695,6 +716,7 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
+                - postgres
                 - sensoroni
                 - yum
                 - elastic_agent_control
@@ -705,6 +727,7 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
+                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -799,6 +822,7 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
+                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - docker_registry
@@ -812,6 +836,7 @@ firewall:
                 - elasticsearch_rest
                 - docker_registry
                 - influxdb
+                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -824,6 +849,7 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
+                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -841,6 +867,7 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
+                - postgres
                 - sensoroni
             searchnode:
               portgroups:
@@ -850,6 +877,7 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
+                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -862,6 +890,7 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
+                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -871,6 +900,7 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
+                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -904,6 +934,7 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
+                - postgres
                 - sensoroni
                 - yum
                 - elastic_agent_control
@@ -914,6 +945,7 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
+                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -1011,6 +1043,7 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
+                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - docker_registry
@@ -1031,6 +1064,7 @@ firewall:
                 - elasticsearch_rest
                 - docker_registry
                 - influxdb
+                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -1043,6 +1077,7 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
+                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -1054,6 +1089,7 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
+                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -1065,6 +1101,7 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
+                - postgres
                 - sensoroni
                 - yum
                 - redis
@@ -1074,6 +1111,7 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
+                - postgres
                 - sensoroni
                 - yum
                 - redis
@@ -1084,6 +1122,7 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
+                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -1120,6 +1159,7 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
+                - postgres
                 - sensoroni
                 - yum
                 - elastic_agent_control
@@ -1130,6 +1170,7 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
+                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -1473,6 +1514,7 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
+                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - elastic_agent_control

salt/kibana/defaults.yaml

@@ -22,7 +22,7 @@ kibana:
           - default
           - file
     migrations:
-      discardCorruptObjects: "8.18.8"
+      discardCorruptObjects: "9.3.3"
     telemetry:
       enabled: False
     xpack:

salt/kibana/tools/sbin_jinja/so-kibana-space-defaults

@@ -9,5 +9,5 @@ SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http:
 # Disable certain Features from showing up in the Kibana UI
 echo
 echo "Setting up default Kibana Space:"
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","entityManager"]} ' >> /opt/so/log/kibana/misc.log
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV1","entityManager","streams","cloudConnect","slo"]} ' >> /opt/so/log/kibana/misc.log
 echo

salt/kratos/soc_kratos.yaml

@@ -3,8 +3,8 @@ kratos:
     description: Enables or disables the Kratos authentication system. WARNING - Disabling this process will cause the grid to malfunction. Re-enabling this setting will require manual effort via SSH.
     forcedType: bool
     advanced: True
+    readonly: True
     helpLink: kratos
-
   oidc:
     enabled:
       description: Set to True to enable OIDC / Single Sign-On (SSO) to SOC. Requires a valid Security Onion license key.
@@ -103,7 +103,7 @@ kratos:
   config:
     session:
       lifespan: 
-        description: Defines the length of a login session.
+        description: Defines the length of a login session before it will timeout, and require a new login.
         global: True
         helpLink: kratos
       whoami:

salt/logstash/defaults.yaml

@@ -26,12 +26,12 @@ logstash:
     manager:
       - so/0011_input_endgame.conf
       - so/0012_input_elastic_agent.conf.jinja
-      - so/0013_input_lumberjack_fleet.conf
+      - so/0013_input_lumberjack_fleet.conf.jinja
       - so/9999_output_redis.conf.jinja
     receiver:
       - so/0011_input_endgame.conf
       - so/0012_input_elastic_agent.conf.jinja
-      - so/0013_input_lumberjack_fleet.conf
+      - so/0013_input_lumberjack_fleet.conf.jinja
       - so/9999_output_redis.conf.jinja
     search:
       - so/0900_input_redis.conf.jinja
@@ -69,4 +69,5 @@ logstash:
     pipeline_x_batch_x_size: 125
     pipeline_x_ecs_compatibility: disabled
   dmz_nodes: []
+  latency_metrics: False
 

salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf.jinja

@@ -1,3 +1,4 @@
+{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
 input {
   elastic_agent {
     port => 5055
@@ -11,6 +12,11 @@ input {
   }
 }
 filter {
+  {% if LOGSTASH_MERGED.get('latency_metrics', False) %}
+  ruby {
+    code => "event.set('[_tmp][logstash_from_agent]', Time.now().utc.iso8601(3));"
+  }
+  {% endif %}
   if ![metadata] {
     mutate {
       rename => {"@metadata" => "metadata"}

salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf

@@ -1,23 +0,0 @@
-input {
-  elastic_agent {
-    port => 5056
-    tags => [ "elastic-agent", "fleet-lumberjack-input" ]
-    ssl_enabled => true
-    ssl_certificate => "/usr/share/logstash/elasticfleet-lumberjack.crt"
-    ssl_key => "/usr/share/logstash/elasticfleet-lumberjack.key"
-    ecs_compatibility => v8
-    id => "fleet-lumberjack-in"  
-    codec => "json"
-  }
-}
-
-
-filter {
-if ![metadata] {
-  mutate {
-    rename => {"@metadata" => "metadata"}
-  }
-}
-}
-
-

salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf.jinja

@@ -0,0 +1,26 @@
+{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
+input {
+  elastic_agent {
+    port => 5056
+    tags => [ "elastic-agent", "fleet-lumberjack-input" ]
+    ssl_enabled => true
+    ssl_certificate => "/usr/share/logstash/elasticfleet-lumberjack.crt"
+    ssl_key => "/usr/share/logstash/elasticfleet-lumberjack.key"
+    ecs_compatibility => v8
+    id => "fleet-lumberjack-in"
+    codec => "json"
+  }
+}
+
+filter {
+  {% if LOGSTASH_MERGED.get('latency_metrics', False) %}
+  ruby {
+    code => "event.set('[_tmp][logstash_from_fleet]', Time.now().utc.iso8601(3));"
+  }
+  {% endif %}
+  if ![metadata] {
+    mutate {
+      rename => {"@metadata" => "metadata"}
+    }
+  }
+}

salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja

@@ -1,3 +1,4 @@
+{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
 {%- set kafka_password = salt['pillar.get']('kafka:config:password') %}
 {%- set kafka_trustpass = salt['pillar.get']('kafka:config:trustpass') %}
 {%- set kafka_brokers = salt['pillar.get']('kafka:nodes', {}) %}
@@ -30,6 +31,11 @@ input {
     }
 }
 filter {
+  {% if LOGSTASH_MERGED.get('latency_metrics', False) %}
+  ruby {
+    code => "event.set('[_tmp][logstash_from_kafka]', Time.now().utc.iso8601(3));"
+  }
+  {% endif %}
   if ![metadata] {
     mutate {
       rename => { "@metadata" => "metadata" }

salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja

@@ -1,4 +1,4 @@
-{%- from 'logstash/map.jinja' import LOGSTASH_REDIS_NODES with context %}
+{%- from 'logstash/map.jinja' import LOGSTASH_REDIS_NODES, LOGSTASH_MERGED %}
 {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %}
 
 {%- for index in range(LOGSTASH_REDIS_NODES|length) %}
@@ -18,3 +18,10 @@ input {
 }
 {%   endfor %}
 {% endfor -%}
+filter {
+  {% if LOGSTASH_MERGED.get('latency_metrics', False) %}
+  ruby {
+    code => "event.set('[_tmp][logstash_from_redis]', Time.now().utc.iso8601(3));"
+  }
+  {% endif %}
+}

salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja

@@ -1,3 +1,11 @@
+{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
+{% if LOGSTASH_MERGED.get('latency_metrics', False) %}
+filter {
+  ruby {
+    code => "event.set('[_tmp][logstash_to_elasticsearch]', Time.now().utc.iso8601(3));"
+  }
+}
+{% endif %}
 output {
   if "elastic-agent" in [tags] and "so-ip-mappings" in [tags] {
     elasticsearch {

salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja

@@ -13,7 +13,14 @@ filter {
                     add_tag => "fleet-lumberjack-{{ GLOBALS.hostname }}"
           }
   }
-
+{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
+{% if LOGSTASH_MERGED.get('latency_metrics', False) %}
+filter {
+  ruby {
+    code => "event.set('[_tmp][fleet_to_logstash]', Time.now().utc.iso8601(3));"
+  }
+}
+{% endif %}
 output {
     lumberjack {
         codec => json

salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja

@@ -1,10 +1,17 @@
+{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
 {%- if grains.role in ['so-heavynode', 'so-receiver'] %}
   {%- set HOST = GLOBALS.hostname %}
 {%- else %}
   {%- set HOST = GLOBALS.manager %}
 {%- endif %}
 {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %}
-
+{% if LOGSTASH_MERGED.get('latency_metrics', False) %}
+filter {
+  ruby {
+    code => "event.set('[_tmp][logstash_to_redis]', Time.now().utc.iso8601(3));"
+  }
+}
+{% endif %}
 output {
 	redis {
 		host => '{{ HOST }}'

salt/logstash/soc_logstash.yaml

@@ -86,3 +86,8 @@ logstash:
     multiline: True
     advanced: True
     forcedType: "[]string"
+  latency_metrics:
+    description: Enable latency metrics within events processed by logstash. Useful for pinpointing log ingest delay.
+    forcedType: bool
+    global: False
+    advanced: True

salt/manager/managed_soc_annotations.sls

@@ -16,40 +16,35 @@
 {%       endif %}
 {%     endfor %}
 {%   endfor %}
-{%   set es_soc_annotations = '/opt/so/saltstack/default/salt/elasticsearch/soc_elasticsearch.yaml' %}
+{%   set soc_annotation_lines = [] %}
-{{   es_soc_annotations }}:
+{%   set defaults_lines = [] %}
-     file.serialize:
-       - dataset:
-           {% set data = salt['file.read'](es_soc_annotations) | load_yaml %}
-           {% set es = data.get('elasticsearch', {}) %}
-           {% set index_settings = es.get('index_settings', {}) %}
-           {% set input = index_settings.get('so-logs', {}) %}
 {%   for k in matched_integration_names %}
-           {%   do index_settings.update({k: input}) %}
+{%     do soc_annotation_lines.append('    ' ~ k ~ ': *dataStreamSettings') %}
+{%     do defaults_lines.append('    ' ~ k ~ ':') %}
+{%     set defaults_yaml = salt['slsutil.serialize']('yaml', ADDON_INTEGRATION_DEFAULTS[k], default_flow_style=False).strip() %}
+{%     for line in defaults_yaml.splitlines() %}
+{%       do defaults_lines.append('      ' ~ line) %}
 {%     endfor %}
-           {% for k in addon_integration_keys %}
-           {%   if k not in matched_integration_names and k in index_settings %}
-           {%     do index_settings.pop(k) %}
-           {%   endif %}
 {%   endfor %}
-           {{ data }}
+{%   set es_soc_annotations = '/opt/so/saltstack/default/salt/elasticsearch/soc_elasticsearch.yaml' %}
+manage_soc_annotations:
+  file.blockreplace:
+    - name: {{ es_soc_annotations }}
+    - marker_start: '    # START managed SOC integration annotations'
+    - marker_end: '    # END managed SOC integration annotations'
+    - content: {{ soc_annotation_lines | join('\n') | tojson }}
+    - insert_after_match: '^    # Managed SOC integration annotations are inserted below this line\.'
+    - append_if_not_found: False
+    - show_changes: True
 
 {#   Managed elasticsearch/defaults.yaml file for enabling 'Revert to default' via SOC UI for newly added config items #}
 {%   set es_defaults = '/opt/so/saltstack/default/salt/elasticsearch/defaults.yaml' %}
 {{   es_defaults }}:
-     file.serialize:
+  file.blockreplace:
-       - dataset:
+    - marker_start: '    # START managed SOC integration defaults'
-           {% set data = salt['file.read'](es_defaults) | load_yaml %}
+    - marker_end: '    # END managed SOC integration defaults'
-           {% set es = data.get('elasticsearch', {}) %}
+    - content: {{ defaults_lines | join('\n') | tojson }}
-           {% set index_settings = es.get('index_settings', {}) %}
+    - insert_after_match: '^  index_settings:$'
-           {% for k in matched_integration_names %}
+    - append_if_not_found: False
-           {%   set input = ADDON_INTEGRATION_DEFAULTS[k] %}
+    - show_changes: True
-           {%     do index_settings.update({k: input})%}
-           {% endfor %}
-           {% for k in addon_integration_keys %}
-           {%   if k not in matched_integration_names and k in index_settings %}
-           {%     do index_settings.pop(k) %}
-           {%   endif %}
-           {% endfor %}
-           {{ data }}
 {% endif %}

salt/manager/sync_es_users.sls

@@ -31,11 +31,13 @@ sync_es_users:
       - http: wait_for_kratos
       - file: so-user.lock # require so-user.lock file to be missing
 
-# we dont want this added too early in setup, so we add the onlyif to verify 'startup_states: highstate'
+# we dont want this added too early in setup, so the onlyif gates on the
-# is in the minion config. That line is added before the final highstate during setup
+# /opt/so/state/setup-complete marker. The marker is written by
+# mark_setup_complete in setup/so-functions just before the final setup
+# highstate (and by an upgrade-path state for systems set up under the old gate).
 so-user_sync:
   cron.present:
     - user: root
     - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log'
     - identifier: so-user_sync
-    - onlyif: "grep -x 'startup_states: highstate' /etc/salt/minion"
+    - onlyif: "test -e /opt/so/state/setup-complete"

salt/manager/tools/sbin/so-detections-overrides-import

@@ -0,0 +1,381 @@
+#!/usr/bin/env python3
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+# Imports detection overrides (e.g. from so-detections-backup) into the so-detection
+# index. Reads <publicId>.<ext> files (NDJSON, one override per line) from a source
+# directory, looks up the matching detection by publicId+engine, validates each
+# override against the same rules SOC enforces, dedupes against existing overrides
+# (operational fields only), and appends new ones.
+
+import argparse
+import ipaddress
+import json
+import os
+import re
+import sys
+from datetime import datetime
+
+import requests
+from requests.auth import HTTPBasicAuth
+import urllib3
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+DEFAULT_INDEX = "so-detection"
+AUTH_FILE = "/opt/so/conf/elasticsearch/curl.config"
+ES_URL = "https://localhost:9200"
+
+# Engines we know how to handle and the file extension the backup script writes.
+ENGINES = {
+    "suricata": "txt",
+}
+
+# Standard Suricata variables that ship with Security Onion. Anything else
+# referenced in an override is "custom" and the user needs to make sure it
+# exists in SOC Config before the override will function.
+BUILTIN_SURICATA_VARS = {
+    "$HOME_NET", "$EXTERNAL_NET",
+    "$HTTP_SERVERS", "$DNS_SERVERS", "$SQL_SERVERS", "$SMTP_SERVERS",
+    "$TELNET_SERVERS", "$AIM_SERVERS", "$DC_SERVERS", "$MODBUS_SERVER",
+    "$MODBUS_CLIENT", "$ENIP_CLIENT", "$ENIP_SERVER",
+    "$HTTP_PORTS", "$SHELLCODE_PORTS", "$ORACLE_PORTS", "$SSH_PORTS",
+    "$FTP_PORTS", "$FILE_DATA_PORTS",
+}
+
+VAR_PATTERN = re.compile(r"\$[A-Z_][A-Z0-9_]*")
+
+# Canonical valid values, per securityonion-soc/model/detection.go.
+SURICATA_OVERRIDE_TYPES = {"suppress", "threshold", "modify"}
+SUPPRESS_TRACKS = {"by_src", "by_dst", "by_either"}
+THRESHOLD_TRACKS = {"by_src", "by_dst", "by_both"}
+THRESHOLD_TYPES = {"limit", "threshold", "both"}
+
+STALE_WARNING = """\
+WARNING: so-detections-backup does not remove backup files when overrides are
+deleted via the Security Onion web UI. As a result, files in the source
+directory may represent overrides that were intentionally deleted and should
+NOT be re-imported.
+
+Before continuing, verify that the source directory reflects the overrides you
+actually want imported. Remove any files corresponding to overrides you previously deleted.
+"""
+
+
+def make_session(auth_file):
+    with open(auth_file, "r") as f:
+        for line in f:
+            if line.startswith("user ="):
+                creds = line.split("=", 1)[1].strip().replace('"', "")
+                user, _, password = creds.partition(":")
+                session = requests.Session()
+                session.auth = HTTPBasicAuth(user, password)
+                session.headers.update({"Content-Type": "application/json"})
+                session.verify = False
+                return session
+    raise RuntimeError(f"Could not find 'user =' line in {auth_file}")
+
+
+def find_detection(session, index, public_id, engine):
+    query = {
+        "query": {"bool": {"must": [
+            {"term": {"so_detection.publicId": public_id}},
+            {"term": {"so_detection.engine": engine}},
+        ]}},
+        "size": 2,
+    }
+    r = session.get(f"{ES_URL}/{index}/_search", json=query)
+    r.raise_for_status()
+    hits = r.json().get("hits", {}).get("hits", [])
+    if not hits:
+        return None, None, None
+    if len(hits) > 1:
+        # Shouldn't happen — publicId is unique per engine — but flag it.
+        print(f"  WARN: {len(hits)} detections matched publicId={public_id} engine={engine}; using first")
+    hit = hits[0]
+    existing = hit["_source"].get("so_detection", {}).get("overrides") or []
+    return hit["_id"], hit["_index"], existing
+
+
+def update_overrides(session, doc_index, doc_id, overrides):
+    body = {"doc": {"so_detection": {"overrides": overrides}}}
+    r = session.post(f"{ES_URL}/{doc_index}/_update/{doc_id}", json=body)
+    r.raise_for_status()
+    return r.json()
+
+
+def dedupe_key(override):
+    """Operational fields only, per Override.Equal() in detection.go.
+    Excludes timestamps and isEnabled so re-imports don't appear unique."""
+    t = override.get("type")
+    if t == "suppress":
+        return (t, override.get("track"), override.get("ip"))
+    if t == "threshold":
+        return (t, override.get("thresholdType"), override.get("track"),
+                override.get("count"), override.get("seconds"))
+    if t == "modify":
+        return (t, override.get("regex"), override.get("value"))
+
+
+def _validate_suricata_ip(ip):
+    if not ip:
+        return "ip cannot be empty"
+    if ip.startswith("$"):
+        return None
+    if ip.startswith("[") and ip.endswith("]"):
+        for part in ip[1:-1].split(","):
+            err = _validate_single_ip(part.strip())
+            if err:
+                return f"invalid IP in list: {err}"
+        return None
+    return _validate_single_ip(ip)
+
+
+def _validate_single_ip(ip):
+    try:
+        if "/" in ip:
+            ipaddress.ip_network(ip, strict=False)
+        else:
+            ipaddress.ip_address(ip)
+    except ValueError:
+        return f"invalid IP/CIDR {ip!r}"
+    return None
+
+
+def validate_override(override, engine):
+    """Mirror Override.Validate() from securityonion-soc/model/detection.go.
+    Returns None on success, an error string otherwise."""
+    t = override.get("type")
+    if not t:
+        return "override type is required"
+    if t not in SURICATA_OVERRIDE_TYPES:
+        return f"invalid type {t!r}: must be one of {sorted(SURICATA_OVERRIDE_TYPES)}"
+
+    has = {k: override.get(k) is not None for k in
+           ("regex", "value", "thresholdType", "track", "ip", "count", "seconds", "customFilter")}
+
+    if t == "suppress":
+        if not has["ip"] or not has["track"]:
+            return "suppress requires 'ip' and 'track'"
+        if any(has[k] for k in ("regex", "value", "thresholdType", "count", "seconds", "customFilter")):
+            return "suppress has unnecessary fields"
+        if override["track"] not in SUPPRESS_TRACKS:
+            return f"invalid track {override['track']!r}: must be one of {sorted(SUPPRESS_TRACKS)}"
+        return _validate_suricata_ip(override["ip"])
+
+    if t == "threshold":
+        if not all(has[k] for k in ("thresholdType", "track", "count", "seconds")):
+            return "threshold requires 'thresholdType', 'track', 'count', 'seconds'"
+        if any(has[k] for k in ("regex", "value", "customFilter")):
+            return "threshold has unnecessary fields"
+        if override["thresholdType"] not in THRESHOLD_TYPES:
+            return f"invalid thresholdType {override['thresholdType']!r}: must be one of {sorted(THRESHOLD_TYPES)}"
+        if override["track"] not in THRESHOLD_TRACKS:
+            return f"invalid track {override['track']!r}: must be one of {sorted(THRESHOLD_TRACKS)}"
+        if not isinstance(override["count"], int) or override["count"] <= 0:
+            return f"count must be a positive integer, got {override['count']!r}"
+        if not isinstance(override["seconds"], int) or override["seconds"] <= 0:
+            return f"seconds must be a positive integer, got {override['seconds']!r}"
+        return None
+
+    if t == "modify":
+        if not has["regex"] or not has["value"]:
+            return "modify requires 'regex' and 'value'"
+        if any(has[k] for k in ("thresholdType", "track", "count", "seconds", "customFilter")):
+            return "modify has unnecessary fields"
+        try:
+            re.compile(override["regex"])
+        except re.error as e:
+            return f"invalid regex: {e}"
+        return None
+
+
+def parse_overrides_file(path):
+    """Parse a file written by so-detections-backup.py: NDJSON, one override
+    per line. Returns a list of (override_dict, line_number)."""
+    overrides = []
+    with open(path, "r") as f:
+        for i, line in enumerate(f, start=1):
+            line = line.strip()
+            if not line:
+                continue
+            overrides.append((json.loads(line), i))
+    return overrides
+
+
+def describe(override):
+    """Human-readable summary of the operational fields for a given override type."""
+    t = override.get("type")
+    if t == "suppress":
+        return f"type=suppress track={override.get('track')} ip={override.get('ip')}"
+    if t == "threshold":
+        return (f"type=threshold track={override.get('track')} "
+                f"thresholdType={override.get('thresholdType')} "
+                f"count={override.get('count')} seconds={override.get('seconds')}")
+    if t == "modify":
+        return f"type=modify regex={override.get('regex')!r}"
+
+
+def collect_custom_vars(override):
+    found = set()
+    for value in override.values():
+        if isinstance(value, str):
+            for match in VAR_PATTERN.findall(value):
+                if match not in BUILTIN_SURICATA_VARS:
+                    found.add(match)
+    return found
+
+
+def parse_args():
+    p = argparse.ArgumentParser(
+        description="Import detection overrides into the so-detection index.",
+    )
+    p.add_argument("--source", "-s", required=True,
+                   help="Source directory containing <publicId>.<ext> override files.")
+    p.add_argument("--engine", "-e", default="suricata", choices=list(ENGINES.keys()),
+                   help="Detection engine (default: suricata).")
+    p.add_argument("--dry-run", "-n", action="store_true",
+                   help="Print what would happen without writing to Elasticsearch.")
+    p.add_argument("--no-import-note", action="store_true",
+                   help="Do not prepend '[Imported YYYY-MM-DD] ' to the override note.")
+    p.add_argument("--index", "-i", default=DEFAULT_INDEX,
+                   help=f"Elasticsearch index to update (default: {DEFAULT_INDEX}).")
+    return p.parse_args()
+
+
+def confirm_proceed(args):
+    """Show the stale-backup warning. Dry-run prints it and continues. Real
+    runs require the user typing 'yes' at the prompt."""
+    print(STALE_WARNING)
+    if args.dry_run:
+        print("(dry-run: no acknowledgement required)\n")
+        return True
+    answer = input("Type 'yes' to acknowledge and continue: ").strip().lower()
+    print()
+    return answer == "yes"
+
+
+def main():
+    args = parse_args()
+
+    if not os.path.isdir(args.source):
+        print(f"ERROR: source directory not found: {args.source}", file=sys.stderr)
+        sys.exit(1)
+
+    extension = ENGINES[args.engine]
+    files = sorted(f for f in os.listdir(args.source) if f.endswith(f".{extension}"))
+    if not files:
+        print(f"No *.{extension} files found in {args.source}")
+        sys.exit(0)
+
+    if not confirm_proceed(args):
+        print("Aborted.")
+        sys.exit(1)
+
+    session = make_session(AUTH_FILE)
+    today = datetime.now().strftime("%Y-%m-%d")
+    note_prefix = "" if args.no_import_note else f"[Imported {today}] "
+
+    counts = {"added": 0, "skipped_dedupe": 0, "skipped_not_found": 0, "invalid": 0, "error": 0}
+    custom_vars = set()
+
+    mode = "DRY-RUN" if args.dry_run else "IMPORT"
+    print(f"[{mode}] engine={args.engine} source={args.source} index={args.index}\n")
+
+    for filename in files:
+        public_id = os.path.splitext(filename)[0]
+        path = os.path.join(args.source, filename)
+        print(f"{public_id}:")
+
+        try:
+            new_overrides = parse_overrides_file(path)
+        except (json.JSONDecodeError, OSError) as e:
+            print(f"  ERROR: could not parse {filename}: {e}")
+            counts["error"] += 1
+            continue
+
+        if not new_overrides:
+            print("  SKIP: empty file")
+            continue
+
+        try:
+            doc_id, doc_index, existing = find_detection(session, args.index, public_id, args.engine)
+        except requests.HTTPError as e:
+            print(f"  ERROR: search failed: {e}")
+            counts["error"] += 1
+            continue
+
+        if doc_id is None:
+            print(f"  WARN: no detection found for publicId={public_id} engine={args.engine}; skipping")
+            counts["skipped_not_found"] += len(new_overrides)
+            continue
+
+        existing_keys = {dedupe_key(o) for o in existing}
+        merged = list(existing)
+        added_this_file = 0
+
+        for override, line_no in new_overrides:
+            err = validate_override(override, args.engine)
+            if err:
+                print(f"  INVALID (line {line_no}): {err}")
+                counts["invalid"] += 1
+                continue
+
+            custom_vars.update(collect_custom_vars(override))
+            key = dedupe_key(override)
+            if key in existing_keys:
+                print(f"  SKIP (line {line_no}): duplicate of existing override [{describe(override)}]")
+                counts["skipped_dedupe"] += 1
+                continue
+
+            if note_prefix:
+                override = dict(override)
+                override["note"] = note_prefix + (override.get("note") or "")
+
+            merged.append(override)
+            existing_keys.add(key)
+            added_this_file += 1
+            print(f"  ADD (line {line_no}): {describe(override)}")
+
+        if added_this_file == 0:
+            continue
+
+        if args.dry_run:
+            print(f"  DRY-RUN: would update {doc_index}/{doc_id} "
+                  f"({len(existing)} existing → {len(merged)} total)")
+            counts["added"] += added_this_file
+            continue
+
+        try:
+            update_overrides(session, doc_index, doc_id, merged)
+            print(f"  UPDATED {doc_index}/{doc_id} ({len(existing)} → {len(merged)})")
+            counts["added"] += added_this_file
+        except requests.HTTPError as e:
+            print(f"  ERROR: update failed: {e}")
+            counts["error"] += 1
+
+    print()
+    print("=" * 60)
+    print(f"Summary ({mode}):")
+    print(f"  Overrides added:           {counts['added']}")
+    print(f"  Skipped (already present): {counts['skipped_dedupe']}")
+    print(f"  Skipped (no detection):    {counts['skipped_not_found']}")
+    print(f"  Invalid (failed checks):   {counts['invalid']}")
+    print(f"  Errors:                    {counts['error']}")
+
+    if custom_vars:
+        print()
+        print("WARNING: detected custom Suricata variables in imported overrides:")
+        for v in sorted(custom_vars):
+            print(f"  {v}")
+        print("If any of these are not already defined in SOC Config (Suricata variables),")
+        print("you must add them manually before the rules will function correctly.")
+
+    sys.exit(0 if counts["error"] == 0 and counts["invalid"] == 0 else 1)
+
+
+if __name__ == "__main__":
+    main()

salt/manager/tools/sbin/so-detections-overrides-import_test.py

@@ -0,0 +1,588 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+import importlib.util
+import json
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+from importlib.machinery import SourceFileLoader
+from io import StringIO
+from unittest.mock import MagicMock, patch
+
+import requests
+
+# The script has no .py extension; spec_from_file_location can't auto-detect a
+# loader, so we hand it a SourceFileLoader explicitly. (load_module() is
+# deprecated in 3.14 and slated for removal in 3.15.)
+HERE = os.path.dirname(os.path.abspath(__file__))
+SCRIPT = os.path.join(HERE, "so-detections-overrides-import")
+_loader = SourceFileLoader("so_overrides_import", SCRIPT)
+_spec = importlib.util.spec_from_loader("so_overrides_import", _loader)
+soi = importlib.util.module_from_spec(_spec)
+_loader.exec_module(soi)
+
+
+class TestValidateSuppress(unittest.TestCase):
+    def test_valid(self):
+        self.assertIsNone(soi.validate_override(
+            {"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}, "suricata"))
+
+    def test_valid_var(self):
+        self.assertIsNone(soi.validate_override(
+            {"type": "suppress", "track": "by_either", "ip": "$HOME_NET"}, "suricata"))
+
+    def test_valid_cidr(self):
+        self.assertIsNone(soi.validate_override(
+            {"type": "suppress", "track": "by_dst", "ip": "10.0.0.0/8"}, "suricata"))
+
+    def test_valid_bracket_list(self):
+        self.assertIsNone(soi.validate_override(
+            {"type": "suppress", "track": "by_src", "ip": "[1.2.3.4,10.0.0.0/8]"}, "suricata"))
+
+    def test_missing_ip(self):
+        err = soi.validate_override({"type": "suppress", "track": "by_src"}, "suricata")
+        self.assertIn("requires", err)
+
+    def test_missing_track(self):
+        err = soi.validate_override({"type": "suppress", "ip": "1.2.3.4"}, "suricata")
+        self.assertIn("requires", err)
+
+    def test_invalid_track(self):
+        err = soi.validate_override(
+            {"type": "suppress", "track": "by_both", "ip": "1.2.3.4"}, "suricata")
+        self.assertIn("invalid track", err)
+
+    def test_invalid_ip(self):
+        err = soi.validate_override(
+            {"type": "suppress", "track": "by_src", "ip": "not-an-ip"}, "suricata")
+        self.assertIn("invalid IP", err)
+
+    def test_unnecessary_field(self):
+        err = soi.validate_override(
+            {"type": "suppress", "track": "by_src", "ip": "1.2.3.4", "count": 5}, "suricata")
+        self.assertIn("unnecessary fields", err)
+
+
+class TestValidateThreshold(unittest.TestCase):
+    def test_valid(self):
+        self.assertIsNone(soi.validate_override({
+            "type": "threshold", "track": "by_src",
+            "thresholdType": "limit", "count": 10, "seconds": 60,
+        }, "suricata"))
+
+    def test_valid_by_both(self):
+        self.assertIsNone(soi.validate_override({
+            "type": "threshold", "track": "by_both",
+            "thresholdType": "both", "count": 1, "seconds": 1,
+        }, "suricata"))
+
+    def test_track_by_either_invalid(self):
+        err = soi.validate_override({
+            "type": "threshold", "track": "by_either",
+            "thresholdType": "limit", "count": 10, "seconds": 60,
+        }, "suricata")
+        self.assertIn("invalid track", err)
+
+    def test_invalid_threshold_type(self):
+        err = soi.validate_override({
+            "type": "threshold", "track": "by_src",
+            "thresholdType": "bogus", "count": 10, "seconds": 60,
+        }, "suricata")
+        self.assertIn("invalid thresholdType", err)
+
+    def test_zero_count(self):
+        err = soi.validate_override({
+            "type": "threshold", "track": "by_src",
+            "thresholdType": "limit", "count": 0, "seconds": 60,
+        }, "suricata")
+        self.assertIn("count", err)
+
+    def test_negative_seconds(self):
+        err = soi.validate_override({
+            "type": "threshold", "track": "by_src",
+            "thresholdType": "limit", "count": 10, "seconds": -1,
+        }, "suricata")
+        self.assertIn("seconds", err)
+
+    def test_missing_field(self):
+        err = soi.validate_override({
+            "type": "threshold", "track": "by_src",
+            "thresholdType": "limit", "count": 10,  # missing seconds
+        }, "suricata")
+        self.assertIn("requires", err)
+
+    def test_unnecessary_field(self):
+        err = soi.validate_override({
+            "type": "threshold", "track": "by_src",
+            "thresholdType": "limit", "count": 10, "seconds": 60,
+            "regex": "foo",
+        }, "suricata")
+        self.assertIn("unnecessary fields", err)
+
+
+class TestValidateModify(unittest.TestCase):
+    def test_valid(self):
+        self.assertIsNone(soi.validate_override(
+            {"type": "modify", "regex": r"content:\"foo\"", "value": "content:bar"}, "suricata"))
+
+    def test_invalid_regex(self):
+        err = soi.validate_override(
+            {"type": "modify", "regex": "(unbalanced", "value": "x"}, "suricata")
+        self.assertIn("invalid regex", err)
+
+    def test_missing_value(self):
+        err = soi.validate_override({"type": "modify", "regex": "x"}, "suricata")
+        self.assertIn("requires", err)
+
+    def test_unnecessary_field(self):
+        err = soi.validate_override(
+            {"type": "modify", "regex": "x", "value": "y", "track": "by_src"}, "suricata")
+        self.assertIn("unnecessary fields", err)
+
+
+class TestValidateMisc(unittest.TestCase):
+    def test_unknown_type(self):
+        err = soi.validate_override({"type": "suppresss", "track": "by_src", "ip": "1.2.3.4"}, "suricata")
+        self.assertIn("invalid type", err)
+
+    def test_missing_type(self):
+        err = soi.validate_override({"track": "by_src"}, "suricata")
+        self.assertIn("type is required", err)
+
+
+class TestValidateIP(unittest.TestCase):
+    def test_plain_ipv4(self):
+        self.assertIsNone(soi._validate_suricata_ip("1.2.3.4"))
+
+    def test_plain_ipv6(self):
+        self.assertIsNone(soi._validate_suricata_ip("::1"))
+
+    def test_cidr(self):
+        self.assertIsNone(soi._validate_suricata_ip("10.0.0.0/8"))
+
+    def test_var(self):
+        self.assertIsNone(soi._validate_suricata_ip("$CONCOURSEWORKERS"))
+
+    def test_bracket_list(self):
+        self.assertIsNone(soi._validate_suricata_ip("[1.2.3.4, 10.0.0.0/8]"))
+
+    def test_bracket_list_bad_member(self):
+        err = soi._validate_suricata_ip("[1.2.3.4,nope]")
+        self.assertIn("invalid IP in list", err)
+
+    def test_empty(self):
+        self.assertIn("empty", soi._validate_suricata_ip(""))
+
+    def test_invalid(self):
+        self.assertIn("invalid", soi._validate_suricata_ip("999.999.999.999"))
+
+
+class TestDedupeKey(unittest.TestCase):
+    def test_suppress(self):
+        a = {"type": "suppress", "track": "by_src", "ip": "1.2.3.4", "count": 99}
+        b = {"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}
+        # count is irrelevant for suppress dedupe
+        self.assertEqual(soi.dedupe_key(a), soi.dedupe_key(b))
+
+    def test_suppress_differs_on_ip(self):
+        a = {"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}
+        b = {"type": "suppress", "track": "by_src", "ip": "5.6.7.8"}
+        self.assertNotEqual(soi.dedupe_key(a), soi.dedupe_key(b))
+
+    def test_threshold(self):
+        a = {"type": "threshold", "track": "by_src", "thresholdType": "limit",
+             "count": 10, "seconds": 60, "ip": "ignored"}
+        b = {"type": "threshold", "track": "by_src", "thresholdType": "limit",
+             "count": 10, "seconds": 60}
+        self.assertEqual(soi.dedupe_key(a), soi.dedupe_key(b))
+
+    def test_threshold_differs_on_count(self):
+        a = {"type": "threshold", "track": "by_src", "thresholdType": "limit",
+             "count": 10, "seconds": 60}
+        b = {"type": "threshold", "track": "by_src", "thresholdType": "limit",
+             "count": 20, "seconds": 60}
+        self.assertNotEqual(soi.dedupe_key(a), soi.dedupe_key(b))
+
+    def test_modify(self):
+        a = {"type": "modify", "regex": "x", "value": "y"}
+        b = {"type": "modify", "regex": "x", "value": "y"}
+        self.assertEqual(soi.dedupe_key(a), soi.dedupe_key(b))
+
+
+class TestDescribe(unittest.TestCase):
+    def test_suppress(self):
+        s = soi.describe({"type": "suppress", "track": "by_src", "ip": "1.2.3.4"})
+        self.assertIn("suppress", s)
+        self.assertIn("by_src", s)
+        self.assertIn("1.2.3.4", s)
+
+    def test_threshold_includes_count(self):
+        s = soi.describe({"type": "threshold", "track": "by_src",
+                          "thresholdType": "limit", "count": 10, "seconds": 60})
+        self.assertIn("count=10", s)
+        self.assertIn("seconds=60", s)
+
+    def test_modify(self):
+        s = soi.describe({"type": "modify", "regex": "foo"})
+        self.assertIn("modify", s)
+        self.assertIn("foo", s)
+
+
+class TestParseOverridesFile(unittest.TestCase):
+    def _write(self, content):
+        fd, path = tempfile.mkstemp(suffix=".txt")
+        os.close(fd)
+        with open(path, "w") as f:
+            f.write(content)
+        self.addCleanup(os.unlink, path)
+        return path
+
+    def test_single_line(self):
+        path = self._write('{"type":"suppress","track":"by_src","ip":"1.2.3.4"}')
+        result = soi.parse_overrides_file(path)
+        self.assertEqual(len(result), 1)
+        self.assertEqual(result[0][0]["type"], "suppress")
+        self.assertEqual(result[0][1], 1)
+
+    def test_ndjson(self):
+        path = self._write(
+            '{"type":"suppress","track":"by_src","ip":"1.2.3.4"}\n'
+            '{"type":"suppress","track":"by_dst","ip":"5.6.7.8"}\n'
+        )
+        result = soi.parse_overrides_file(path)
+        self.assertEqual(len(result), 2)
+        self.assertEqual(result[1][1], 2)
+
+    def test_empty(self):
+        path = self._write("")
+        self.assertEqual(soi.parse_overrides_file(path), [])
+
+    def test_blank_lines_skipped(self):
+        path = self._write('\n{"type":"suppress","track":"by_src","ip":"1.2.3.4"}\n\n')
+        result = soi.parse_overrides_file(path)
+        self.assertEqual(len(result), 1)
+        self.assertEqual(result[0][1], 2)  # line number reflects original position
+
+    def test_invalid_raises(self):
+        path = self._write("not json")
+        with self.assertRaises(json.JSONDecodeError):
+            soi.parse_overrides_file(path)
+
+
+class TestCollectCustomVars(unittest.TestCase):
+    def test_finds_custom(self):
+        v = soi.collect_custom_vars({"ip": "$CONCOURSEWORKERS"})
+        self.assertEqual(v, {"$CONCOURSEWORKERS"})
+
+    def test_filters_builtins(self):
+        v = soi.collect_custom_vars({"ip": "$HOME_NET"})
+        self.assertEqual(v, set())
+
+    def test_mixed(self):
+        v = soi.collect_custom_vars({"ip": "[$HOME_NET,$MYNET]"})
+        self.assertEqual(v, {"$MYNET"})
+
+    def test_non_string_fields_ignored(self):
+        v = soi.collect_custom_vars({"count": 10, "isEnabled": True})
+        self.assertEqual(v, set())
+
+
+class TestMakeSession(unittest.TestCase):
+    def _write(self, content):
+        fd, path = tempfile.mkstemp()
+        os.close(fd)
+        with open(path, "w") as f:
+            f.write(content)
+        self.addCleanup(os.unlink, path)
+        return path
+
+    def test_valid_auth_file(self):
+        path = self._write('user = "admin:secret"\n')
+        session = soi.make_session(path)
+        self.assertEqual(session.auth.username, "admin")
+        self.assertEqual(session.auth.password, "secret")
+        self.assertFalse(session.verify)
+
+    def test_missing_user_line(self):
+        path = self._write("# no user line here\n")
+        with self.assertRaises(RuntimeError):
+            soi.make_session(path)
+
+
+class TestFindDetection(unittest.TestCase):
+    def _session_with_response(self, payload):
+        session = MagicMock()
+        response = MagicMock()
+        response.json.return_value = payload
+        response.raise_for_status.return_value = None
+        session.get.return_value = response
+        return session
+
+    def test_found(self):
+        session = self._session_with_response({"hits": {"hits": [{
+            "_id": "abc", "_index": "so-detection",
+            "_source": {"so_detection": {"overrides": [{"type": "suppress"}]}},
+        }]}})
+        doc_id, idx, existing = soi.find_detection(session, "so-detection", "2049201", "suricata")
+        self.assertEqual(doc_id, "abc")
+        self.assertEqual(idx, "so-detection")
+        self.assertEqual(len(existing), 1)
+
+    def test_not_found(self):
+        session = self._session_with_response({"hits": {"hits": []}})
+        doc_id, idx, existing = soi.find_detection(session, "so-detection", "x", "suricata")
+        self.assertIsNone(doc_id)
+        self.assertIsNone(idx)
+        self.assertIsNone(existing)
+
+    def test_no_overrides_field(self):
+        session = self._session_with_response({"hits": {"hits": [{
+            "_id": "abc", "_index": "so-detection",
+            "_source": {"so_detection": {}},
+        }]}})
+        _, _, existing = soi.find_detection(session, "so-detection", "x", "suricata")
+        self.assertEqual(existing, [])
+
+    def test_multiple_hits_warns(self):
+        session = self._session_with_response({"hits": {"hits": [
+            {"_id": "a", "_index": "i", "_source": {"so_detection": {"overrides": []}}},
+            {"_id": "b", "_index": "i", "_source": {"so_detection": {"overrides": []}}},
+        ]}})
+        with patch("sys.stdout", new=StringIO()) as out:
+            doc_id, _, _ = soi.find_detection(session, "i", "x", "suricata")
+        self.assertEqual(doc_id, "a")
+        self.assertIn("WARN", out.getvalue())
+
+
+class TestUpdateOverrides(unittest.TestCase):
+    def test_posts_to_update_endpoint(self):
+        session = MagicMock()
+        response = MagicMock()
+        response.raise_for_status.return_value = None
+        response.json.return_value = {"result": "updated"}
+        session.post.return_value = response
+
+        result = soi.update_overrides(session, "so-detection", "abc", [{"type": "suppress"}])
+
+        self.assertEqual(result, {"result": "updated"})
+        url = session.post.call_args[0][0]
+        self.assertIn("/_update/abc", url)
+        body = session.post.call_args[1]["json"]
+        self.assertEqual(body["doc"]["so_detection"]["overrides"], [{"type": "suppress"}])
+
+
+class TestConfirmProceed(unittest.TestCase):
+    def test_dry_run_skips_prompt(self):
+        args = MagicMock(dry_run=True)
+        with patch("sys.stdout", new=StringIO()):
+            self.assertTrue(soi.confirm_proceed(args))
+
+    def test_yes_input(self):
+        args = MagicMock(dry_run=False)
+        with patch("sys.stdout", new=StringIO()):
+            with patch("builtins.input", return_value="yes"):
+                self.assertTrue(soi.confirm_proceed(args))
+
+    def test_yes_input_case_insensitive(self):
+        args = MagicMock(dry_run=False)
+        with patch("sys.stdout", new=StringIO()):
+            with patch("builtins.input", return_value="YES"):
+                self.assertTrue(soi.confirm_proceed(args))
+
+    def test_no_input_aborts(self):
+        args = MagicMock(dry_run=False)
+        with patch("sys.stdout", new=StringIO()):
+            with patch("builtins.input", return_value="no"):
+                self.assertFalse(soi.confirm_proceed(args))
+
+    def test_empty_input_aborts(self):
+        args = MagicMock(dry_run=False)
+        with patch("sys.stdout", new=StringIO()):
+            with patch("builtins.input", return_value=""):
+                self.assertFalse(soi.confirm_proceed(args))
+
+
+class TestParseArgs(unittest.TestCase):
+    def test_defaults(self):
+        with patch.object(sys, "argv", ["cmd", "--source", "/some/path"]):
+            args = soi.parse_args()
+        self.assertEqual(args.source, "/some/path")
+        self.assertEqual(args.engine, "suricata")
+        self.assertFalse(args.dry_run)
+        self.assertFalse(args.no_import_note)
+        self.assertEqual(args.index, soi.DEFAULT_INDEX)
+
+    def test_all_options(self):
+        argv = ["cmd", "-s", "/x", "-e", "suricata", "-n",
+                "--no-import-note", "-i", "alt-index"]
+        with patch.object(sys, "argv", argv):
+            args = soi.parse_args()
+        self.assertEqual(args.source, "/x")
+        self.assertTrue(args.dry_run)
+        self.assertTrue(args.no_import_note)
+        self.assertEqual(args.index, "alt-index")
+
+
+class TestMain(unittest.TestCase):
+    def setUp(self):
+        self.tmpdir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.tmpdir, ignore_errors=True)
+        # Stub make_session so tests don't need /opt/so/conf/elasticsearch/curl.config.
+        p = patch.object(soi, "make_session", return_value=MagicMock())
+        p.start()
+        self.addCleanup(p.stop)
+
+    def _write_file(self, public_id, overrides, ext="txt"):
+        """Write an NDJSON override file. Entries may be dicts or raw strings (for malformed input)."""
+        path = os.path.join(self.tmpdir, f"{public_id}.{ext}")
+        with open(path, "w") as f:
+            for o in overrides:
+                f.write(o if isinstance(o, str) else json.dumps(o))
+                f.write("\n")
+        return path
+
+    def _run_main(self, *extra_argv, input_response="yes"):
+        """Run main() with stdout/stderr captured and input mocked. Returns (stdout, stderr, exit_code)."""
+        argv = ["cmd", "--source", self.tmpdir, *extra_argv]
+        out, err = StringIO(), StringIO()
+        with patch.object(sys, "argv", argv), \
+                patch("sys.stdout", new=out), \
+                patch("sys.stderr", new=err), \
+                patch("builtins.input", return_value=input_response):
+            with self.assertRaises(SystemExit) as cm:
+                soi.main()
+        return out.getvalue(), err.getvalue(), cm.exception.code
+
+    def test_source_dir_missing(self):
+        argv = ["cmd", "--source", "/no/such/path/here"]
+        err = StringIO()
+        with patch.object(sys, "argv", argv), patch("sys.stderr", new=err):
+            with self.assertRaises(SystemExit) as cm:
+                soi.main()
+        self.assertEqual(cm.exception.code, 1)
+        self.assertIn("source directory not found", err.getvalue())
+
+    def test_no_files_found(self):
+        out, _, code = self._run_main()
+        self.assertEqual(code, 0)
+        self.assertIn("No *.txt files found", out)
+
+    def test_user_aborts(self):
+        self._write_file("1001", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}])
+        out, _, code = self._run_main(input_response="no")
+        self.assertEqual(code, 1)
+        self.assertIn("Aborted", out)
+
+    def test_parse_error_increments_error(self):
+        # Malformed JSON line — parse_overrides_file raises JSONDecodeError.
+        self._write_file("1002", ["not json"])
+        out, _, code = self._run_main("--dry-run")
+        self.assertEqual(code, 1)  # invalid+error → non-zero
+        self.assertIn("could not parse", out)
+        self.assertIn("Errors:                    1", out)
+
+    def test_empty_file_skipped(self):
+        # Blank lines only — parse_overrides_file returns []; main reports "empty file" and continues.
+        path = os.path.join(self.tmpdir, "1003.txt")
+        with open(path, "w") as f:
+            f.write("\n\n")
+        out, _, code = self._run_main("--dry-run")
+        self.assertEqual(code, 0)
+        self.assertIn("empty file", out)
+
+    @patch.object(soi, "find_detection")
+    def test_search_http_error(self, mock_find):
+        mock_find.side_effect = requests.HTTPError("boom")
+        self._write_file("1004", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}])
+        out, _, code = self._run_main("--dry-run")
+        self.assertEqual(code, 1)
+        self.assertIn("search failed", out)
+
+    @patch.object(soi, "find_detection")
+    def test_no_detection_found(self, mock_find):
+        mock_find.return_value = (None, None, None)
+        self._write_file("1005", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}])
+        out, _, code = self._run_main("--dry-run")
+        self.assertEqual(code, 0)
+        self.assertIn("no detection found", out)
+        self.assertIn("Skipped (no detection):    1", out)
+
+    @patch.object(soi, "find_detection")
+    def test_all_duplicates_no_update(self, mock_find):
+        existing = [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}]
+        mock_find.return_value = ("doc1", "so-detection", existing)
+        self._write_file("1006", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}])
+        out, _, code = self._run_main("--dry-run")
+        self.assertEqual(code, 0)
+        self.assertIn("SKIP", out)
+        self.assertNotIn("DRY-RUN: would update", out)  # added_this_file == 0 branch
+
+    @patch.object(soi, "update_overrides")
+    @patch.object(soi, "find_detection")
+    def test_happy_path_full(self, mock_find, mock_update):
+        # Exercises: ADD, dedupe SKIP, INVALID, note prefix, UPDATE, custom-vars warning, exit=1 (invalid present)
+        existing = [{"type": "suppress", "track": "by_src", "ip": "9.9.9.9"}]
+        mock_find.return_value = ("doc1", "so-detection", existing)
+        mock_update.return_value = {"result": "updated"}
+        self._write_file("1007", [
+            {"type": "suppress", "track": "by_src", "ip": "1.2.3.4"},                # ADD
+            {"type": "suppress", "track": "by_src", "ip": "9.9.9.9"},                # SKIP (dupe of existing)
+            {"type": "suppress", "track": "bogus",  "ip": "1.2.3.4"},                # INVALID
+            {"type": "suppress", "track": "by_src", "ip": "$CONCOURSEWORKERS"},      # ADD + custom var
+        ])
+        out, _, code = self._run_main()
+        self.assertEqual(code, 1)  # one invalid -> non-zero
+
+        mock_update.assert_called_once()
+        merged = mock_update.call_args[0][3]
+        self.assertEqual(len(merged), 3)  # 1 existing + 2 new
+        new_notes = [o.get("note", "") for o in merged if o.get("ip") in ("1.2.3.4", "$CONCOURSEWORKERS")]
+        self.assertTrue(all(n.startswith("[Imported ") for n in new_notes))
+
+        self.assertIn("ADD", out)
+        self.assertIn("SKIP", out)
+        self.assertIn("INVALID", out)
+        self.assertIn("UPDATED", out)
+        self.assertIn("$CONCOURSEWORKERS", out)
+
+    @patch.object(soi, "update_overrides")
+    @patch.object(soi, "find_detection")
+    def test_no_import_note_preserves_note(self, mock_find, mock_update):
+        mock_find.return_value = ("doc1", "so-detection", [])
+        mock_update.return_value = {"result": "updated"}
+        self._write_file("1008", [
+            {"type": "suppress", "track": "by_src", "ip": "1.2.3.4", "note": "original"},
+        ])
+        _, _, code = self._run_main("--no-import-note")
+        self.assertEqual(code, 0)
+        merged = mock_update.call_args[0][3]
+        self.assertEqual(merged[0]["note"], "original")  # no prefix applied
+
+    @patch.object(soi, "find_detection")
+    def test_dry_run_skips_update(self, mock_find):
+        mock_find.return_value = ("doc1", "so-detection", [])
+        self._write_file("1009", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}])
+        with patch.object(soi, "update_overrides") as mock_update:
+            out, _, code = self._run_main("--dry-run")
+        self.assertEqual(code, 0)
+        mock_update.assert_not_called()
+        self.assertIn("DRY-RUN: would update", out)
+
+    @patch.object(soi, "update_overrides")
+    @patch.object(soi, "find_detection")
+    def test_update_http_error(self, mock_find, mock_update):
+        mock_find.return_value = ("doc1", "so-detection", [])
+        mock_update.side_effect = requests.HTTPError("nope")
+        self._write_file("1010", [{"type": "suppress", "track": "by_src", "ip": "1.2.3.4"}])
+        out, _, code = self._run_main()
+        self.assertEqual(code, 1)
+        self.assertIn("update failed", out)
+
+
+if __name__ == "__main__":
+    unittest.main()

salt/manager/tools/sbin/so-minion

@@ -133,7 +133,7 @@ function getinstallinfo() {
 		return 1
 	fi
 
-	export $(echo "$INSTALLVARS" | xargs)
+	while read -r var; do export "$var"; done <<< "$INSTALLVARS"
 	if [ $? -ne 0 ]; then
 		log "ERROR" "Failed to source install variables"
 		return 1
@@ -281,6 +281,39 @@ function deleteMinionFiles () {
 	fi
 }
 
+# Remove this minion's postgres Telegraf credential from the shared creds
+# pillar and drop the matching role in Postgres. Always returns 0 so a dead
+# or unreachable so-postgres doesn't block minion deletion — in that case we
+# log a warning and leave the role behind for manual cleanup.
+function remove_postgres_telegraf_from_minion() {
+	local MINION_SAFE
+	MINION_SAFE=$(echo "$MINION_ID" | tr '.-' '__' | tr '[:upper:]' '[:lower:]')
+	local PG_USER="so_telegraf_${MINION_SAFE}"
+
+	log "INFO" "Removing postgres telegraf cred for $MINION_ID"
+
+	so-telegraf-cred remove "$MINION_ID" >/dev/null 2>&1 || true
+
+	if docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^so-postgres$'; then
+		if ! docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf >/dev/null 2>&1 <<EOSQL
+DO \$\$
+BEGIN
+    IF EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '$PG_USER') THEN
+        EXECUTE format('REASSIGN OWNED BY %I TO so_telegraf', '$PG_USER');
+        EXECUTE format('DROP OWNED BY %I', '$PG_USER');
+        EXECUTE format('DROP ROLE %I', '$PG_USER');
+    END IF;
+END
+\$\$;
+EOSQL
+		then
+			log "WARN" "Failed to drop postgres role $PG_USER; pillar entry was removed — drop manually if the role persists"
+		fi
+	else
+		log "WARN" "so-postgres container is not running; skipping DB role cleanup for $PG_USER"
+	fi
+}
+
 # Create the minion file
 function ensure_socore_ownership() {
 	log "INFO" "Setting socore ownership on minion files"
@@ -542,6 +575,17 @@ function add_telegraf_to_minion() {
         log "ERROR" "Failed to add telegraf configuration to $PILLARFILE"
         return 1
     fi
+
+    # Provision the per-minion postgres Telegraf credential in the shared
+    # telegraf/creds.sls pillar. so-telegraf-cred is the only writer; it
+    # generates a password on first add and is a no-op on re-add so the cred
+    # is stable across repeated so-minion runs. postgres.telegraf_users on the
+    # manager creates/updates the DB role from the same pillar.
+    so-telegraf-cred add "$MINION_ID"
+    if [ $? -ne 0 ]; then
+        log "ERROR" "Failed to provision postgres telegraf cred for $MINION_ID"
+        return 1
+    fi
 }
 
 function add_influxdb_to_minion() {
@@ -1069,6 +1113,7 @@ case "$OPERATION" in
 
 	"delete")
 		log "INFO" "Removing minion $MINION_ID"
+		remove_postgres_telegraf_from_minion
 		deleteMinionFiles || {
 			log "ERROR" "Failed to delete minion files for $MINION_ID"
 			exit 1

salt/manager/tools/sbin/so-telegraf-cred

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+# Single writer for the Telegraf Postgres credentials pillar. Thin wrapper
+# around so-yaml.py that generates a password on first add and no-ops on
+# re-add so the cred is stable across repeated so-minion runs.
+#
+# Note: so-yaml.py splits keys on '.' with no escape. SO minion ids are
+# dot-free by construction (setup/so-functions:1884 takes the short_name
+# before the first '.'), so using the raw minion id as the key is safe.
+
+CREDS=/opt/so/saltstack/local/pillar/telegraf/creds.sls
+
+usage() {
+    echo "Usage: $0 <add|remove> <minion_id>" >&2
+    exit 2
+}
+
+seed_creds_file() {
+    mkdir -p "$(dirname "$CREDS")" || return 1
+    if [[ ! -f "$CREDS" ]]; then
+        (umask 027 && printf 'telegraf:\n  postgres_creds: {}\n' > "$CREDS") || return 1
+        chown socore:socore "$CREDS" 2>/dev/null || true
+        chmod 640 "$CREDS" || return 1
+    fi
+}
+
+OP=$1
+MID=$2
+[[ -z "$OP" || -z "$MID" ]] && usage
+
+case "$OP" in
+    add)
+        SAFE=$(echo "$MID" | tr '.-' '__' | tr '[:upper:]' '[:lower:]')
+        seed_creds_file || exit 1
+        if so-yaml.py get -r "$CREDS" "telegraf.postgres_creds.${MID}.user" >/dev/null 2>&1; then
+            exit 0
+        fi
+        PASS=$(tr -dc 'A-Za-z0-9~!@#^&*()_=+[]|;:,.<>?-' < /dev/urandom | head -c 72)
+        so-yaml.py replace "$CREDS" "telegraf.postgres_creds.${MID}.user" "so_telegraf_${SAFE}" >/dev/null
+        so-yaml.py replace "$CREDS" "telegraf.postgres_creds.${MID}.pass" "$PASS" >/dev/null
+        ;;
+    remove)
+        [[ -f "$CREDS" ]] || exit 0
+        so-yaml.py remove "$CREDS" "telegraf.postgres_creds.${MID}" >/dev/null 2>&1 || true
+        ;;
+    *)
+        usage
+        ;;
+esac

salt/manager/tools/sbin/so-yaml.py

@@ -39,9 +39,16 @@ def showUsage(args):
 
 
 def loadYaml(filename):
-    file = open(filename, "r")
+    try:
+        with open(filename, "r") as file:
             content = file.read()
             return yaml.safe_load(content)
+    except FileNotFoundError:
+        print(f"File not found: {filename}", file=sys.stderr)
+        sys.exit(1)
+    except Exception as e:
+        print(f"Error reading file {filename}: {e}", file=sys.stderr)
+        sys.exit(1)
 
 
 def writeYaml(filename, content):
@@ -285,6 +292,7 @@ def add(args):
 def removeKey(content, key):
     pieces = key.split(".", 1)
     if len(pieces) > 1:
+        if pieces[0] in content:
             removeKey(content[pieces[0]], pieces[1])
     else:
         content.pop(key, None)

salt/manager/tools/sbin/so-yaml_test.py

@@ -973,3 +973,21 @@ class TestReplaceListObject(unittest.TestCase):
 
         expected = "key1:\n- id: '1'\n  status: updated\n- id: '2'\n  status: inactive\n"
         self.assertEqual(actual, expected)
+
+
+class TestLoadYaml(unittest.TestCase):
+
+    def test_load_yaml_missing_file(self):
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                soyaml.loadYaml("/tmp/so-yaml_test-does-not-exist.yaml")
+                sysmock.assert_called_with(1)
+                self.assertIn("File not found:", mock_stderr.getvalue())
+
+    def test_load_yaml_read_error(self):
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stderr', new=StringIO()) as mock_stderr:
+                with patch('builtins.open', side_effect=PermissionError("denied")):
+                    soyaml.loadYaml("/tmp/so-yaml_test-unreadable.yaml")
+                    sysmock.assert_called_with(1)
+                    self.assertIn("Error reading file", mock_stderr.getvalue())

salt/manager/tools/sbin/soup

@@ -24,6 +24,14 @@ BACKUPTOPFILE=/opt/so/saltstack/default/salt/top.sls.backup
 SALTUPGRADED=false
 SALT_CLOUD_INSTALLED=false
 SALT_CLOUD_CONFIGURED=false
+# Check if salt-cloud is installed
+if rpm -q salt-cloud &>/dev/null; then
+  SALT_CLOUD_INSTALLED=true
+fi
+# Check if salt-cloud is configured
+if [[ -f /etc/salt/cloud.profiles.d/socloud.conf ]]; then
+  SALT_CLOUD_CONFIGURED=true
+fi
 # used to display messages to the user at the end of soup
 declare -a FINAL_MESSAGE_QUEUE=()
 
@@ -180,13 +188,6 @@ airgap_update_dockers() {
   fi
 }
 
-backup_old_states_pillars() {
-
-	tar czf /nsm/backup/$(echo $INSTALLEDVERSION)_$(date +%Y%m%d-%H%M%S)_soup_default_states_pillars.tar.gz /opt/so/saltstack/default/
-	tar czf /nsm/backup/$(echo $INSTALLEDVERSION)_$(date +%Y%m%d-%H%M%S)_soup_local_states_pillars.tar.gz /opt/so/saltstack/local/
-
-}
-
 update_registry() {
   docker stop so-dockerregistry
   docker rm so-dockerregistry
@@ -363,6 +364,8 @@ preupgrade_changes() {
     echo "Checking to see if changes are needed."
 
     [[ "$INSTALLEDVERSION" =~ ^2\.4\.21[0-9]+$ ]] && up_to_3.0.0
+    [[ "$INSTALLEDVERSION" == "3.0.0" ]] && up_to_3.1.0
+    [[ "$INSTALLEDVERSION" == "3.1.0" ]] && up_to_3.2.0
     true
 }
 
@@ -371,6 +374,8 @@ postupgrade_changes() {
     echo "Running post upgrade processes."
 
     [[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0
+    [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0
+    [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0
     true
 }
 
@@ -445,7 +450,6 @@ migrate_pcap_to_suricata() {
 }
 
 up_to_3.0.0() {
-  determine_elastic_agent_upgrade
   migrate_pcap_to_suricata
 
   INSTALLEDVERSION=3.0.0
@@ -469,6 +473,363 @@ post_to_3.0.0() {
 
 ### 3.0.0 End ###
 
+### 3.1.0 Scripts ###
+
+elasticsearch_backup_index_templates() {
+  echo "Backing up current elasticsearch index templates in /opt/so/conf/elasticsearch/templates/index/ to /nsm/backup/3.0.0_elasticsearch_index_templates.tar.gz"
+  tar -czf /nsm/backup/3.0.0_elasticsearch_index_templates.tar.gz -C /opt/so/conf/elasticsearch/templates/index/ .
+}
+
+elasticfleet_set_agent_logging_level_warn() {
+    . /usr/sbin/so-elastic-fleet-common
+
+    local current_agent_policies
+    if ! current_agent_policies=$(fleet_api "agent_policies?perPage=1000"); then
+        echo "Warning: unable to retrieve Fleet agent policies"
+        return 0
+    fi
+
+    # Only updating policies that are within Security Onion defaults and do not already have any user configured advanced_settings.
+    local policies_to_update
+    policies_to_update=$(jq -c '
+        .items[]
+        | select(has("advanced_settings") | not)
+        | select(
+            .id == "so-grid-nodes_general"
+            or .id == "so-grid-nodes_heavy"
+            or .id == "endpoints-initial"
+            or (.id | startswith("FleetServer_"))
+          )
+    ' <<< "$current_agent_policies")
+
+    if [[ -z "$policies_to_update" ]]; then
+        return 0
+    fi
+
+    while IFS= read -r policy; do
+        [[ -z "$policy" ]] && continue
+
+        local policy_id policy_name policy_namespace
+        policy_id=$(jq -r '.id' <<< "$policy")
+        policy_name=$(jq -r '.name' <<< "$policy")
+        policy_namespace=$(jq -r '.namespace' <<< "$policy")
+
+        local update_logging
+        update_logging=$(jq -n \
+            --arg name "$policy_name" \
+            --arg namespace "$policy_namespace" \
+            '{name: $name, namespace: $namespace, advanced_settings: {agent_logging_level: "warning"}}'
+        )
+
+        echo "Setting elastic agent_logging_level to warning on policy '$policy_name' ($policy_id)."
+        if ! fleet_api "agent_policies/$policy_id" -XPUT -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$update_logging" >/dev/null; then
+            echo "  warning: failed to update agent policy '$policy_name' ($policy_id)" >&2
+        fi
+    done <<< "$policies_to_update"
+}
+
+update_logstash_pipeline_name() {
+    local original_pipeline_name="$1"
+    local new_pipeline_name="$2"
+
+    echo "Checking for conflicting logstash defined_pipelines pillar value."
+    local LOGSTASH_FILE=/opt/so/saltstack/local/pillar/logstash/soc_logstash.sls
+    local MINIONDIR=/opt/so/saltstack/local/pillar/minions
+    for pillar_file in "$LOGSTASH_FILE" "$MINIONDIR"/*.sls; do
+        [[ -f "$pillar_file" ]] || continue
+        if grep -q "$original_pipeline_name$" "$pillar_file"; then
+            echo "Found conflicting defined_pipeline pillar value in $pillar_file. Updating to use the new logstash pipeline name."
+            sed -i "s#$original_pipeline_name\$#$new_pipeline_name#g" "$pillar_file"
+            chown socore:socore "$pillar_file"
+        fi
+    done
+}
+
+check_transform_health_and_reauthorize() {
+    . /usr/sbin/so-elastic-fleet-common
+
+    echo "Checking integration transform jobs for unhealthy / unauthorized status..."
+
+    local transforms_doc stats_doc installed_doc
+    if ! transforms_doc=$(so-elasticsearch-query "_transform/_all?size=1000" --fail --retry 3 --retry-delay 5 2>/dev/null); then
+        echo "Unable to query for transform jobs, skipping reauthorization."
+        return 0
+    fi
+    if ! stats_doc=$(so-elasticsearch-query "_transform/_all/_stats?size=1000" --fail --retry 3 --retry-delay 5 2>/dev/null); then
+        echo "Unable to query for transform job stats, skipping reauthorization."
+        return 0
+    fi
+    if ! installed_doc=$(fleet_api "epm/packages/installed?perPage=500"); then
+        echo "Unable to list installed Fleet packages, skipping reauthorization."
+        return 0
+    fi
+
+    # Get all transforms that meet the following
+    # - unhealthy (any non-green health status)
+    # - metadata has run_as_kibana_system: false (this fix is specific to transforms started prior to Kibana 9.3.3)
+    # - are not orphaned (integration is not somehow missing/corrupt/uninstalled)
+    local tmp_transforms tmp_stats tmp_installed
+    tmp_transforms=$(mktemp)
+    tmp_stats=$(mktemp)
+    tmp_installed=$(mktemp)
+
+    echo "$transforms_doc" > "$tmp_transforms"
+    echo "$stats_doc"      > "$tmp_stats"
+    echo "$installed_doc"  > "$tmp_installed"
+
+    local unhealthy_transforms
+    unhealthy_transforms=$(jq -c -n \
+        --slurpfile t "$tmp_transforms" \
+        --slurpfile s "$tmp_stats" \
+        --slurpfile i "$tmp_installed" '
+        ($i[0].items | map({key: .name, value: .version}) | from_entries) as $pkg_ver
+        | ($s[0].transforms | map({key: .id, value: .health.status}) | from_entries) as $health
+        | [ $t[0].transforms[]
+            | select(._meta.run_as_kibana_system == false)
+            | select(($health[.id] // "unknown") != "green")
+            | {id, pkg: ._meta.package.name, ver: ($pkg_ver[._meta.package.name])}
+          ]
+        | if length == 0 then empty else . end
+        | (map(select(.ver == null)) | map({orphan: .id})[]),
+          (map(select(.ver != null))
+           | group_by(.pkg)
+           | map({pkg: .[0].pkg, ver: .[0].ver, transformIds: map(.id)})[])
+    ')
+
+    if [[ -z "$unhealthy_transforms" ]]; then
+        return 0
+    fi
+
+    local unhealthy_count
+    unhealthy_count=$(jq -s '[.[].transformIds? // empty | .[]] | length' <<< "$unhealthy_transforms")
+    echo "Found $unhealthy_count transform(s) needing reauthorization."
+
+    local total_failures=0
+    while IFS= read -r transform; do
+        [[ -z "$transform" ]] && continue
+        if jq -e 'has("orphan")' <<< "$transform" >/dev/null 2>&1; then
+            echo "Skipping transform not owned by any installed Fleet package: $(jq -r '.orphan' <<< "$transform")"
+            continue
+        fi
+
+        local pkg ver body resp
+        pkg=$(jq -r '.pkg' <<< "$transform")
+        ver=$(jq -r '.ver' <<< "$transform")
+        body=$(jq -c '{transforms: (.transformIds | map({transformId: .}))}' <<< "$transform")
+
+        echo "Reauthorizing transform(s) for ${pkg}-${ver}..."
+        resp=$(fleet_api "epm/packages/${pkg}/${ver}/transforms/authorize" \
+                        -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' \
+                        -d "$body") || { echo "Could not reauthorize transform(s) for ${pkg}-${ver}"; continue; }
+
+        (( total_failures += $(jq 'map(select(.success != true)) | length' <<< "$resp" 2>/dev/null) ))
+    done <<< "$unhealthy_transforms"
+
+    rm -f "$tmp_transforms" "$tmp_stats" "$tmp_installed"
+
+    if [[ "$total_failures" -gt 0 ]]; then
+        echo "Some transform(s) failed to reauthorize."
+    fi
+}
+
+ensure_postgres_local_pillar() {
+  # Postgres was added as a service after 3.0.0, so the new pillar/top.sls
+  # references postgres.soc_postgres / postgres.adv_postgres unconditionally.
+  # Managers upgrading from 3.0.0 have no /opt/so/saltstack/local/pillar/postgres/
+  # (make_some_dirs only runs at install time), so the stubs must be created
+  # here before salt-master restarts against the new top.sls.
+  echo "Ensuring postgres local pillar stubs exist."
+  local dir=/opt/so/saltstack/local/pillar/postgres
+  mkdir -p "$dir"
+  [[ -f "$dir/soc_postgres.sls" ]] || touch "$dir/soc_postgres.sls"
+  [[ -f "$dir/adv_postgres.sls" ]] || touch "$dir/adv_postgres.sls"
+  chown -R socore:socore "$dir"
+}
+
+ensure_postgres_secret() {
+  # On a fresh install, generate_passwords + secrets_pillar seed
+  # secrets:postgres_pass in /opt/so/saltstack/local/pillar/secrets.sls. That
+  # code path is skipped on upgrade (secrets.sls already exists from 3.0.0
+  # with import_pass/influx_pass but no postgres_pass), so the postgres
+  # container's POSTGRES_PASSWORD_FILE and SOC's PG_ADMIN_PASS would be empty
+  # after highstate. Generate one now if missing.
+  local secrets_file=/opt/so/saltstack/local/pillar/secrets.sls
+  if [[ ! -f "$secrets_file" ]]; then
+    echo "WARNING: $secrets_file missing; skipping postgres_pass backfill."
+    return 0
+  fi
+  if so-yaml.py get -r "$secrets_file" secrets.postgres_pass >/dev/null 2>&1; then
+    echo "secrets.postgres_pass already set; leaving as-is."
+    return 0
+  fi
+  echo "Seeding secrets.postgres_pass in $secrets_file."
+  so-yaml.py add "$secrets_file" secrets.postgres_pass "$(get_random_value)"
+  chown socore:socore "$secrets_file"
+}
+
+rename_strelka_scan_lnk() {
+  echo "Renaming strelka pillar ScanLNK to ScanLnk."
+  local STRELKA_FILE=/opt/so/saltstack/local/pillar/strelka/soc_strelka.sls
+  local MINIONDIR=/opt/so/saltstack/local/pillar/minions
+  local OLD_KEY=strelka.backend.config.backend.scanners.ScanLNK
+  local NEW_KEY=strelka.backend.config.backend.scanners.ScanLnk
+  local TMP_VALUE_FILE
+  TMP_VALUE_FILE=$(mktemp)
+
+  for pillar_file in "$STRELKA_FILE" "$MINIONDIR"/*.sls; do
+    [[ -f "$pillar_file" ]] || continue
+    # Skip if ScanLNK doesn't exist
+    so-yaml.py get "$pillar_file" "$OLD_KEY" > "$TMP_VALUE_FILE" 2>/dev/null || continue
+    echo "Found 'ScanLNK' key in $pillar_file. Renaming to 'ScanLnk'."
+    so-yaml.py add "$pillar_file" "$NEW_KEY" "file:$TMP_VALUE_FILE"
+    so-yaml.py remove "$pillar_file" "$OLD_KEY"
+  done
+
+  rm -f "$TMP_VALUE_FILE"
+}
+
+fix_logstash_0013_lumberjack_pipeline_name() {
+    update_logstash_pipeline_name "so/0013_input_lumberjack_fleet.conf" "so/0013_input_lumberjack_fleet.conf.jinja"
+}
+
+up_to_3.1.0() {
+  ensure_postgres_local_pillar
+  ensure_postgres_secret
+  determine_elastic_agent_upgrade
+  elasticsearch_backup_index_templates
+  # Clear existing component template state file.
+  rm -f /opt/so/state/esfleet_component_templates.json
+  rename_strelka_scan_lnk
+  fix_logstash_0013_lumberjack_pipeline_name
+
+  INSTALLEDVERSION=3.1.0
+}
+
+post_to_3.1.0() {
+  /usr/sbin/so-kibana-space-defaults
+  # ensure manager has new version of socloud.conf
+  if [[ $SALT_CLOUD_CONFIGURED == true ]]; then
+    salt-call state.apply salt.cloud.config concurrent=True
+  fi
+
+  # Backfill the Telegraf creds pillar for every accepted minion. so-telegraf-cred
+  # add is idempotent — it no-ops when an entry already exists — so this is safe
+  # to run on every soup. The subsequent state.apply creates/updates the matching
+  # Postgres roles from the reconciled pillar.
+  echo "Reconciling Telegraf Postgres creds for accepted minions."
+  for mid in $(salt-key --out=json --list=accepted 2>/dev/null | jq -r '.minions[]?' 2>/dev/null); do
+    [[ -n "$mid" ]] || continue
+    /usr/sbin/so-telegraf-cred add "$mid" || echo "  warning: so-telegraf-cred add $mid failed" >&2
+  done
+  # Run through the master (not --local) so state compilation uses the
+  # master's configured file_roots; the manager's /etc/salt/minion has no
+  # file_roots of its own and --local would fail with "No matching sls found".
+  salt-call state.apply postgres.telegraf_users queue=True || true
+
+  # Update default agent policies to use logging level warn.
+  elasticfleet_set_agent_logging_level_warn || true
+
+  # Check for unhealthy / unauthorized integration transform jobs and attempt reauthorizations
+  check_transform_health_and_reauthorize || true
+
+  POSTVERSION=3.1.0
+}
+
+### 3.1.0 End ###
+
+### 3.2.0 Scripts ###
+
+bootstrap_so_soc_database() {
+  # init-db.sh is mounted into so-postgres at /docker-entrypoint-initdb.d/init-db.sh
+  # and runs automatically only on a fresh data directory. Hosts upgrading from
+  # 3.1.0 already have /nsm/postgres populated, so the so_soc bootstrap block
+  # added in 3.2 never fires. Re-run the script explicitly; it's idempotent.
+  echo "Bootstrapping so_soc database via init-db.sh."
+  # The postgres image has no USER directive, so `docker exec` defaults to
+  # root, and the container env intentionally omits POSTGRES_USER (the upstream
+  # entrypoint defaults it transiently during first-init only). Recreate both
+  # so psql inside init-db.sh resolves the connect user correctly.
+  local exec_cmd="docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh"
+  if ! /usr/sbin/so-postgres-wait; then
+    FINAL_MESSAGE_QUEUE+=("WARNING: so-postgres was not ready during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd")
+    return 0
+  fi
+  if ! $exec_cmd; then
+    FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd")
+    return 0
+  fi
+  echo "so_soc bootstrap complete."
+}
+
+# Existing grids should keep ILM unless an admin explicitly opts in to DLM.
+pin_elasticsearch_data_retention_method() {
+  local elasticsearch_file=/opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls
+  mkdir -p "$(dirname "$elasticsearch_file")"
+  [[ -f "$elasticsearch_file" ]] || touch "$elasticsearch_file"
+
+  if so-yaml.py get -r "$elasticsearch_file" elasticsearch.data_retention_method >/dev/null 2>&1; then
+    echo "elasticsearch.data_retention_method already set; leaving as-is."
+    return 0
+  fi
+
+  echo "Pinning existing grid to ILM data retention."
+  so-yaml.py add "$elasticsearch_file" elasticsearch.data_retention_method ILM
+  chown socore:socore "$elasticsearch_file"
+}
+
+# Addes auto_expand_replicas setting to .kibana_streams index template
+#
+# In Kibana 9.3.3 the auto_expand_replicas setting was not added to the .kibana_streams index template. Causing single node deployments to be stuck in yellow state (unable to assign replica). Here we update the template in place using the so_kibana system user (system managed index template) to include the auto_expand_replicas setting
+#
+# Reference: https://github.com/elastic/kibana/issues/263048
+kibana_backport_streams_index_template() {
+    local current_template updated_template
+    current_template=$(so-elasticsearch-query "_index_template/.kibana_streams" --retry 3 --retry-delay 5 --fail)
+
+    if [[ -z "$current_template" ]]; then
+        echo "Unable to retrieve current .kibana_streams index template, skipping backport."
+        return 0
+    fi
+
+    updated_template=$(jq '.index_templates[0].index_template | .template.settings += {"index.auto_expand_replicas": "0-1"} | del(.created_date_millis, .modified_date_millis)' <<< "$current_template")
+
+    if ! kibana_user_pass=$(/usr/sbin/so-yaml.py get -r /opt/so/saltstack/local/pillar/elasticsearch/auth.sls elasticsearch.auth.users.so_kibana_user.pass); then
+        echo "Unable to retrieve so_kibana_user password, skipping .kibana_streams index template backport."
+        return 0
+    fi
+
+    if ! so-elasticsearch-query "_index_template/.kibana_streams" -XPUT -d "$updated_template" -u "so_kibana:$kibana_user_pass" --retry 3 --retry-delay 5 --fail; then
+        echo "Unable to automatically update .kibana_streams index template"
+        return 0
+    fi
+
+    ## NOTE: Should really add a check here for existing .kibana_streams index and then update its config in place
+
+}
+
+up_to_3.2.0() {
+  fix_logstash_0013_lumberjack_pipeline_name
+
+  pin_elasticsearch_data_retention_method
+
+  INSTALLEDVERSION=3.2.0
+}
+
+post_to_3.2.0() {
+  bootstrap_so_soc_database
+
+  # Including agent regen script here since it was missed in post_to_3.1.0
+  echo "Regenerating Elastic Agent Installers"
+  /sbin/so-elastic-agent-gen-installers
+
+  kibana_backport_streams_index_template
+
+  POSTVERSION=3.2.0
+}
+
+### 3.2.0 End ###
+
+
 repo_sync() {
   echo "Sync the local repo."
   su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
@@ -636,15 +997,6 @@ upgrade_check_salt() {
 upgrade_salt() {
   echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION."
   echo ""
-  # Check if salt-cloud is installed
-  if rpm -q salt-cloud &>/dev/null; then
-    SALT_CLOUD_INSTALLED=true
-  fi
-  # Check if salt-cloud is configured
-  if [[ -f /etc/salt/cloud.profiles.d/socloud.conf ]]; then
-    SALT_CLOUD_CONFIGURED=true
-  fi
-
   echo "Removing yum versionlock for Salt."
   echo ""
   yum versionlock delete "salt"
@@ -728,12 +1080,15 @@ verify_es_version_compatibility() {
     local is_active_intermediate_upgrade=1
     # supported upgrade paths for SO-ES versions
     declare -A es_upgrade_map=(
+        ["8.18.4"]="8.18.6 8.18.8 9.0.8"
+	    ["8.18.6"]="8.18.8 9.0.8"
 	    ["8.18.8"]="9.0.8"
+        ["9.0.8"]="9.3.3"
     )
 
     # Elasticsearch MUST upgrade through these versions
     declare -A es_to_so_version=(
-	    ["8.18.8"]="2.4.190-20251024"
+        ["9.0.8"]="3.0.0-20260331"
     )
 
     # Get current Elasticsearch version
@@ -745,26 +1100,182 @@ verify_es_version_compatibility() {
         exit 160
     fi
 
-    if ! target_es_version_raw=$(so-yaml.py get $UPDATE_DIR/salt/elasticsearch/defaults.yaml elasticsearch.version); then
+    if ! target_es_version=$(so-yaml.py get -r $UPDATE_DIR/salt/elasticsearch/defaults.yaml elasticsearch.version); then
-        # so-yaml.py failed to get the ES version from upgrade versions elasticsearch/defaults.yaml file. Likely they are upgrading to an SO version older than 2.4.110 prior to the ES version pinning and should be OKAY to continue with the upgrade.
-
-        # if so-yaml.py failed to get the ES version AND the version we are upgrading to is newer than 2.4.110 then we should bail
-        if [[ $(cat $UPDATE_DIR/VERSION | cut -d'.' -f3) > 110 ]]; then
         echo "Couldn't determine the target Elasticsearch version (post soup version) to ensure compatibility with current Elasticsearch version. Exiting"
 
         exit 160
     fi
 
-        # allow upgrade to version < 2.4.110 without checking ES version compatibility
+    compatible_es_versions="$target_es_version"
+    for current_version in "${!es_upgrade_map[@]}"; do
+        # shellcheck disable=SC2076
+        if [[ " ${es_upgrade_map[$current_version]} " =~ " $target_es_version " ]]; then
+            compatible_es_versions+=" $current_version"
+        fi
+    done
+
+    # Check if the given ES version can directly upgrade to the target ES version. Used to assist with catching lagging nodes during the upgrade process
+    es_version_can_upgrade_to_target() {
+        local current_version="$1"
+        # shellcheck disable=SC2076
+        if [[ -n "$current_version" && " $compatible_es_versions " =~ " $current_version " ]]; then
             return 0
-    else
+        fi
-        target_es_version=$(sed -n '1p' <<< "$target_es_version_raw")
+
+        return 1
+    }
+
+    # Gather Elasticsearch cluster version info and verify that each node in the cluster is running a version compatible with the target ES version.
+    verify_searchnodes_es_target_compatibility() {
+        local retries=20
+        local retry_count=0
+        local delay=180
+        local expected_es_nodes searchnode_minions attempt
+        local searchnode_discovery_success=false
+        SEARCHNODE_ES_VERSIONS=""
+
+        for attempt in {1..3}; do
+            if searchnode_minions=$(set -o pipefail; salt-key --out=json --list=accepted 2> /dev/null | jq -r '.minions[]? | select(endswith("searchnode"))'); then
+                searchnode_discovery_success=true
+                break
+            fi
+
+            echo "Failed to retrieve grid searchnodes via salt-key... Retrying in 30 seconds. Attempt $attempt of 3."
+            sleep 30
+        done
+
+        if [[ "$searchnode_discovery_success" != "true" ]]; then
+            echo "Failed to retrieve grid searchnodes via salt-key."
+            return 1
+        fi
+
+        # Always add node running soup to expected es nodes
+        expected_es_nodes="${MINIONID%_*}"
+        while IFS= read -r searchnode_minion; do
+            [[ -z "$searchnode_minion" ]] && continue
+            expected_es_nodes+=$'\n'"${searchnode_minion%_searchnode}"
+        done <<< "$searchnode_minions"
+
+        while [[ $retry_count -lt $retries ]]; do
+            SEARCHNODE_ES_VERSIONS=$(so-elasticsearch-query _nodes/_all/version --retry 5 --retry-delay 10 --fail 2>&1)
+            local exit_status=$?
+
+            if [[ $exit_status -ne 0 ]]; then
+                echo "Failed to retrieve Elasticsearch versions from searchnodes... Retrying in $delay seconds. Attempt $((retry_count + 1)) of $retries."
+                ((retry_count++))
+                sleep $delay
+                continue
+            fi
+
+            local all_searchnodes_compatible=true
+            while IFS=$'\t' read -r node current_version; do
+                [[ -z "$node" ]] && continue
+                if ! es_version_can_upgrade_to_target "$current_version"; then
+                    echo "Searchnode $node is running Elasticsearch $current_version, which is not directly upgradable to Elasticsearch $target_es_version."
+                    all_searchnodes_compatible=false
+                fi
+            done < <(echo "$SEARCHNODE_ES_VERSIONS" | jq -r '.nodes | to_entries[] | [.value.name, .value.version] | @tsv')
+
+            while IFS= read -r expected_es_node; do
+                [[ -z "$expected_es_node" ]] && continue
+                if ! echo "$SEARCHNODE_ES_VERSIONS" | jq -e --arg node "$expected_es_node" '.nodes | to_entries | any(.value.name == $node)' > /dev/null; then
+                    echo "Searchnode $expected_es_node did not report an Elasticsearch version. It may be offline or still upgrading."
+                    all_searchnodes_compatible=false
+                fi
+            done <<< "$expected_es_nodes"
+
+            if [[ "$all_searchnodes_compatible" == true ]]; then
+                echo "All Searchnodes are upgradable to Elasticsearch $target_es_version."
+                return 0
+            fi
+
+            echo "One or more Searchnodes cannot upgrade directly to Elasticsearch $target_es_version. Rechecking in $delay seconds. Attempt $((retry_count + 1)) of $retries."
+            ((retry_count++))
+            sleep $delay
+        done
+
+        return 1
+    }
+
+    # Gather heavynode version info and verify that each node is running a version compatible with the target ES version.
+    verify_heavynodes_es_target_compatibility() {
+        local heavynode_minions attempt
+        local retries=20
+        local retry_count=0
+        local delay=180
+        local heavynode_discovery_success=false
+        HEAVYNODE_ES_VERSIONS=""
+
+        for attempt in {1..3}; do
+            if heavynode_minions=$(set -o pipefail; salt-key --out=json --list=accepted 2> /dev/null | jq -r '.minions[]? | select(endswith("heavynode"))'); then
+                heavynode_discovery_success=true
+                break
+            fi
+
+            echo "Failed to retrieve grid heavynodes via salt-key... Retrying in 30 seconds. Attempt $attempt of 3."
+            sleep 30
+        done
+
+        if [[ "$heavynode_discovery_success" != "true" ]]; then
+            echo "Failed to retrieve grid heavynodes via salt-key."
+            return 1
+        fi
+
+        if [[ -z "$heavynode_minions" ]]; then
+            echo "No heavynodes detected. Skipping heavynode Elasticsearch version compatibility check."
+            return 0
+        fi
+
+        while [[ $retry_count -lt $retries ]]; do
+            HEAVYNODE_ES_VERSIONS=$(salt -C 'G@role:so-heavynode' cmd.run 'set -o pipefail; so-elasticsearch-query / --retry 5 --retry-delay 10 | jq -er ".version.number"' shell=/bin/bash --out=json 2> /dev/null)
+            local exit_status=$?
+
+            if [[ $exit_status -ne 0 ]]; then
+                echo "Failed to retrieve Elasticsearch version from one or more heavynodes... Retrying in $delay seconds. Attempt $((retry_count + 1)) of $retries."
+                ((retry_count++))
+                sleep $delay
+                continue
+            fi
+
+            local all_heavynodes_compatible=true
+            while IFS=$'\t' read -r node current_version; do
+                [[ -z "$node" ]] && continue
+                if ! es_version_can_upgrade_to_target "$current_version"; then
+                    echo "Heavynode $node is running Elasticsearch $current_version, which is not directly upgradable to Elasticsearch $target_es_version."
+                    all_heavynodes_compatible=false
+                fi
+            done < <(echo "$HEAVYNODE_ES_VERSIONS" | jq -r 'to_entries[] | [.key, .value] | @tsv')
+
+            while IFS= read -r heavynode_minion; do
+                [[ -z "$heavynode_minion" ]] && continue
+                if ! echo "$HEAVYNODE_ES_VERSIONS" | jq -se --arg minion "$heavynode_minion" 'add | has($minion)' > /dev/null; then
+                    echo "Heavynode $heavynode_minion did not report an Elasticsearch version. It may be offline or still upgrading."
+                    all_heavynodes_compatible=false
+                fi
+            done <<< "$heavynode_minions"
+
+            if [[ "$all_heavynodes_compatible" == true ]]; then
+                echo -e "\nAll heavynodes can upgrade to Elasticsearch $target_es_version."
+                return 0
+            fi
+
+            echo "One or more heavynodes cannot upgrade directly to Elasticsearch $target_es_version. Rechecking in $delay seconds. Attempt $((retry_count + 1)) of $retries."
+            ((retry_count++))
+            sleep $delay
+        done
+
+        return 1
+    }
+
+    if [[ ! -f "$es_verification_script" ]]; then
+        create_intermediate_upgrade_verification_script "$es_verification_script"
     fi
 
     for statefile in "${es_required_version_statefile_base}"-*; do
         [[ -f $statefile ]] || continue
 
-        local es_required_version_statefile_value=$(cat "$statefile")
+        local es_required_version_statefile_value
+        es_required_version_statefile_value=$(cat "$statefile")
 
         if [[ "$es_required_version_statefile_value" == "$target_es_version" ]]; then
             echo "Intermediate upgrade to ES $target_es_version is in progress. Skipping Elasticsearch version compatibility check."
@@ -773,19 +1284,14 @@ verify_es_version_compatibility() {
         fi
 
         # use sort to check if es_required_statefile_value is < the current es_version.
-        if [[ "$(printf '%s\n' $es_required_version_statefile_value $es_version | sort -V | head -n1)" == "$es_required_version_statefile_value" ]]; then
+        if [[ "$(printf '%s\n' "$es_required_version_statefile_value" "$es_version" | sort -V | head -n1)" == "$es_required_version_statefile_value" ]]; then
             rm -f "$statefile"
             continue
         fi
 
-        if [[ ! -f "$es_verification_script" ]]; then
-            create_intermediate_upgrade_verification_script "$es_verification_script"
-        fi
-
         echo -e "\n##############################################################################################################################\n"
         echo "A previously required intermediate Elasticsearch upgrade was detected. Verifying that all Searchnodes/Heavynodes have successfully upgraded Elasticsearch to $es_required_version_statefile_value before proceeding with soup to avoid potential data loss! This command can take up to an hour to complete."
-        timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$statefile"
+        if ! timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$statefile"; then
-        if [[ $? -ne 0 ]]; then
             echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
 
             echo "A previous required intermediate Elasticsearch upgrade to $es_required_version_statefile_value has yet to successfully complete across the grid. Please allow time for all Searchnodes/Heavynodes to have upgraded Elasticsearch to $es_required_version_statefile_value before running soup again to avoid potential data loss!"
@@ -802,7 +1308,28 @@ verify_es_version_compatibility() {
         return 0
     fi
 
+    # shellcheck disable=SC2076 # Do not want a regex here eg usage " 8.18.8 9.0.8 " =~ " 9.0.8 "
     if [[ " ${es_upgrade_map[$es_version]} " =~ " $target_es_version " || "$es_version" == "$target_es_version" ]]; then
+        if ! verify_searchnodes_es_target_compatibility || ! verify_heavynodes_es_target_compatibility; then
+            echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
+
+            echo "One or more Searchnode(s)/Heavynode(s) cannot upgrade directly to Elasticsearch $target_es_version. This can happen with soups that include Elasticsearch upgrades being run in quick succession. Typically, this will resolve itself as the grid synchronizes. Please allow time for all Searchnodes/Heavynodes to have upgraded Elasticsearch to a compatible version with $target_es_version before running soup again to avoid potential data loss!"
+
+            if [[ -n "$HEAVYNODE_ES_VERSIONS" ]]; then
+                echo "Current heavynode Elasticsearch versions:"
+                echo "$HEAVYNODE_ES_VERSIONS" | jq '.'
+            fi
+
+            if [[ -n "$SEARCHNODE_ES_VERSIONS" ]]; then
+                echo "Current searchnode Elasticsearch versions:"
+                echo "$SEARCHNODE_ES_VERSIONS" | jq '.nodes | to_entries | map({(.value.name): .value.version}) | sort | add'
+            fi
+
+            echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
+
+            exit 161
+        fi
+
         # supported upgrade
         return 0
     else
@@ -810,7 +1337,7 @@ verify_es_version_compatibility() {
         if [[ -z "$compatible_versions" ]]; then
             # If current ES version is not explicitly defined in the upgrade map, we know they have an intermediate upgrade to do.
             # We default to the lowest ES version defined in es_to_so_version as $first_es_required_version
-            local first_es_required_version=$(printf '%s\n' "${!es_to_so_version[@]}" | sort -V | head -n1)
+            first_es_required_version=$(printf '%s\n' "${!es_to_so_version[@]}" | sort -V | head -n1)
             next_step_so_version=${es_to_so_version[$first_es_required_version]}
             required_es_upgrade_version="$first_es_required_version"
         else
@@ -829,7 +1356,7 @@ verify_es_version_compatibility() {
         if [[ $is_airgap -eq 0 ]]; then
             run_airgap_intermediate_upgrade
         else
-            if [[ ! -z $ISOLOC ]]; then
+            if [[ -n $ISOLOC ]]; then
                 originally_requested_iso_location="$ISOLOC"
             fi
             # Make sure ISOLOC is not set. Network installs that used soup -f would have ISOLOC set.
@@ -861,7 +1388,8 @@ wait_for_salt_minion_with_restart() {
 }
 
 run_airgap_intermediate_upgrade() {
-    local originally_requested_so_version=$(cat $UPDATE_DIR/VERSION)
+    local originally_requested_so_version
+    originally_requested_so_version=$(cat "$UPDATE_DIR/VERSION")
     # preserve ISOLOC value, so we can try to use it post intermediate upgrade
     local originally_requested_iso_location="$ISOLOC"
 
@@ -873,7 +1401,8 @@ run_airgap_intermediate_upgrade() {
 
     while [[ -z "$next_iso_location" ]] || [[ ! -f "$next_iso_location" && ! -b "$next_iso_location" ]]; do
         # List removable devices if any are present
-        local removable_devices=$(lsblk -no PATH,SIZE,TYPE,MOUNTPOINTS,RM | awk '$NF==1')
+        local removable_devices
+        removable_devices=$(lsblk -no PATH,SIZE,TYPE,MOUNTPOINTS,RM | awk '$NF==1')
         if [[ -n "$removable_devices" ]]; then
             echo "PATH        SIZE    TYPE    MOUNTPOINTS    RM"
             echo "$removable_devices"
@@ -894,21 +1423,21 @@ run_airgap_intermediate_upgrade() {
 
     echo "Using $next_iso_location for required intermediary upgrade."
     exec bash <<EOF
-        ISOLOC=$next_iso_location soup -y && \
+        ISOLOC="$next_iso_location" soup -y && \
-        ISOLOC=$next_iso_location soup -y && \
+        ISOLOC="$next_iso_location" soup -y && \
 
         echo -e "\n##############################################################################################################################\n" && \
         echo -e "Verifying Elasticsearch was successfully upgraded to $required_es_upgrade_version across the grid. This part can take a while as Searchnodes/Heavynodes sync up with the Manager! \n\nOnce verification completes the next soup will begin automatically. If verification takes longer than 1 hour it will stop waiting and your grid will remain at $next_step_so_version. Allowing for all Searchnodes/Heavynodes to upgrade Elasticsearch to the required version on their own time.\n" && \
 
-        timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh $required_es_upgrade_version $es_required_version_statefile && \
+        timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh "$required_es_upgrade_version" "$es_required_version_statefile" && \
 
         echo -e "\n##############################################################################################################################\n" && \
 
         # automatically start the next soup if the original ISO isn't using the same block device we just used
         if [[ -n "$originally_requested_iso_location" ]] && [[ "$originally_requested_iso_location" != "$next_iso_location" ]]; then
             umount /tmp/soagupdate
-            ISOLOC=$originally_requested_iso_location soup -y && \
+            ISOLOC="$originally_requested_iso_location" soup -y && \
-            ISOLOC=$originally_requested_iso_location soup -y
+            ISOLOC="$originally_requested_iso_location" soup -y
         else
             echo "Could not automatically start next soup to $originally_requested_so_version. Soup will now exit here at $(cat /etc/soversion)" && \
 
@@ -924,29 +1453,29 @@ run_network_intermediate_upgrade() {
     if [[ -n "$BRANCH" ]]; then
         local originally_requested_so_branch="$BRANCH"
     else
-        local originally_requested_so_branch="2.4/main"
+        local originally_requested_so_branch="3/main"
     fi
 
     echo "Starting automated intermediate upgrade to $next_step_so_version."
     echo "After completion, the system will automatically attempt to upgrade to the latest version."
     echo -e "\n##############################################################################################################################\n"
     exec bash << EOF
-        BRANCH=$next_step_so_version soup -y && \
+        BRANCH="$next_step_so_version" soup -y && \
-        BRANCH=$next_step_so_version soup -y && \
+        BRANCH="$next_step_so_version" soup -y && \
 
         echo -e "\n##############################################################################################################################\n" && \
         echo -e "Verifying Elasticsearch was successfully upgraded to $required_es_upgrade_version across the grid. This part can take a while as Searchnodes/Heavynodes sync up with the Manager! \n\nOnce verification completes the next soup will begin automatically. If verification takes longer than 1 hour it will stop waiting and your grid will remain at $next_step_so_version. Allowing for all Searchnodes/Heavynodes to upgrade Elasticsearch to the required version on their own time.\n" && \
 
-        timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh $required_es_upgrade_version $es_required_version_statefile && \
+        timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh "$required_es_upgrade_version" "$es_required_version_statefile" && \
 
         echo -e "\n##############################################################################################################################\n" && \
         if [[ -n "$originally_requested_iso_location" ]]; then
             # nonairgap soup that used -f originally, runs intermediate upgrade using network + BRANCH, later coming back to the original ISO for the last soup
-            ISOLOC=$originally_requested_iso_location soup -y && \
+            ISOLOC="$originally_requested_iso_location" soup -y && \
-            ISOLOC=$originally_requested_iso_location soup -y
+            ISOLOC="$originally_requested_iso_location" soup -y
         else
-            BRANCH=$originally_requested_so_branch soup -y && \
+            BRANCH="$originally_requested_so_branch" soup -y && \
-            BRANCH=$originally_requested_so_branch soup -y
+            BRANCH="$originally_requested_so_branch" soup -y
         fi
         echo -e "\n##############################################################################################################################\n"
 EOF
@@ -1158,13 +1687,13 @@ main() {
   echo "Verifying we have the latest soup script."
   verify_latest_update_script
 
-  echo "Verifying Elasticsearch version compatibility before upgrading."
-  verify_es_version_compatibility
-
   echo "Let's see if we need to update Security Onion."
   upgrade_check
   upgrade_space
 
+  echo "Verifying Elasticsearch version compatibility across the grid before upgrading."
+  verify_es_version_compatibility
+
   echo "Checking for Salt Master and Minion updates."
   upgrade_check_salt
   set -e
@@ -1184,7 +1713,8 @@ main() {
     echo "Applying $HOTFIXVERSION hotfix"
     # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars
     if [[ ! "$MINION_ROLE" == "import" ]]; then
-      backup_old_states_pillars
+        echo "Running so-config-backup script."
+        /sbin/so-config-backup
     fi
     copy_new_files
     create_local_directories "/opt/so/saltstack/default"
@@ -1240,8 +1770,8 @@ main() {
     # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars
     if [[ ! "$MINION_ROLE" == "import" ]]; then
       echo ""
-      echo "Creating snapshots of default and local Salt states and pillars and saving to /nsm/backup/"
+      echo "Running so-config-backup script."
-      backup_old_states_pillars
+      /sbin/so-config-backup
     fi
 
     echo ""

salt/nginx/etc/nginx.conf

@@ -225,6 +225,7 @@ http {
 			limit_req             zone=auth_throttle burst={{ NGINXMERGED.config.throttle_login_burst }} nodelay;
 			limit_req_status      429;
 			proxy_pass            http://{{ GLOBALS.manager }}:4433;
+			proxy_set_header      Connection "Close";
 			proxy_read_timeout    90;
 			proxy_connect_timeout 90;
 			proxy_set_header      Host $host;
@@ -237,6 +238,7 @@ http {
 		location ~ ^/auth/.*?(whoami|logout|settings|errors|webauthn.js) {
 			rewrite               /auth/(.*) /$1 break;
 			proxy_pass            http://{{ GLOBALS.manager }}:4433;
+			proxy_set_header      Connection "Close";
 			proxy_read_timeout    90;
 			proxy_connect_timeout 90;
 			proxy_set_header      Host $host;

salt/orch/delete_hypervisor.sls

@@ -3,7 +3,14 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-{% set hypervisor = pillar.minion_id %}
+{% set hypervisor = pillar.get('minion_id', '') %}
+
+{% if not hypervisor|regex_match('^([A-Za-z0-9._-]{1,253})$') %}
+{%   do salt.log.error('delete_hypervisor_orch: refusing unsafe minion_id=' ~ hypervisor) %}
+delete_hypervisor_invalid_minion_id:
+  test.fail_without_changes:
+    - name: delete_hypervisor_invalid_minion_id
+{% else %}
 
 ensure_hypervisor_mine_deleted:
   salt.function:
@@ -20,3 +27,5 @@ update_salt_cloud_profile:
     - sls:
       - salt.cloud.config
     - concurrent: True
+
+{% endif %}

salt/orch/deploy_newnode.sls

@@ -25,8 +25,33 @@ manager_run_es_soc:
         - salt: {{NEWNODE}}_update_mine
 {% endif %}
 
+# so-minion has already added the new minion's entry to telegraf/creds.sls
+# via so-telegraf-cred before this orch fires. Reconcile the Postgres role
+# on the manager so the new minion can authenticate on its first highstate,
+# then refresh the minion's pillar so its telegraf.conf renders with the
+# freshly-written cred.
+manager_create_postgres_telegraf_role:
+  salt.state:
+    - tgt: {{ MANAGER }}
+    - sls:
+      - postgres.telegraf_users
+    - queue: True
+    - require:
+      - salt: {{NEWNODE}}_update_mine
+
+{{NEWNODE}}_refresh_pillar:
+  salt.function:
+    - name: saltutil.refresh_pillar
+    - tgt: {{ NEWNODE }}
+    - kwarg:
+        wait: True
+    - require:
+      - salt: manager_create_postgres_telegraf_role
+
 {{NEWNODE}}_run_highstate:
   salt.state:
     - tgt: {{ NEWNODE }}
     - highstate: True
     - queue: True
+    - require:
+      - salt: {{NEWNODE}}_refresh_pillar

salt/orch/vm_pillar_clean.sls

@@ -12,7 +12,14 @@
 {% if 'vrt' in salt['pillar.get']('features', []) %}
 
 {%   do salt.log.debug('vm_pillar_clean_orch: Running') %}
-{%   set vm_name = pillar.get('vm_name') %}
+{%   set vm_name = pillar.get('vm_name', '') %}
+
+{%   if not vm_name|regex_match('^([A-Za-z0-9._-]{1,253})$') %}
+{%     do salt.log.error('vm_pillar_clean_orch: refusing unsafe vm_name=' ~ vm_name) %}
+vm_pillar_clean_invalid_name:
+  test.fail_without_changes:
+    - name: vm_pillar_clean_invalid_name
+{%   else %}
 
 delete_adv_{{ vm_name }}_pillar:
   module.run:
@@ -24,6 +31,8 @@ delete_{{ vm_name }}_pillar:
     - file.remove:
       - path: /opt/so/saltstack/local/pillar/minions/{{ vm_name }}.sls
 
+{%   endif %}
+
 {% else %}
 
 {%   do salt.log.error(

salt/postgres/auth.sls

@@ -0,0 +1,37 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls in allowed_states %}
+
+  {% set DIGITS = "1234567890" %}
+  {% set LOWERCASE = "qwertyuiopasdfghjklzxcvbnm" %}
+  {% set UPPERCASE = "QWERTYUIOPASDFGHJKLZXCVBNM" %}
+  {% set SYMBOLS = "~!@#^&*()-_=+[]|;:,.<>?" %}
+  {% set CHARS = DIGITS~LOWERCASE~UPPERCASE~SYMBOLS %}
+  {% set so_postgres_user_pass = salt['pillar.get']('postgres:auth:users:so_postgres_user:pass', salt['random.get_str'](72, chars=CHARS)) %}
+
+# Admin cred only. Per-minion Telegraf creds live in telegraf/creds.sls,
+# managed by /usr/sbin/so-telegraf-cred (called from so-minion).
+postgres_auth_pillar:
+  file.managed:
+    - name: /opt/so/saltstack/local/pillar/postgres/auth.sls
+    - mode: 640
+    - reload_pillar: True
+    - contents: |
+        postgres:
+          auth:
+            users:
+              so_postgres_user:
+                user: so_postgres
+                pass: "{{ so_postgres_user_pass }}"
+    - show_changes: False
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/postgres/config.sls

@@ -0,0 +1,111 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'postgres/map.jinja' import PGMERGED %}
+
+# Postgres Setup
+postgresconfdir:
+  file.directory:
+    - name: /opt/so/conf/postgres
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+postgressecretsdir:
+  file.directory:
+    - name: /opt/so/conf/postgres/secrets
+    - user: 939
+    - group: 939
+    - mode: 700
+    - require:
+      - file: postgresconfdir
+
+postgresdatadir:
+  file.directory:
+    - name: /nsm/postgres
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+postgreslogdir:
+  file.directory:
+    - name: /opt/so/log/postgres
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+postgresinitdir:
+  file.directory:
+    - name: /opt/so/conf/postgres/init
+    - user: 939
+    - group: 939
+    - require:
+      - file: postgresconfdir
+
+postgresinitdb:
+  file.managed:
+    - name: /opt/so/conf/postgres/init/init-db.sh
+    - source: salt://postgres/files/init-db.sh
+    - user: 939
+    - group: 939
+    - mode: 755
+
+postgresconf:
+  file.managed:
+    - name: /opt/so/conf/postgres/postgresql.conf
+    - source: salt://postgres/files/postgresql.conf.jinja
+    - user: 939
+    - group: 939
+    - template: jinja
+    - defaults:
+        PGMERGED: {{ PGMERGED }}
+
+postgreshba:
+  file.managed:
+    - name: /opt/so/conf/postgres/pg_hba.conf
+    - source: salt://postgres/files/pg_hba.conf
+    - user: 939
+    - group: 939
+    - mode: 640
+
+postgres_super_secret:
+  file.managed:
+    - name: /opt/so/conf/postgres/secrets/postgres_password
+    - user: 939
+    - group: 939
+    - mode: 600
+    - contents_pillar: 'secrets:postgres_pass'
+    - show_changes: False
+    - require:
+      - file: postgressecretsdir
+
+postgres_app_secret:
+  file.managed:
+    - name: /opt/so/conf/postgres/secrets/so_postgres_pass
+    - user: 939
+    - group: 939
+    - mode: 600
+    - contents_pillar: 'postgres:auth:users:so_postgres_user:pass'
+    - show_changes: False
+    - require:
+      - file: postgressecretsdir
+
+postgres_sbin:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://postgres/tools/sbin
+    - user: root
+    - group: root
+    - file_mode: 755
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/postgres/defaults.yaml

@@ -0,0 +1,19 @@
+postgres:
+  enabled: True
+  telegraf:
+    retention_days: 14
+  config:
+    listen_addresses: '*'
+    port: 5432
+    max_connections: 100
+    shared_buffers: 256MB
+    ssl: 'on'
+    ssl_cert_file: '/conf/postgres.crt'
+    ssl_key_file: '/conf/postgres.key'
+    ssl_ca_file: '/conf/ca.crt'
+    hba_file: '/conf/pg_hba.conf'
+    log_destination: 'stderr'
+    logging_collector: 'off'
+    log_min_messages: 'warning'
+    shared_preload_libraries: pg_cron
+    cron.database_name: so_telegraf

salt/postgres/disabled.sls

@@ -0,0 +1,33 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+
+include:
+  - postgres.sostatus
+
+so-postgres:
+  docker_container.absent:
+    - force: True
+
+so-postgres_so-status.disabled:
+  file.comment:
+    - name: /opt/so/conf/so-status/so-status.conf
+    - regex: ^so-postgres$
+
+so_postgres_backup:
+  cron.absent:
+    - name: /usr/sbin/so-postgres-backup > /dev/null 2>&1
+    - identifier: so_postgres_backup
+    - user: root
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/postgres/enabled.sls

@@ -0,0 +1,109 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   set SO_POSTGRES_USER = salt['pillar.get']('postgres:auth:users:so_postgres_user:user', 'so_postgres') %}
+
+include:
+  - postgres.auth
+  - postgres.ssl
+  - postgres.config
+  - postgres.sostatus
+  - postgres.telegraf_users
+
+so-postgres:
+  docker_container.running:
+    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-postgres:{{ GLOBALS.so_version }}
+    - hostname: so-postgres
+    - networks:
+      - sobridge:
+        - ipv4_address: {{ DOCKERMERGED.containers['so-postgres'].ip }}
+    - port_bindings:
+      {% for BINDING in DOCKERMERGED.containers['so-postgres'].port_bindings %}
+      - {{ BINDING }}
+      {% endfor %}
+    - environment:
+      - POSTGRES_DB=securityonion
+      # Passwords are delivered via mounted 0600 secret files, not plaintext env vars.
+      # The upstream postgres image resolves POSTGRES_PASSWORD_FILE; entrypoint.sh and
+      # init-db.sh resolve SO_POSTGRES_PASS_FILE the same way.
+      - POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password
+      - SO_POSTGRES_USER={{ SO_POSTGRES_USER }}
+      - SO_POSTGRES_PASS_FILE=/run/secrets/so_postgres_pass
+      {% if DOCKERMERGED.containers['so-postgres'].extra_env %}
+        {% for XTRAENV in DOCKERMERGED.containers['so-postgres'].extra_env %}
+      - {{ XTRAENV }}
+        {% endfor %}
+      {% endif %}
+    - binds:
+      - /opt/so/log/postgres/:/log:rw
+      - /nsm/postgres:/var/lib/postgresql/data:rw
+      - /opt/so/conf/postgres/postgresql.conf:/conf/postgresql.conf:ro
+      - /opt/so/conf/postgres/pg_hba.conf:/conf/pg_hba.conf:ro
+      - /opt/so/conf/postgres/secrets:/run/secrets:ro
+      - /opt/so/conf/postgres/init/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh:ro
+      - /etc/pki/postgres.crt:/conf/postgres.crt:ro
+      - /etc/pki/postgres.key:/conf/postgres.key:ro
+      - /etc/pki/tls/certs/intca.crt:/conf/ca.crt:ro
+      {% if DOCKERMERGED.containers['so-postgres'].custom_bind_mounts %}
+        {% for BIND in DOCKERMERGED.containers['so-postgres'].custom_bind_mounts %}
+      - {{ BIND }}
+        {% endfor %}
+      {% endif %}
+    {% if DOCKERMERGED.containers['so-postgres'].extra_hosts %}
+    - extra_hosts:
+      {% for XTRAHOST in DOCKERMERGED.containers['so-postgres'].extra_hosts %}
+      - {{ XTRAHOST }}
+      {% endfor %}
+    {% endif %}
+    {% if DOCKERMERGED.containers['so-postgres'].ulimits %}
+    - ulimits:
+    {%   for ULIMIT in DOCKERMERGED.containers['so-postgres'].ulimits %}
+      - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
+    {%   endfor %}
+    {% endif %}
+    - watch:
+      - file: postgresconf
+      - file: postgreshba
+      - file: postgresinitdb
+      - file: postgres_super_secret
+      - file: postgres_app_secret
+      - x509: postgres_crt
+      - x509: postgres_key
+    - require:
+      - file: postgresconf
+      - file: postgreshba
+      - file: postgresinitdb
+      - file: postgres_super_secret
+      - file: postgres_app_secret
+      - x509: postgres_crt
+      - x509: postgres_key
+
+delete_so-postgres_so-status.disabled:
+  file.uncomment:
+    - name: /opt/so/conf/so-status/so-status.conf
+    - regex: ^so-postgres$
+
+so_postgres_backup:
+  cron.present:
+    - name: /usr/sbin/so-postgres-backup > /dev/null 2>&1
+    - identifier: so_postgres_backup
+    - user: root
+    - minute: '5'
+    - hour: '0'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/postgres/files/init-db.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+set -e
+
+# Create or update application user for SOC platform access
+# This script runs on first database initialization via docker-entrypoint-initdb.d
+# The password is properly escaped to handle special characters
+if [ -z "${SO_POSTGRES_PASS:-}" ] && [ -n "${SO_POSTGRES_PASS_FILE:-}" ] && [ -r "$SO_POSTGRES_PASS_FILE" ]; then
+    SO_POSTGRES_PASS="$(< "$SO_POSTGRES_PASS_FILE")"
+fi
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    DO \$\$
+    BEGIN
+        IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${SO_POSTGRES_USER}') THEN
+            EXECUTE format('CREATE ROLE %I WITH LOGIN PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}');
+        ELSE
+            EXECUTE format('ALTER ROLE %I WITH PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}');
+        END IF;
+    END
+    \$\$;
+    GRANT ALL ON SCHEMA public TO "$SO_POSTGRES_USER";
+    GRANT ALL PRIVILEGES ON DATABASE "$POSTGRES_DB" TO "$SO_POSTGRES_USER";
+    -- Lock the SOC database down at the connect layer; PUBLIC gets CONNECT
+    -- by default, which would let per-minion telegraf roles open sessions
+    -- here. They have no schema/table grants inside so reads fail, but
+    -- revoking CONNECT closes the soft edge entirely.
+    REVOKE CONNECT ON DATABASE "$POSTGRES_DB" FROM PUBLIC;
+    GRANT CONNECT ON DATABASE "$POSTGRES_DB" TO "$SO_POSTGRES_USER";
+EOSQL
+
+# Bootstrap the Telegraf metrics database. Per-minion roles + schemas are
+# reconciled on every state.apply by postgres/telegraf_users.sls; this block
+# only ensures the shared database exists on first initialization.
+if ! psql -U "$POSTGRES_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1; then
+    psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE DATABASE so_telegraf"
+fi

salt/postgres/files/pg_hba.conf

@@ -0,0 +1,16 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+#
+# Managed by Salt — do not edit by hand.
+# Client authentication config: only local (Unix socket) connections and TLS-wrapped TCP
+# connections are accepted. Plain-text `host ...` lines are intentionally omitted so a
+# misconfigured client with sslmode=disable cannot negotiate a cleartext session.
+
+# Local connections (Unix socket, container-internal) use peer/trust.
+local   all             all                                     trust
+
+# TCP connections MUST use TLS (hostssl) and authenticate with SCRAM.
+hostssl all             all             0.0.0.0/0               scram-sha-256
+hostssl all             all             ::/0                    scram-sha-256

salt/postgres/files/postgresql.conf.jinja

@@ -0,0 +1,8 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
+{% for key, value in PGMERGED.config.items() %}
+{{ key }} = '{{ value | string | replace("'", "''") }}'
+{% endfor %}

salt/postgres/init.sls

@@ -0,0 +1,13 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'postgres/map.jinja' import PGMERGED %}
+
+include:
+{% if PGMERGED.enabled %}
+  - postgres.enabled
+{% else %}
+  - postgres.disabled
+{% endif %}

salt/postgres/map.jinja

@@ -0,0 +1,7 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
+{% import_yaml 'postgres/defaults.yaml' as PGDEFAULTS %}
+{% set PGMERGED = salt['pillar.get']('postgres', PGDEFAULTS.postgres, merge=True) %}