Update so-nsm-clear

.github/DISCUSSION_TEMPLATE/3-0.yml

@@ -10,8 +10,6 @@ body:
       options:
         -
         - 3.0.0
-        - 3.1.0
-        - 3.2.0
         - Other (please provide detail below)
     validations:
       required: true

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 3.1.0-20260528 ISO image released on 2026/05/28
+### 3.0.0-20260331 ISO image released on 2026/03/31
 
 
 ### Download and Verify
 
-3.1.0-20260528 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-3.1.0-20260528.iso
+3.0.0-20260331 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
  
-MD5: 9D6FF58DEEE24089D722C73169765B3E  
-SHA1: 2B8B816B6CEC3B7F96B3C5E040EBF502DD2C412F  
-SHA256: 62FAB57E247C843D6A04F0796D8162C732B65D82FC3E4A59D087135B9FD32912  
+MD5: ECD318A1662A6FDE0EF213F5A9BD4B07  
+SHA1: E55BE314440CCF3392DC0B06BC5E270B43176D9C  
+SHA256: 7FC47405E335CBE5C2B6C51FE7AC60248F35CBE504907B8B5A33822B23F8F4D5  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.1.0-20260528.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.1.0-20260528.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-3.1.0-20260528.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-3.1.0-20260528.iso.sig securityonion-3.1.0-20260528.iso
+gpg --verify securityonion-3.0.0-20260331.iso.sig securityonion-3.0.0-20260331.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Wed 27 May 2026 03:03:59 PM EDT using RSA key ID FE507013
+gpg: Signature made Mon 30 Mar 2026 06:22:14 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

HOTFIX

@@ -1 +0,0 @@
-

VERSION

@@ -1 +1 @@
-3.2.0
+3.0.0

pillar/elasticsearch/index_templates.sls

@@ -0,0 +1,2 @@
+elasticsearch:
+  index_settings:

pillar/telegraf/creds.sls

@@ -1,12 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Per-minion Telegraf Postgres credentials. so-telegraf-cred on the manager is
-# the single writer; it mutates /opt/so/saltstack/local/pillar/telegraf/creds.sls
-# under flock. Pillar_roots order (local before default) means the populated
-# copy shadows this default on any real grid; this file exists so the pillar
-# key is always defined on fresh installs and when no minions have creds yet.
-telegraf:
-  postgres_creds: {}

pillar/top.sls

@@ -17,7 +17,6 @@ base:
     - sensoroni.adv_sensoroni
     - telegraf.soc_telegraf
     - telegraf.adv_telegraf
-    - telegraf.creds
     - versionlock.soc_versionlock
     - versionlock.adv_versionlock
     - soc.license
@@ -39,9 +38,6 @@ base:
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
-    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
-    - postgres.auth
-    {% endif %}
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
     - kibana.secrets
     {% endif %}
@@ -64,8 +60,6 @@ base:
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
-    - postgres.soc_postgres
-    - postgres.adv_postgres
     - elasticsearch.nodes
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
@@ -103,12 +97,10 @@ base:
     - node_data.ips
     - secrets
     - healthcheck.eval
+    - elasticsearch.index_templates
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
-    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
-    - postgres.auth
-    {% endif %}
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
     - kibana.secrets
     {% endif %}
@@ -134,8 +126,6 @@ base:
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
-    - postgres.soc_postgres
-    - postgres.adv_postgres
     - backup.soc_backup
     - backup.adv_backup
     - zeek.soc_zeek
@@ -152,12 +142,10 @@ base:
     - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
+    - elasticsearch.index_templates
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
-    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
-    - postgres.auth
-    {% endif %}
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
     - kibana.secrets
     {% endif %}
@@ -172,8 +160,6 @@ base:
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
-    - postgres.soc_postgres
-    - postgres.adv_postgres
     - elasticsearch.nodes
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
@@ -270,12 +256,10 @@ base:
   '*_import':
     - node_data.ips
     - secrets
+    - elasticsearch.index_templates
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
-    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/postgres/auth.sls') %}
-    - postgres.auth
-    {% endif %}
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
     - kibana.secrets
     {% endif %}
@@ -301,8 +285,6 @@ base:
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
-    - postgres.soc_postgres
-    - postgres.adv_postgres
     - zeek.soc_zeek
     - zeek.adv_zeek
     - bpf.soc_bpf

salt/_beacons/pillar_db.py

@@ -1,142 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Custom salt beacon that watches the SOC audit_settings table in postgres for
-# new settings changes and emits a beacon event per new row. This replaces the
-# inotify watch on /opt/so/saltstack/local/pillar -- instead of monitoring pillar
-# files on disk, we monitor the so_soc.audit_settings table that SOC writes to.
-#
-# Detection is poll-based with a monotonic `id` watermark persisted to
-# WATERMARK_FILE: each pass selects rows with id greater than the last id seen,
-# which makes it self-healing (a missed poll simply catches up on the next one).
-#
-# Each emitted event carries setting_id and node_id; the push_pillar reactor maps
-# setting_id -> app via pillar_push_map.yaml and writes a push intent, after which
-# the existing so-push-drainer / orch.push_batch pipeline takes over unchanged.
-
-import logging
-import os
-import subprocess
-
-log = logging.getLogger(__name__)
-
-WATERMARK_FILE = '/opt/so/state/pillar_db_watch.id'
-CONTAINER = 'so-postgres'
-DATABASE = 'so_soc'
-
-# Unaligned, tuples-only psql output with a field separator that cannot appear in
-# an id/setting_id/node_id, so we can split each row reliably.
-FIELD_SEP = '\x1f'
-
-
-def __virtual__():
-    return True
-
-
-def validate(config):
-    return True, 'valid'
-
-
-def _read_watermark():
-    # Returns the last processed id, or None if the watermark has not been seeded.
-    try:
-        with open(WATERMARK_FILE, 'r') as f:
-            return int((f.read() or '').strip())
-    except (IOError, ValueError):
-        return None
-
-
-def _write_watermark(value):
-    try:
-        os.makedirs(os.path.dirname(WATERMARK_FILE), exist_ok=True)
-        tmp = WATERMARK_FILE + '.tmp'
-        with open(tmp, 'w') as f:
-            f.write(str(int(value)))
-        os.rename(tmp, WATERMARK_FILE)
-    except OSError:
-        log.exception('pillar_db beacon: failed to persist watermark to %s', WATERMARK_FILE)
-
-
-def _query(sql):
-    # Run a query against so_soc inside the so-postgres container over the unix
-    # socket (trust auth, no password). Returns stdout on success, or None on any
-    # failure so the caller can no-op and retry on the next interval.
-    cmd = [
-        'docker', 'exec', CONTAINER,
-        'psql', '-U', 'postgres', '-d', DATABASE,
-        '-tA', '-F', FIELD_SEP, '-c', sql,
-    ]
-    try:
-        result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
-    except subprocess.TimeoutExpired:
-        log.warning('pillar_db beacon: psql timed out')
-        return None
-    except Exception:
-        log.exception('pillar_db beacon: failed to exec psql')
-        return None
-    if result.returncode != 0:
-        log.warning('pillar_db beacon: psql failed (rc=%s): %s',
-                    result.returncode, (result.stderr or '').strip())
-        return None
-    return result.stdout
-
-
-def beacon(config):
-    retval = []
-
-    watermark = _read_watermark()
-
-    # First run / missing watermark: seed to the current MAX(id) and emit nothing
-    # so we never replay the entire settings history into a fleetwide push.
-    if watermark is None:
-        seed = _query('SELECT COALESCE(MAX(id), 0) FROM audit_settings;')
-        if seed is None:
-            return retval  # postgres not ready yet; retry next interval
-        try:
-            _write_watermark(int((seed or '0').strip() or 0))
-        except ValueError:
-            log.warning('pillar_db beacon: could not parse MAX(id) seed: %r', seed)
-        return retval
-
-    rows = _query(
-        "SELECT id, setting_id, COALESCE(node_id, '') FROM audit_settings "
-        "WHERE id > %d ORDER BY id;" % watermark
-    )
-    if rows is None:
-        return retval
-
-    max_id = watermark
-    for line in rows.splitlines():
-        # Do NOT str.strip() the whole line: Python treats the \x1f field
-        # separator (and \x1c-\x1e) as whitespace, so stripping would eat an
-        # empty trailing node_id field and make the row look malformed.
-        if not line.strip():
-            continue
-        parts = line.split(FIELD_SEP)
-        if len(parts) < 3:
-            log.warning('pillar_db beacon: skipping malformed row: %r', line)
-            continue
-        try:
-            row_id = int(parts[0])
-        except ValueError:
-            log.warning('pillar_db beacon: skipping row with non-int id: %r', line)
-            continue
-        setting_id = parts[1]
-        node_id = parts[2]
-        retval.append({
-            'tag': 'audit_settings',
-            'id': row_id,
-            'setting_id': setting_id,
-            'node_id': node_id,
-        })
-        if row_id > max_id:
-            max_id = row_id
-
-    if max_id > watermark:
-        _write_watermark(max_id)
-        log.info('pillar_db beacon: emitted %d change(s), watermark %d -> %d',
-                 len(retval), watermark, max_id)
-
-    return retval

salt/allowed_states.map.jinja

@@ -29,14 +29,10 @@
     'manager',
     'nginx',
     'influxdb',
-    'postgres',
-    'postgres.auth',
     'soc',
     'kratos',
     'hydra',
     'elasticfleet',
-    'elasticfleet.manager',
-    'elasticsearch.cluster',
     'elastic-fleet-package-registry',
     'utility'
 ] %}
@@ -81,7 +77,7 @@
         ),
         'so-heavynode': (
             sensor_states +
-            ['elasticagent', 'elasticsearch', 'elasticsearch.cluster', 'logstash', 'redis', 'nginx']
+            ['elasticagent', 'elasticsearch', 'logstash', 'redis', 'nginx']
         ),
         'so-idh': (
             ['idh']

salt/backup/config_backup.sls

@@ -32,4 +32,3 @@ so_config_backup:
     - daymonth: '*'
     - month: '*'
     - dayweek: '*'
-

salt/backup/tools/sbin/so-config-backup.jinja

@@ -25,11 +25,9 @@ if [ ! -f $BACKUPFILE ]; then
   # Create empty backup file
   tar -cf $BACKUPFILE -T /dev/null
 
-  # Loop through all paths defined in global.sls, and append them to backup file if they exist
+  # Loop through all paths defined in global.sls, and append them to backup file
   {%- for LOCATION in BACKUPLOCATIONS %}
-  if [[ -d {{ LOCATION }} || -f {{ LOCATION }} ]]; then
-    tar -rf $BACKUPFILE "${EXCLUSIONS[@]}" {{ LOCATION }}
-  fi
+  tar -rf $BACKUPFILE "${EXCLUSIONS[@]}" {{ LOCATION }}
   {%- endfor %}
 
 fi

salt/ca/files/signing_policies.conf

@@ -54,20 +54,6 @@ x509_signing_policies:
     - extendedKeyUsage: serverAuth
     - days_valid: 820
     - copypath: /etc/pki/issued_certs/
-  postgres:
-    - minions: '*'
-    - signing_private_key: /etc/pki/ca.key
-    - signing_cert: /etc/pki/ca.crt
-    - C: US
-    - ST: Utah
-    - L: Salt Lake City
-    - basicConstraints: "critical CA:false"
-    - keyUsage: "critical keyEncipherment"
-    - subjectKeyIdentifier: hash
-    - authorityKeyIdentifier: keyid,issuer:always
-    - extendedKeyUsage: serverAuth
-    - days_valid: 820
-    - copypath: /etc/pki/issued_certs/
   elasticfleet:
     - minions: '*'
     - signing_private_key: /etc/pki/ca.key

salt/common/init.sls

@@ -130,17 +130,6 @@ common_sbin:
       - so-pcap-import
 {% endif %}
 
-# Pin physical NIC names by MAC (run-once) so a kernel upgrade can't renumber the
-# interfaces SO binds by name. The marker keeps it a one-time setup; an admin can
-# pre-create the marker to opt out.
-pin_nic_names:
-  cmd.run:
-    - name: /usr/sbin/so-nic-pin
-    - unless: 'test -e /opt/so/state/nic_names_pinned'
-    - require:
-      - file: common_sbin
-      - file: statedir
-
 common_sbin_jinja:
   file.recurse:
     - name: /usr/sbin

salt/common/tools/sbin/so-common

@@ -142,11 +142,6 @@ check_elastic_license() {
 	fi  
 }
 
-check_elasticsearch_responsive() {
-    retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" ||
-        fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot."
-}
-
 check_salt_master_status() {
 	local count=0
     local attempts="${1:- 10}"

salt/common/tools/sbin/so-image-common

@@ -31,7 +31,6 @@ container_list() {
       "so-hydra"
       "so-nginx"
       "so-pcaptools"
-      "so-postgres"
       "so-soc"
       "so-suricata"
       "so-telegraf"
@@ -56,7 +55,6 @@ container_list() {
       "so-logstash"
       "so-nginx"
       "so-pcaptools"
-      "so-postgres"
       "so-redis"
       "so-soc"
       "so-strelka-backend"
@@ -164,8 +162,8 @@ update_docker_containers() {
     # Pull down the trusted docker image
     run_check_net_err \
     "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" \
-    "Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1
-
+    "Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 
+    
     # Get signature
     run_check_net_err \
     "curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' $sig_url --output $SIGNPATH/$image.sig" \
@@ -188,27 +186,8 @@ update_docker_containers() {
         if [ -z "$HOSTNAME" ]; then
           HOSTNAME=$(hostname)
         fi
-        docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$image $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 || {
-          echo "Unable to tag $image" >> "$LOG_FILE" 2>&1
-          exit 1
-        }
-        # Push to the embedded registry via a registry-to-registry copy. Avoids
-        # `docker push`, which on Docker 29.x with the containerd image store
-        # represents freshly-pulled images as an index whose layer content
-        # isn't reachable through the push path. The local `docker tag` above
-        # is preserved so so-image-pull's `:5000` existence check still works.
-        # Pin to the digest already gpg-verified above so we copy exactly the
-        # bytes we approved.
-        local VERIFIED_REF
-        VERIFIED_REF=$(echo "$DOCKERINSPECT" | jq -r ".[0].RepoDigests[] | select(. | contains(\"$CONTAINER_REGISTRY\"))" | head -n 1)
-        if [ -z "$VERIFIED_REF" ] || [ "$VERIFIED_REF" = "null" ]; then
-          echo "Unable to determine verified digest for $image" >> "$LOG_FILE" 2>&1
-          exit 1
-        fi
-        docker buildx imagetools create --tag $HOSTNAME:5000/$IMAGEREPO/$image "$VERIFIED_REF" >> "$LOG_FILE" 2>&1 || {
-          echo "Unable to copy $image to embedded registry" >> "$LOG_FILE" 2>&1
-          exit 1
-        }
+        docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$image $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 
+        docker push $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 
       fi
     else
       echo "There is a problem downloading the $image image. Details: " >> "$LOG_FILE" 2>&1 

salt/common/tools/sbin/so-log-check

@@ -165,8 +165,6 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading component template"  # false positive (elasticsearch index or template names contain 'error')
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading composable template" # false positive (elasticsearch composable template names contain 'error')
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error while parsing document for index \[.ds-logs-kratos-so-.*object mapping for \[file\]" # false positive (mapping error occuring BEFORE kratos index has rolled over in 2.4.210)
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|No such container"            # false positive (telegraf trying to run stats on an old container)
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|passwords do not match"       # false positive (automated hydra test)
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
@@ -229,7 +227,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|from NIC checksum offloading" # zeek reporter.log
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal"           # docker container getting recycled
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-(tychon|aws_billing|microsoft_defender_endpoint|armis|o365_metrics|microsoft_sentinel|snyk|cyera|island_browser).*user so_kibana lacks the required permissions \[(logs|metrics)-\1" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user. This error should not be seen on fresh ES 9.3.3 installs or after SO 3.1.0 with soups addition of check_transform_health_and_reauthorize()
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-(tychon|aws_billing|microsoft_defender_endpoint).*user so_kibana lacks the required permissions \[logs-\1" # Known issue with 3 integrations using kibana_system role vs creating unique api creds with proper permissions.
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown"             # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1
 fi
 

salt/common/tools/sbin/so-nic-pin

@@ -1,76 +0,0 @@
-#!/bin/bash
-#
-# so-nic-pin — pin physical NIC names by permanent MAC via classic by-MAC udev
-#              rules, so a kernel upgrade can't renumber them.
-#
-# Security Onion binds its management and monitor interfaces BY NAME in pillar
-# (host:mainint, sensor:mainint, and bond0 is built on a specific physical NIC).
-# A kernel upgrade can change the kernel/systemd-udevd predictable-naming output
-# and renumber those NICs (e.g. enp1s0 -> enp2s0), which breaks the grid: the
-# pillar references a name that no longer exists and bond/bridge bring-up fails.
-#
-# This writes /etc/udev/rules.d/70-persistent-net.rules pinning each PHYSICAL NIC
-# to its CURRENT name by its PERMANENT MAC, freezing the names across future kernel
-# changes. It only writes the rules file; it does NOT live-trigger a rename (the
-# rules apply on the next boot/kernel, and a live rename would be disruptive).
-#
-# Run-once: gated by the drop file /opt/so/state/nic_names_pinned. If the marker is
-# present the script does nothing, so an admin can pre-create it to opt out. Invoked
-# from the common state on every highstate; the marker keeps it a one-time setup.
-
-NET_RULES_FILE="/etc/udev/rules.d/70-persistent-net.rules"
-MARKER="/opt/so/state/nic_names_pinned"
-
-log() { echo -e "[so-nic-pin] $*"; }
-
-# Echo "<name> <permanent-mac>" for every PHYSICAL NIC. A physical NIC is backed by a
-# real device (has device/driver), which excludes bond0/sobridge/docker0/veth*/lo whose
-# MACs are dynamic and must never be pinned. The PERMANENT MAC is used (ethtool -P, with
-# fallbacks), not the current one: an enslaved bond member's current MAC is rewritten to
-# the bond's, so matching on it would be wrong/ambiguous.
-physical_nics() {
-    local path n mac
-    for path in /sys/class/net/*; do
-        n="${path##*/}"
-        [ "$n" = "lo" ] && continue
-        [ -e "${path}/device/driver" ] || continue          # real device only
-        mac="$(ethtool -P "$n" 2>/dev/null | awk '/Permanent address/{print $NF}')"
-        case "$mac" in ""|00:00:00:00:00:00) mac="$(cat "${path}/bonding_slave/perm_hwaddr" 2>/dev/null)" ;; esac
-        case "$mac" in ""|00:00:00:00:00:00) mac="$(cat "${path}/address" 2>/dev/null)" ;; esac
-        case "$mac" in ""|00:00:00:00:00:00) continue ;; esac
-        echo "$n $mac"
-    done
-}
-
-# Turn "<name> <mac>" lines on stdin into classic by-MAC persistent-net udev rules.
-render_net_rules() {
-    echo "# Generated by so-nic-pin: pin NIC names by MAC so kernel upgrades can't renumber them."
-    echo "# Security Onion binds its management/monitor interfaces by name; do not hand-edit."
-    local n mac
-    while read -r n mac; do
-        [ -n "$n" ] || continue
-        printf 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="%s", NAME="%s"\n' \
-            "$mac" "$n"
-    done
-}
-
-[ "$(id -u)" -eq 0 ] || exit 0                   # salt runs us as root; bail quietly otherwise
-[ -e "${MARKER}" ] && exit 0                      # run-once guard (mirrors the state's unless)
-
-nics="$(physical_nics)"
-if [ -z "${nics}" ]; then
-    log "no physical NICs detected — nothing to pin (will retry on next highstate)"
-    exit 0                                         # do NOT drop the marker; let it retry later
-fi
-
-log "pinning physical NICs by permanent MAC:"
-echo "${nics}" | sed 's/^/    /'
-
-[ -f "${NET_RULES_FILE}" ] && cp -f "${NET_RULES_FILE}" "${NET_RULES_FILE}.bak"
-echo "${nics}" | render_net_rules > "${NET_RULES_FILE}" || {
-    log "ERROR: failed to write ${NET_RULES_FILE}"
-    exit 1
-}
-
-mkdir -p "$(dirname "${MARKER}")" && touch "${MARKER}"
-log "wrote ${NET_RULES_FILE} ($(grep -c '^SUBSYSTEM' "${NET_RULES_FILE}") NIC(s) pinned); dropped ${MARKER}"

salt/common/tools/sbin/so-nsm-clear

@@ -66,11 +66,22 @@ delete_zeek() {
   ZEEK_LOG="/nsm/zeek/logs/"
   [ -d $ZEEK_LOG ] && so-zeek-stop && rm -rf $ZEEK_LOG/* && so-zeek-start
 }
+delete_import() {
+  IMPORT_DATA="/nsm/import/"
+  [ -d $IMPORT_DATA ] && rm -rf $IMPORT_DATA/*
+}
+delete_strelka() {
+  STRELKA_HISTORY_DATA="/nsm/strelka/history/"
+  STRELKA_PROCESSED_DATA="/nsm/strelka/processed/"
+  [ -d $STRELKA_HISTORY_DATA ] && rm -rf $STRELKA_HISTORY_DATA/*
+  [ -d $STRELKA_PROCESSED_DATA ] && rm -rf $STRELKA_PROCESSED_DATA/*
+}
+
 
 so-suricata-stop
 delete_pcap
 delete_suricata
 delete_zeek
 so-suricata-start
-
-
+delete_import
+delete_strelka

salt/common/tools/sbin_jinja/so-raid-status

@@ -9,7 +9,7 @@
 
 . /usr/sbin/so-common
 
-software_raid=("SOSMN" "SOSMN-DE02" "SOSSNNV" "SOSSNNV-DE02" "SOS10k-DE02" "SOS10KNV" "SOS10KNV-DE02" "SOS10KNV-DE02" "SOS2000-DE02" "SOS-GOFAST-LT-DE02" "SOS-GOFAST-MD-DE02" "SOS-GOFAST-HV-DE02" "HVGUEST")
+software_raid=("SOSMN" "SOSMN-DE02" "SOSSNNV" "SOSSNNV-DE02" "SOS10k-DE02" "SOS10KNV" "SOS10KNV-DE02" "SOS10KNV-DE02" "SOS2000-DE02" "SOS-GOFAST-LT-DE02" "SOS-GOFAST-MD-DE02" "SOS-GOFAST-HV-DE02")
 hardware_raid=("SOS1000" "SOS1000F" "SOSSN7200" "SOS5000" "SOS4000")
 
 {%- if salt['grains.get']('sosmodel', '') %}
@@ -87,11 +87,6 @@ check_boss_raid() {
 }
 
 check_software_raid() {
-  if [[ ! -f /proc/mdstat ]]; then
-    SWRAID=0
-    return
-  fi
-
   SWRC=$(grep "_" /proc/mdstat)
   if [[ -n $SWRC ]]; then
       # RAID is failed in some way
@@ -112,9 +107,7 @@ if [[ "$is_hwraid" == "true" ]]; then
 fi
 if [[ "$is_softwareraid" == "true" ]]; then
 	check_software_raid
-  if [ "$model" != "HVGUEST" ]; then
-    check_boss_raid
-  fi
+  check_boss_raid
 fi
 
 sum=$(($SWRAID + $BOSSRAID + $HWRAID))

salt/common/tools/sbin_jinja/so-salt-minion-check

@@ -1,3 +1,5 @@
+{% import_yaml 'salt/minion.defaults.yaml' as SALT_MINION_DEFAULTS -%}
+
 #!/bin/bash
 #
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
@@ -23,8 +25,7 @@ SYSTEM_START_TIME=$(date -d "$(</proc/uptime awk '{print $1}') seconds ago" +%s)
 LAST_HIGHSTATE_END=$([ -e "/opt/so/log/salt/lasthighstate" ] && date -r /opt/so/log/salt/lasthighstate +%s || echo 0)
 LAST_HEALTHCHECK_STATE_APPLY=$([ -e "/opt/so/log/salt/state-apply-test" ] && date -r /opt/so/log/salt/state-apply-test +%s || echo 0)
 # SETTING THRESHOLD TO ANYTHING UNDER 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default
-# THRESHOLD is derived from the global push highstate interval + 1 hour, so the minion-check grace period tracks the schedule automatically.
-THRESHOLD=$(( ({{ salt['pillar.get']('global:push:highstate_interval_hours', 2) }} + 1) * 3600 )) #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted
+THRESHOLD={{SALT_MINION_DEFAULTS.salt.minion.check_threshold}} #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted
 THRESHOLD_DATE=$((LAST_HEALTHCHECK_STATE_APPLY+THRESHOLD))
 
 logCmd() {

salt/docker/defaults.yaml

@@ -237,11 +237,3 @@ docker:
       extra_hosts: []
       extra_env: []
       ulimits: []
-    'so-postgres':
-      final_octet: 47
-      port_bindings:
-        - 0.0.0.0:5432:5432
-      custom_bind_mounts: []
-      extra_hosts: []
-      extra_env: []
-      ulimits: []

salt/docker_clean/init.sls

@@ -9,8 +9,7 @@
 prune_images:
   cmd.run:
     - name: so-docker-prune
-    - onlyif: command -v /usr/sbin/so-docker-prune >/dev/null 2>&1
-    - order: 9000
+    - order: last
 
 {% else %}
 

salt/elastalert/enabled.sls

@@ -19,7 +19,6 @@ wait_for_elasticsearch:
 so-elastalert:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastalert:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: elastalert
     - name: so-elastalert
     - user: so-elastalert

salt/elastic-fleet-package-registry/enabled.sls

@@ -15,7 +15,6 @@ include:
 so-elastic-fleet-package-registry:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-fleet-package-registry:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - name: so-elastic-fleet-package-registry
     - hostname: Fleet-package-reg-{{ GLOBALS.hostname }}
     - detach: True
@@ -52,16 +51,6 @@ so-elastic-fleet-package-registry:
       - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
     {%   endfor %}
     {% endif %}
-
-wait_for_so-elastic-fleet-package-registry:
-  http.wait_for_successful_query:
-    - name: "http://localhost:8080/health"
-    - status: 200
-    - wait_for: 300
-    - request_interval: 15
-    - require:
-      - docker_container: so-elastic-fleet-package-registry
-
 delete_so-elastic-fleet-package-registry_so-status.disabled:
   file.uncomment:
     - name: /opt/so/conf/so-status/so-status.conf

salt/elasticagent/enabled.sls

@@ -16,7 +16,6 @@ include:
 so-elastic-agent:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - name: so-elastic-agent
     - hostname: {{ GLOBALS.hostname }}
     - detach: True

salt/elasticfleet/content-defaults.map.jinja

@@ -1,122 +0,0 @@
-{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-  or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
-  this file except in compliance with the Elastic License 2.0. #}
-
-
-{% import_json '/opt/so/state/esfleet_content_package_components.json' as ADDON_CONTENT_PACKAGE_COMPONENTS %}
-{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
-{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
-
-{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
-{% set ADDON_CONTENT_INTEGRATION_DEFAULTS = {} %}
-
-{% for pkg in ADDON_CONTENT_PACKAGE_COMPONENTS %}
-{%   if pkg.name in CORE_ESFLEET_PACKAGES %}
-{#     skip core content packages #}
-{%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
-{#     generate defaults for each content package #}
-{%     if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0%}
-{%       for pattern in pkg.dataStreams %}
-{#         in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config)
-             title: generic is an artifact of that and is not in use #}
-{%         if pattern.title == "generic" %}
-{%           continue %}
-{%         endif %}
-{%         if "metrics-" in pattern.name %}
-{%           set integration_type = "metrics-" %}
-{%         elif "logs-" in pattern.name %}
-{%           set integration_type = "logs-" %}
-{%         else %}
-{%           set integration_type = "" %}
-{%         endif %}
-{#         on content integrations the component name is user defined at the time it is added to an agent policy #}
-{%         set component_name = pattern.title %}
-{%         set index_pattern = pattern.name %}
-{#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
-{%           set component_name_x = component_name.replace(".","_x_") %}
-{#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
-{%           set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %}
-{#           Default integration settings #}
-{%           set integration_defaults = {
-               "index_sorting": false,
-               "index_template": {
-                 "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
-                 "data_stream": {
-                   "allow_custom_routing": false,
-                   "hidden": false
-                 },
-                 "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
-                 "index_patterns": [index_pattern],
-                 "priority": 501,
-                 "template": {
-                   "settings": {
-                     "index": {
-                       "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
-                         "number_of_replicas": 0
-                     }
-                   }
-                 }
-               },
-               "policy": {
-                 "phases": {
-                   "cold": {
-                     "actions": {
-                        "allocate":{
-                          "number_of_replicas": ""
-                        },
-                       "set_priority": {"priority": 0}
-                     },
-                     "min_age": "60d"
-                   },
-                   "delete": {
-                     "actions": {
-                       "delete": {}
-                     },
-                     "min_age": "365d"
-                   },
-                   "hot": {
-                     "actions": {
-                       "rollover": {
-                         "max_age": "30d",
-                         "max_primary_shard_size": "50gb"
-                       },
-                       "forcemerge":{
-                         "max_num_segments": ""
-                       },
-                       "shrink":{
-                         "max_primary_shard_size": "",
-                         "method": "COUNT",
-                         "number_of_shards": ""
-                       },
-                       "set_priority": {"priority": 100}
-                      },
-                      "min_age": "0ms"
-                   },
-                   "warm": {
-                     "actions": {
-                        "allocate": {
-                          "number_of_replicas": ""
-                        },
-                        "forcemerge": {
-                          "max_num_segments": ""
-                        },
-                        "shrink":{
-                          "max_primary_shard_size": "",
-                          "method": "COUNT",
-                          "number_of_shards": ""
-                        },
-                       "set_priority": {"priority": 50}
-                     },
-                     "min_age": "30d"
-                   }
-                 }
-               }
-             } %}
-
-
-{%         do ADDON_CONTENT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
-{%       endfor %}
-{%     else %}
-{%     endif %}
-{%   endif %}
-{% endfor %}

salt/elasticfleet/defaults.yaml

@@ -1,6 +1,5 @@
 elasticfleet:
   enabled: False
-  patch_version: 9.3.3+build202604082258 # Elastic Agent specific patch release.
   enable_manager_output: True
   config:
     server:

salt/elasticfleet/enabled.sls

@@ -17,19 +17,65 @@ include:
   - logstash.ssl
   - elasticfleet.config
   - elasticfleet.sostatus
-{%- if GLOBALS.role != "so-fleet" %}
-  - elasticfleet.manager
-{%- endif %}
 
-{% if GLOBALS.role != "so-fleet" %}
+{% if grains.role not in ['so-fleet'] %}
 # Wait for Elasticsearch to be ready - no reason to try running Elastic Fleet server if ES is not ready
 wait_for_elasticsearch_elasticfleet:
   cmd.run:
     - name: so-elasticsearch-wait
 {% endif %}
 
-{% if GLOBALS.role == "so-fleet" %}
+# If enabled, automatically update Fleet Logstash Outputs
+{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval', 'so-fleet'] %}
+so-elastic-fleet-auto-configure-logstash-outputs:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-outputs-update
+    - retry:
+        attempts: 4
+        interval: 30
+
+{# Separate from above in order to catch elasticfleet-logstash.crt changes and force update to fleet output policy #}
+so-elastic-fleet-auto-configure-logstash-outputs-force:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-outputs-update --certs
+    - retry:
+        attempts: 4
+        interval: 30
+    - onchanges:
+        - x509: etc_elasticfleet_logstash_crt
+        - x509: elasticfleet_kafka_crt
+{% endif %}
+
+# If enabled, automatically update Fleet Server URLs & ES Connection
+{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-fleet'] %}
+so-elastic-fleet-auto-configure-server-urls:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-urls-update
+    - retry:
+        attempts: 4
+        interval: 30
+{% endif %}
+
+# Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
+{% if grains.role not in ['so-fleet'] %}
+so-elastic-fleet-auto-configure-elasticsearch-urls:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-es-url-update
+    - retry:
+        attempts: 4
+        interval: 30
+
+so-elastic-fleet-auto-configure-artifact-urls:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-artifacts-url-update
+    - retry:
+        attempts: 4
+        interval: 30
+
+{% endif %}
+
 # Sync Elastic Agent artifacts to Fleet Node
+{% if grains.role in ['so-fleet'] %}
 elasticagent_syncartifacts:
   file.recurse:
     - name: /nsm/elastic-fleet/artifacts/beats
@@ -42,7 +88,6 @@ elasticagent_syncartifacts:
 so-elastic-fleet:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - name: so-elastic-fleet
     - hostname: FleetServer-{{ GLOBALS.hostname }}
     - detach: True
@@ -102,19 +147,59 @@ so-elastic-fleet:
       - file: trusttheca
       - x509: etc_elasticfleet_key
       - x509: etc_elasticfleet_crt
-
-wait_for_so-elastic-fleet:
-  http.wait_for_successful_query:
-    - name: "https://localhost:8220/api/status"
-    - ssl: True
-    - verify_ssl: False
-    - status: 200
-    - wait_for: 300
-    - request_interval: 15
-    - require:
-      - docker_container: so-elastic-fleet
 {%   endif %}
 
+{%  if GLOBALS.role != "so-fleet" %}
+so-elastic-fleet-package-statefile:
+  file.managed:
+    - name: /opt/so/state/elastic_fleet_packages.txt
+    - contents: {{ELASTICFLEETMERGED.packages}}
+
+so-elastic-fleet-package-upgrade:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-package-upgrade
+    - retry:
+        attempts: 3
+        interval: 10
+    - onchanges:
+      - file: /opt/so/state/elastic_fleet_packages.txt
+
+so-elastic-fleet-integrations:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-integration-policy-load
+    - retry:
+        attempts: 3
+        interval: 10
+
+so-elastic-agent-grid-upgrade:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-agent-grid-upgrade
+    - retry:
+        attempts: 12
+        interval: 5
+
+so-elastic-fleet-integration-upgrade:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-integration-upgrade
+    - retry:
+        attempts: 3
+        interval: 10
+
+{# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
+so-elastic-fleet-addon-integrations:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
+
+{%   if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
+so-elastic-defend-manage-filters-file-watch:
+  cmd.run:
+    - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log
+    - onchanges:
+      - file: elasticdefendcustom
+      - file: elasticdefenddisabled
+{%    endif %}
+{%  endif %}
+
 delete_so-elastic-fleet_so-status.disabled:
   file.uncomment:
     - name: /opt/so/conf/so-status/so-status.conf

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json

@@ -9,22 +9,16 @@
   "namespace": "so",
   "description": "Zeek Import logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/zeek/logs/*.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "import",
             "pipeline": "",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -40,8 +34,7 @@
             "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
           }
         }
       }

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json

@@ -15,25 +15,19 @@
     "version": ""
   },
   "name": "kratos-logs",
-  "namespace": "so",
   "description": "Kratos logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/kratos/kratos.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "kratos",
             "pipeline": "kratos",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -54,10 +48,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
           }
         }
       }

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json

@@ -9,22 +9,16 @@
   "namespace": "so",
   "description": "Zeek logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/zeek/logs/current/*.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "zeek",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"],
@@ -36,10 +30,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
           }
         }
       }

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -5,7 +5,7 @@
   "package": {
     "name": "endpoint",
     "title": "Elastic Defend",
-    "version": "9.3.0",
+    "version": "9.0.2",
     "requires_root": true
   },
   "enabled": true,

salt/elasticfleet/files/integrations/grid-nodes_general/elastic-agent-monitor.json

@@ -6,23 +6,21 @@
   "name": "agent-monitor",
   "namespace": "",
   "description": "",
-  "policy_id": "so-grid-nodes_general",
   "policy_ids": [
     "so-grid-nodes_general"
   ],
+  "output_id": null,
   "vars": {},
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/agents/agent-monitor.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "agentmonitor",
             "pipeline": "elasticagent.monitor",
             "parsers": "",
@@ -36,16 +34,15 @@
             "ignore_older": "72h",
             "clean_inactive": -1,
             "harvester_limit": 0,
-            "fingerprint": false,
+            "fingerprint": true,
             "fingerprint_offset": 0,
-            "file_identity_native": true,
+            "fingerprint_length": 64,
+            "file_identity_native": false,
             "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
           }
         }
       }
     }
-  },
-  "force": true
+  }
 }

salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json

@@ -4,25 +4,19 @@
     "version": ""
   },
   "name": "hydra-logs",
-  "namespace": "so",
   "description": "Hydra logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/hydra/hydra.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "hydra",
             "pipeline": "hydra",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -40,10 +34,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json

@@ -4,25 +4,19 @@
     "version": ""
   },
   "name": "idh-logs",
-  "namespace": "so",
   "description": "IDH integration",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/idh/opencanary.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "idh",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -37,10 +31,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -4,32 +4,26 @@
     "version": ""
   },
   "name": "import-evtx-logs",
-  "namespace": "so",
   "description": "Import Windows EVTX logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/evtx/*.json"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "import",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": [
               "\\.gz$"
             ],
             "include_files": [],
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.15.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.15.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.15.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.6.1\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.1.2\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.6.1\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.6.1\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.1.2\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
             ],
@@ -39,10 +33,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json

@@ -4,25 +4,19 @@
     "version": ""
   },
   "name": "import-suricata-logs",
-  "namespace": "so",
   "description": "Import Suricata logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/suricata/eve*.json"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "import",
             "pipeline": "suricata.common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -38,10 +32,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json

@@ -4,18 +4,14 @@
     "version": ""
   },
   "name": "rita-logs",
-  "namespace": "so",
   "description": "RITA Logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
@@ -23,8 +19,6 @@
               "/nsm/rita/exploded-dns.csv",
               "/nsm/rita/long-connections.csv"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "rita",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": [
@@ -39,10 +33,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json

@@ -4,25 +4,19 @@
     "version": ""
   },
   "name": "so-ip-mappings",
-  "namespace": "so",
   "description": "IP Description mappings",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/custom-mappings/ip-descriptions.csv"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "hostnamemappings",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
             "exclude_files": [
@@ -38,10 +32,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json

@@ -4,25 +4,19 @@
     "version": ""
   },
   "name": "soc-auth-sync-logs",
-  "namespace": "so",
   "description": "Security Onion - Elastic Auth Sync - Logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/sync.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -37,10 +31,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json

@@ -4,26 +4,20 @@
     "version": ""
   },
   "name": "soc-detections-logs",
-  "namespace": "so",
   "description": "Security Onion Console - Detections Logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/detections_runtime-status_sigma.log",
               "/opt/so/log/soc/detections_runtime-status_yara.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -41,10 +35,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json

@@ -4,25 +4,19 @@
     "version": ""
   },
   "name": "soc-salt-relay-logs",
-  "namespace": "so",
   "description": "Security Onion - Salt Relay - Logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/salt-relay.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -39,10 +33,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json

@@ -4,25 +4,19 @@
     "version": ""
   },
   "name": "soc-sensoroni-logs",
-  "namespace": "so",
   "description": "Security Onion - Sensoroni - Logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/sensoroni/sensoroni.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -37,10 +31,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json

@@ -4,25 +4,19 @@
     "version": ""
   },
   "name": "soc-server-logs",
-  "namespace": "so",
   "description": "Security Onion Console Logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/sensoroni-server.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "soc",
             "pipeline": "common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -39,10 +33,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json

@@ -4,25 +4,19 @@
     "version": ""
   },
   "name": "strelka-logs",
-  "namespace": "so",
   "description": "Strelka Logs",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/strelka/log/strelka.log"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "strelka",
             "pipeline": "strelka.file",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -37,10 +31,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json

@@ -4,25 +4,19 @@
     "version": ""
   },
   "name": "suricata-logs",
-  "namespace": "so",
   "description": "Suricata integration",
   "policy_id": "so-grid-nodes_general",
-  "policy_ids": [
-    "so-grid-nodes_general"
-  ],
-  "vars": {},
+  "namespace": "so",
   "inputs": {
     "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "filestream.filestream": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/suricata/eve*.json"
             ],
-            "compression_gzip": false,
-            "use_logs_stream": false,
             "data_stream.dataset": "suricata",
             "pipeline": "suricata.common",
             "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
@@ -37,10 +31,10 @@
             "harvester_limit": 0,
             "fingerprint": false,
             "fingerprint_offset": 0,
+            "fingerprint_length": "64",
             "file_identity_native": true,
             "exclude_lines": [],
-            "include_lines": [],
-            "delete_enabled": false
+            "include_lines": []
           }
         }
       }

salt/elasticfleet/input-defaults.map.jinja

@@ -1,121 +0,0 @@
-{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-  or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
-  this file except in compliance with the Elastic License 2.0. #}
-
-
-{% import_json '/opt/so/state/esfleet_input_package_components.json' as ADDON_INPUT_PACKAGE_COMPONENTS %}
-{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
-{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
-
-{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
-{% set ADDON_INPUT_INTEGRATION_DEFAULTS = {} %}
-
-{% for pkg in ADDON_INPUT_PACKAGE_COMPONENTS %}
-{%   if pkg.name in CORE_ESFLEET_PACKAGES %}
-{#     skip core input packages #}
-{%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
-{#     generate defaults for each input package #}
-{%     if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %}
-{%       for pattern in pkg.dataStreams %}
-{#         in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config)
-             title: generic is an artifact of that and is not in use #}
-{%         if pattern.title == "generic" %}
-{%           continue %}
-{%         endif %}
-{%         if "metrics-" in pattern.name %}
-{%           set integration_type = "metrics-" %}
-{%         elif "logs-" in pattern.name %}
-{%           set integration_type = "logs-" %}
-{%         else %}
-{%           set integration_type = "" %}
-{%         endif %}
-{#         on input integrations the component name is user defined at the time it is added to an agent policy #}
-{%         set component_name = pattern.title %}
-{%         set index_pattern = pattern.name %}
-{#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
-{%           set component_name_x = component_name.replace(".","_x_") %}
-{#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
-{%           set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %}
-{#           Default integration settings #}
-{%           set integration_defaults = {
-               "index_sorting": false,
-               "index_template": {
-                 "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
-                 "data_stream": {
-                   "allow_custom_routing": false,
-                   "hidden": false
-                 },
-                 "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
-                 "index_patterns": [index_pattern],
-                 "priority": 501,
-                 "template": {
-                   "settings": {
-                     "index": {
-                       "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
-                         "number_of_replicas": 0
-                     }
-                   }
-                 }
-               },
-               "policy": {
-                 "phases": {
-                   "cold": {
-                     "actions": {
-                        "allocate":{
-                          "number_of_replicas": ""
-                        },
-                       "set_priority": {"priority": 0}
-                     },
-                     "min_age": "60d"
-                   },
-                   "delete": {
-                     "actions": {
-                       "delete": {}
-                     },
-                     "min_age": "365d"
-                   },
-                   "hot": {
-                     "actions": {
-                       "rollover": {
-                         "max_age": "30d",
-                         "max_primary_shard_size": "50gb"
-                       },
-                       "forcemerge":{
-                         "max_num_segments": ""
-                       },
-                       "shrink":{
-                         "max_primary_shard_size": "",
-                         "method": "COUNT",
-                         "number_of_shards": ""
-                       },
-                       "set_priority": {"priority": 100}
-                      },
-                      "min_age": "0ms"
-                   },
-                   "warm": {
-                     "actions": {
-                        "allocate": {
-                          "number_of_replicas": ""
-                        },
-                        "forcemerge": {
-                          "max_num_segments": ""
-                        },
-                        "shrink":{
-                          "max_primary_shard_size": "",
-                          "method": "COUNT",
-                          "number_of_shards": ""
-                        },
-                       "set_priority": {"priority": 50}
-                     },
-                     "min_age": "30d"
-                   }
-                 }
-               }
-             } %}
-
-
-{%         do ADDON_INPUT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
-{%       endfor %}
-{%     endif %}
-{%   endif %}
-{% endfor %}

salt/elasticfleet/integration-defaults.map.jinja

@@ -59,8 +59,8 @@
 {#     skip core integrations #}
 {%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
 {#     generate defaults for each integration #}
-{%     if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %}
-{%       for pattern in pkg.dataStreams %}
+{%     if pkg.es_index_patterns is defined and pkg.es_index_patterns is not none %}
+{%       for pattern in pkg.es_index_patterns %}
 {%         if "metrics-" in pattern.name %}
 {%           set integration_type = "metrics-" %}
 {%         elif "logs-" in pattern.name %}
@@ -75,27 +75,44 @@
 {%         if component_name in WEIRD_INTEGRATIONS %}
 {%           set component_name = WEIRD_INTEGRATIONS[component_name] %}
 {%         endif %}
+
+{#         create duplicate of component_name, so we can split generics from @custom component templates in the index template below and overwrite the default @package when needed
+           eg. having to replace unifiedlogs.generic@package with filestream.generic@package, but keep the ability to customize unifiedlogs.generic@custom and its ILM policy #}
+{%         set custom_component_name = component_name %}
+
+{#         duplicate integration_type to assist with sometimes needing to overwrite component templates with 'logs-filestream.generic@package' (there is no metrics-filestream.generic@package) #}
+{%         set generic_integration_type = integration_type %}
+
 {#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
 {%           set component_name_x = component_name.replace(".","_x_") %}
 {#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
 {%           set integration_key = "so-" ~ integration_type ~ component_name_x %}
 
+{#         if its a .generic template make sure that a .generic@package for the integration exists. Else default to logs-filestream.generic@package #}
+{%         if ".generic" in component_name and integration_type ~ component_name ~ "@package" not in INSTALLED_COMPONENT_TEMPLATES %}
+{#           these generic templates by default are directed to index_pattern of 'logs-generic-*', overwrite that here to point to eg gcp_pubsub.generic-* #}
+{%           set index_pattern = integration_type ~ component_name ~ "-*" %}
+{#           includes use of .generic component template, but it doesn't exist in installed component templates. Redirect it to filestream.generic@package #}
+{%           set component_name = "filestream.generic" %}
+{%           set generic_integration_type = "logs-" %}
+{%         endif %}
+
 {#           Default integration settings #}
 {%           set integration_defaults = {
                "index_sorting": false,
                "index_template": {
-                 "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
+                 "composed_of": [generic_integration_type ~ component_name ~ "@package", integration_type ~ custom_component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
                  "data_stream": {
                    "allow_custom_routing": false,
                    "hidden": false
                  },
-                 "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
+                 "ignore_missing_component_templates": [integration_type ~ custom_component_name ~ "@custom"],
                  "index_patterns": [index_pattern],
                  "priority": 501,
                  "template": {
                    "settings": {
                      "index": {
-                       "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
+                       "lifecycle": {"name": "so-" ~ integration_type ~ custom_component_name ~ "-logs"},
                          "number_of_replicas": 0
                      }
                    }

salt/elasticfleet/manager.sls

@@ -1,124 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls in allowed_states %}
-{%   from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
-
-include:
-  - elasticfleet.config
-  - kibana.enabled
-
-# If enabled, automatically update Fleet Logstash Outputs
-{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration %}
-{%   if grains.role not in ['so-import', 'so-eval']%}
-so-elastic-fleet-auto-configure-logstash-outputs:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-outputs-update
-    - retry:
-        attempts: 4
-        interval: 30
-    - require:
-      - http: wait_for_so-kibana
-{%   endif %}
-
-# If enabled, automatically update Fleet Server URLs & ES Connection
-so-elastic-fleet-auto-configure-server-urls:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-urls-update
-    - retry:
-        attempts: 4
-        interval: 30
-    - require:
-      - http: wait_for_so-kibana
-{% endif %}
-
-# Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
-so-elastic-fleet-auto-configure-elasticsearch-urls:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-es-url-update
-    - retry:
-        attempts: 4
-        interval: 30
-    - require:
-      - http: wait_for_so-kibana
-
-so-elastic-fleet-auto-configure-artifact-urls:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-artifacts-url-update
-    - retry:
-        attempts: 4
-        interval: 30
-    - require:
-      - http: wait_for_so-kibana
-
-so-elastic-fleet-package-statefile:
-  file.managed:
-    - name: /opt/so/state/elastic_fleet_packages.txt
-    - contents: {{ELASTICFLEETMERGED.packages}}
-
-so-elastic-fleet-package-upgrade:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-package-upgrade
-    - retry:
-        attempts: 3
-        interval: 30
-    - require:
-      - http: wait_for_so-kibana
-    - onchanges:
-      - file: /opt/so/state/elastic_fleet_packages.txt
-
-so-elastic-fleet-integrations:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-integration-policy-load
-    - retry:
-        attempts: 3
-        interval: 10
-    - require:
-      - http: wait_for_so-kibana
-
-so-elastic-agent-grid-upgrade:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-agent-grid-upgrade
-    - retry:
-        attempts: 12
-        interval: 5
-    - require:
-      - http: wait_for_so-kibana
-
-so-elastic-fleet-integration-upgrade:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-integration-upgrade
-    - retry:
-        attempts: 3
-        interval: 10
-    - require:
-      - http: wait_for_so-kibana
-
-{# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
-so-elastic-fleet-addon-integrations:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
-    - require:
-      - http: wait_for_so-kibana
-
-{% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
-so-elastic-defend-manage-filters-file-watch:
-  cmd.run:
-    - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log
-    - require:
-      - http: wait_for_so-kibana
-    - onchanges:
-      - file: elasticdefendcustom
-      - file: elasticdefenddisabled
-{% endif %}
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/elasticfleet/tools/sbin/so-elastic-fleet-common

@@ -135,33 +135,9 @@ elastic_fleet_bulk_package_install() {
     fi
 }
 
-elastic_fleet_get_package_list_by_type() {
-    if ! output=$(fleet_api "epm/packages"); then
+elastic_fleet_installed_packages() {
+    if ! fleet_api "epm/packages/installed?perPage=500"; then
         return 1
-    else
-        is_integration=$(jq '[.items[] | select(.type=="integration") | .name ]' <<< "$output")
-        is_input=$(jq '[.items[] | select(.type=="input") | .name ]' <<< "$output")
-        is_content=$(jq '[.items[] | select(.type=="content") | .name ]' <<< "$output")
-        jq -n --argjson is_integration "${is_integration:-[]}" \
-              --argjson is_input "${is_input:-[]}" \
-              --argjson is_content "${is_content:-[]}" \
-              '{"integration": $is_integration,"input": $is_input, "content": $is_content}'
-    fi
-}
-elastic_fleet_installed_packages_components() {
-    package_type=${1,,}
-    if [[ "$package_type" != "integration" && "$package_type" != "input" && "$package_type" != "content" ]]; then
-        echo "Error: Invalid package type ${package_type}. Valid types are 'integration', 'input', or 'content'."
-        return 1
-    fi
-
-    packages_by_type=$(elastic_fleet_get_package_list_by_type)
-    packages=$(jq --arg package_type "$package_type" '.[$package_type]' <<< "$packages_by_type")
-
-    if ! output=$(fleet_api "epm/packages/installed?perPage=500"); then
-        return 1
-    else
-        jq -c --argjson packages "$packages" '[.items[] | select(.name | IN($packages[])) | {name: .name, dataStreams: .dataStreams}]' <<< "$output"
     fi
 }
 
@@ -240,7 +216,7 @@ elastic_fleet_policy_create() {
         --arg DESC "$DESC" \
         --arg TIMEOUT $TIMEOUT \
         --arg FLEETSERVER "$FLEETSERVER" \
-            '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER,"advanced_settings":{"agent_logging_level": "warning"}}'
+            '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER}'
         )
     # Create Fleet Policy
     if ! fleet_api "agent_policies" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -108,12 +108,9 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
   done
 
   # Only create the state file if all policies were created/updated successfully
-  if [[ $RETURN_CODE -eq 0 ]]; then
+  if [[ "$RETURN_CODE" != "1" ]]; then
     touch /opt/so/state/eaintegrations.txt
-  else
-    exit 1
   fi
 else
-  echo "Fleet integration policies already loaded."
-  exit 0
+  exit $RETURN_CODE
 fi

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade

@@ -5,13 +5,7 @@
 # this file except in compliance with the Elastic License 2.0.
 
 . /usr/sbin/so-common
-. /usr/sbin/so-elastic-fleet-common
 {%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
-{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
-{# Optionally override Elasticsearch version for Elastic Agent patch releases #}
-{%- if ELASTICFLEETDEFAULTS.elasticfleet.patch_version is defined %}
-{%-   do ELASTICSEARCHDEFAULTS.elasticsearch.update({'version': ELASTICFLEETDEFAULTS.elasticfleet.patch_version}) %}
-{%- endif %}
 
 # Only run on Managers
 if ! is_manager_node; then
@@ -20,10 +14,13 @@ if ! is_manager_node; then
 fi
 
 # Get current list of Grid Node Agents that need to be upgraded
-if ! RAW_JSON=$(fleet_api "agents?perPage=20&page=1&kuery=NOT%20agent.version%3A%20{{ELASTICSEARCHDEFAULTS.elasticsearch.version | urlencode }}%20AND%20policy_id%3A%20so-grid-nodes_%2A&showInactive=false&getStatusSummary=true" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
+RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=NOT%20agent.version%3A%20{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}%20AND%20policy_id%3A%20so-grid-nodes_%2A&showInactive=false&getStatusSummary=true" --retry 3 --retry-delay 30 --fail 2>/dev/null)
 
-    printf "Failed to query for current Grid Agents...\n"
-    exit 1
+# Check to make sure that the server responded with good data - else, bail from script
+CHECKSUM=$(jq -r '.page' <<< "$RAW_JSON")
+if [ "$CHECKSUM" -ne 1 ]; then
+ printf "Failed to query for current Grid Agents...\n"
+ exit 1
 fi
 
 # Generate list of Node Agents that need updates
@@ -34,12 +31,10 @@ if [ "$OUTDATED_LIST" != '[]' ]; then
    printf "Initiating upgrades for $AGENTNUMBERS Agents to Elastic {{ELASTICSEARCHDEFAULTS.elasticsearch.version}}...\n\n"
 
    # Generate updated JSON payload
-   JSON_STRING=$(jq -n --arg ELASTICVERSION "{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}" --argjson UPDATELIST "$OUTDATED_LIST" '{"version": $ELASTICVERSION,"agents": $UPDATELIST }')
+   JSON_STRING=$(jq -n --arg ELASTICVERSION {{ELASTICSEARCHDEFAULTS.elasticsearch.version}} --arg UPDATELIST $OUTDATED_LIST '{"version": $ELASTICVERSION,"agents": $UPDATELIST }')
 
    # Update Node Agents
-   if ! fleet_api "agents/bulk_upgrade" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
-       printf "Failed to initiate Agent upgrades...\n"
-   fi
+   curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "http://localhost:5601/api/fleet/agents/bulk_upgrade" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
 else
     printf "No Agents need updates... Exiting\n\n"
     exit 0

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load

@@ -18,9 +18,7 @@ INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json
 BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
 BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
 BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json
-INTEGRATION_PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
-INPUT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_input_package_components.json
-CONTENT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_content_package_components.json
+PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
 COMPONENT_TEMPLATES=/opt/so/state/esfleet_component_templates.json
 
 PENDING_UPDATE=false
@@ -181,13 +179,10 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
         else
             echo "Elastic integrations don't appear to need installation/updating..."
         fi
-        # Write out file for generating index/component/ilm templates, keeping each package type separate
-        for package_type in "INTEGRATION" "INPUT" "CONTENT"; do
-            if latest_installed_package_list=$(elastic_fleet_installed_packages_components "$package_type"); then
-                outfile="${package_type}_PACKAGE_COMPONENTS"
-                echo $latest_installed_package_list > "${!outfile}"
-            fi
-        done
+        # Write out file for generating index/component/ilm templates
+        if latest_installed_package_list=$(elastic_fleet_installed_packages); then
+            echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
+        fi
         if retry 3 1 "so-elasticsearch-query / --fail --output /dev/null"; then
             # Refresh installed component template list
             latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.')

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update

@@ -235,16 +235,6 @@ function update_kafka_outputs() {
 
 {% endif %}
 
-# Compare the current Elastic Fleet certificate against what is on disk
-POLICY_CERT_SHA=$(jq -r '.item.ssl.certificate' <<< $RAW_JSON | openssl x509 -noout -sha256 -fingerprint)
-DISK_CERT_SHA=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt -noout -sha256 -fingerprint)
-
-if [[ "$POLICY_CERT_SHA" != "$DISK_CERT_SHA" ]]; then
-    printf "Certificate on disk doesn't match certificate in policy - forcing update\n"
-    UPDATE_CERTS=true
-    FORCE_UPDATE=true
-fi
-
 # Sort & hash the new list of Logstash Outputs
 NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}")
 NEW_HASH=$(sha256sum <<< "$NEW_LIST_JSON" | awk '{print $1}')

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade

@@ -8,33 +8,18 @@
 
 . /usr/sbin/so-elastic-fleet-common
 
-PKG_LOAD_FAILURES=0
-PKG_LOAD_FAILURES_NAMES=()
-
 {%- for PACKAGE in SUPPORTED_PACKAGES %}
 echo "Upgrading {{ PACKAGE }} package..."
 if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
     if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
-        PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
-        PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
+        # exit 1 on failure to upgrade a default package, allow salt to handle retries
+        echo -e "\nERROR: Failed to upgrade $PACKAGE to version: $VERSION"
+        exit 1
     fi
 else
-    PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
-    PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
+    echo -e "\nERROR: Failed to get version information for integration $PACKAGE"
 fi
 echo
 {%- endfor %}
-
-if [ $PKG_LOAD_FAILURES -gt 0 ]; then
-    echo "ERROR: Failed to upgrade $PKG_LOAD_FAILURES package(s):"
-    for PKG in "${PKG_LOAD_FAILURES_NAMES[@]}"; do
-        echo " - $PKG"
-    done
-    # exit 1 on failure to upgrade a default package, allow salt to handle retries
-    exit 1
-else
-    echo "Successfully upgraded all packages."
-fi
-
 echo
 /usr/sbin/so-elasticsearch-templates-load

salt/elasticsearch/cluster.sls

@@ -1,194 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls in allowed_states %}
-{%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
-{%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %}
-{%   if GLOBALS.role != 'so-heavynode' %}
-{%     from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS, ADDON_INDICES %}
-{%   endif %}
-
-include:
-  - elasticsearch.enabled
-
-escomponenttemplates:
-  file.recurse:
-    - name: /opt/so/conf/elasticsearch/templates/component
-    - source: salt://elasticsearch/templates/component
-    - user: 930
-    - group: 939
-    - clean: True
-    - onchanges_in:
-      - file: so-elasticsearch-templates-reload
-    - show_changes: False
-
-# Clean up legacy and non-SO managed templates from the elasticsearch/templates/index/ directory
-so_index_template_dir:
-  file.directory:
-    - name: /opt/so/conf/elasticsearch/templates/index
-    - clean: True
-    {%- if SO_MANAGED_INDICES %}
-    - require:
-      {%- for index in SO_MANAGED_INDICES %}
-      - file: so_index_template_{{index}}
-      {%- endfor %}
-    {%- endif %}
-
-{%  if GLOBALS.role != "so-heavynode" %}
-# Clean up legacy and non-SO managed templates from the elasticsearch/templates/addon-index/ directory
-addon_index_template_dir:
-  file.directory:
-    - name: /opt/so/conf/elasticsearch/templates/addon-index
-    - clean: True
-    {%- if ADDON_INDICES %}
-    - require:
-      {%- for index in ADDON_INDICES %}
-      - file: addon_index_template_{{index}}
-      {%- endfor %}
-    {%- endif %}
-{%  endif %}
-
-# Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml)
-#   These index templates are for the core SO datasets and are always required
-{%  for index, settings in ES_INDEX_SETTINGS.items() %}
-{%    if settings.index_template is defined %}
-so_index_template_{{index}}:
-  file.managed:
-    - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json
-    - source: salt://elasticsearch/base-template.json.jinja
-    - defaults:
-        TEMPLATE_CONFIG: {{ settings.index_template }}
-    - template: jinja
-    - onchanges_in:
-      - file: so-elasticsearch-templates-reload
-{%    endif %}
-{%  endfor %}
-
-{%  if GLOBALS.role != "so-heavynode" %}
-# Auto-generate optional index templates for integration | input | content packages
-#   These index templates are not used by default (until user adds package to an agent policy).
-#   Pre-configured with standard defaults, and incorporated into SOC configuration for user customization.
-{%    for index,settings in ALL_ADDON_SETTINGS.items() %}
-{%      if settings.index_template is defined %}
-addon_index_template_{{index}}:
-  file.managed:
-    - name: /opt/so/conf/elasticsearch/templates/addon-index/{{ index }}-template.json
-    - source: salt://elasticsearch/base-template.json.jinja
-    - defaults:
-        TEMPLATE_CONFIG: {{ settings.index_template }}
-    - template: jinja
-    - show_changes: False
-    - onchanges_in:
-      - file: addon-elasticsearch-templates-reload
-{%       endif %}
-{%     endfor %}
-{%   endif %}
-
-{%  if GLOBALS.role in GLOBALS.manager_roles %}
-so-es-cluster-settings:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-cluster-settings
-    - cwd: /opt/so
-    - template: jinja
-    - require:
-      - docker_container: so-elasticsearch
-      - file: elasticsearch_sbin_jinja
-      - http: wait_for_so-elasticsearch
-{%  endif %}
-
-# heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)
-so-elasticsearch-ilm-policy-load:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-ilm-policy-load
-    - cwd: /opt/so
-    - require:
-      - docker_container: so-elasticsearch
-      - file: so-elasticsearch-ilm-policy-load-script
-    - onchanges:
-      - file: so-elasticsearch-ilm-policy-load-script
-
-so-elasticsearch-templates-reload:
-  file.absent:
-    - name: /opt/so/state/estemplates.txt
-
-addon-elasticsearch-templates-reload:
-  file.absent:
-    - name: /opt/so/state/addon_estemplates.txt
-
-# so-elasticsearch-templates-load will have its first successful run during the 'so-elastic-fleet-setup' script
-so-elasticsearch-templates:
-  cmd.run:
-{%-  if GLOBALS.role == "so-heavynode" %}
-    - name: /usr/sbin/so-elasticsearch-templates-load --heavynode
-{%-  else %}
-    - name: /usr/sbin/so-elasticsearch-templates-load
-{%-  endif %}
-    - cwd: /opt/so
-    - template: jinja
-    - require:
-      - docker_container: so-elasticsearch
-      - file: elasticsearch_sbin_jinja
-
-so-elasticsearch-dlm-apply:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-dlm-apply
-    - cwd: /opt/so
-    - require:
-      - docker_container: so-elasticsearch
-      - file: elasticsearch_sbin_jinja
-      - cmd: so-elasticsearch-templates
-    - retry:
-        attempts: 3
-        interval: 10
-
-so-elasticsearch-pipelines:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }}
-    - require:
-      - docker_container: so-elasticsearch
-      - file: so-elasticsearch-pipelines-script
-
-so-elasticsearch-roles-load:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-roles-load
-    - cwd: /opt/so
-    - template: jinja
-    - require:
-      - docker_container: so-elasticsearch
-      - file: elasticsearch_sbin_jinja
-
-{%  if grains.role in ['so-managersearch', 'so-manager', 'so-managerhype'] %}
-{%    set ap = "absent" %}
-{%  endif %}
-{%  if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %}
-{#    Remove so-elasticsearch-indices-delete script when using DLM #}
-{%    if ELASTICSEARCHMERGED.index_clean and ELASTICSEARCHMERGED.data_retention_method == "ILM" %}
-{%      set ap = "present" %}
-{%    else %}
-{%      set ap = "absent" %}
-{%    endif %}
-{%  endif %}
-{%  if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
-so-elasticsearch-indices-delete:
-  cron.{{ap}}:
-    - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1
-    - identifier: so-elasticsearch-indices-delete
-    - user: root
-    - minute: '*/5'
-    - hour: '*'
-    - daymonth: '*'
-    - month: '*'
-    - dayweek: '*'
-{%  endif %}
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/elasticsearch/config.sls

@@ -66,8 +66,6 @@ so-elasticsearch-ilm-policy-load-script:
     - group: 939
     - mode: 754
     - template: jinja
-    - defaults:
-        GLOBALS: {{ GLOBALS }}
     - show_changes: False
 
 so-elasticsearch-pipelines-script:
@@ -93,13 +91,6 @@ estemplatedir:
     - group: 939
     - makedirs: True
 
-esaddontemplatedir:
-  file.directory:
-    - name: /opt/so/conf/elasticsearch/templates/addon-index
-    - user: 930
-    - group: 939
-    - makedirs: True
-
 esrolesdir:
   file.directory:
     - name: /opt/so/conf/elasticsearch/roles

salt/elasticsearch/defaults.yaml

@@ -1,8 +1,7 @@
 elasticsearch:
   enabled: false
-  version: 9.3.3
+  version: 9.0.8
   index_clean: true
-  data_retention_method: DLM
   vm:
     max_map_count: 1048576
   config:
@@ -64,8 +63,6 @@ elasticsearch:
             verification_mode: none
   index_settings:
     global_overrides:
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         template:
           settings:
@@ -146,8 +143,6 @@ elasticsearch:
                 order: desc
     so-common:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -309,8 +304,6 @@ elasticsearch:
               number_of_shards: 1
     so-assistant-chat:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: ""
       index_template:
         composed_of:
         - assistant-chat-mappings
@@ -351,8 +344,6 @@ elasticsearch:
             min_age: 0ms
     so-assistant-session:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: ""
       index_template:
         composed_of:
         - assistant-session-mappings
@@ -506,8 +497,6 @@ elasticsearch:
             min_age: 30d
     so-idh:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -616,8 +605,6 @@ elasticsearch:
             min_age: 30d
     so-import:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -800,8 +787,6 @@ elasticsearch:
             min_age: 0ms
     so-kismet:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - kismet-mappings
@@ -851,8 +836,6 @@ elasticsearch:
             min_age: 30d
     so-kratos:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -921,8 +904,6 @@ elasticsearch:
             min_age: 30d
     so-hydra:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -1068,8 +1049,6 @@ elasticsearch:
             min_age: 0ms
     so-logs:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - so-data-streams-mappings
@@ -1150,8 +1129,6 @@ elasticsearch:
             min_age: 30d
     so-logs-detections_x_alerts:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - so-data-streams-mappings
@@ -1215,8 +1192,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1332,8 +1307,6 @@ elasticsearch:
             min_age: 30d
     so-elastic-agent-monitor:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1396,8 +1369,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_apm_server:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.apm_server@package
@@ -1462,8 +1433,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_auditbeat:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.auditbeat@package
@@ -1528,8 +1497,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_cloudbeat:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.cloudbeat@package
@@ -1594,8 +1561,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_endpoint_security:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1655,8 +1620,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_filebeat:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1716,8 +1679,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_fleet_server:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1774,8 +1735,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_heartbeat:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.heartbeat@package
@@ -1840,8 +1799,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_metricbeat:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1901,8 +1858,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_osquerybeat:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1962,8 +1917,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_packetbeat:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.packetbeat@package
@@ -2028,8 +1981,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elasticsearch_x_server:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-elasticsearch.server@package
@@ -2094,13 +2045,10 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_actions:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - .logs-endpoint.actions@package
         - .logs-endpoint.actions@custom
-        - endpoint@custom
         - event-mappings
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -2110,9 +2058,8 @@ elasticsearch:
           hidden: false
         ignore_missing_component_templates:
         - .logs-endpoint.actions@custom
-        - endpoint@custom
         index_patterns:
-        - .logs-endpoint.actions-*
+        - logs-endpoint.actions-*
         priority: 501
         template:
           settings:
@@ -2157,13 +2104,10 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_action_x_responses:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - .logs-endpoint.action.responses@package
         - .logs-endpoint.action.responses@custom
-        - endpoint@custom
         - event-mappings
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -2173,15 +2117,14 @@ elasticsearch:
           hidden: false
         ignore_missing_component_templates:
         - .logs-endpoint.action.responses@custom
-        - endpoint@custom
         index_patterns:
-        - .logs-endpoint.action.responses-*
+        - logs-endpoint.action.responses-*
         priority: 501
         template:
           settings:
             index:
               lifecycle:
-                name: so-logs-endpoint.action.responses-logs
+                name: so-logs-endpoint.actions-logs
               mapping:
                 total_fields:
                   limit: 5000
@@ -2220,8 +2163,6 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_alerts:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.alerts@package
@@ -2281,8 +2222,6 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_diagnostic_x_collection:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - .logs-endpoint.diagnostic.collection@package
@@ -2358,8 +2297,6 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_api:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.api@package
@@ -2419,8 +2356,6 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_file:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.file@package
@@ -2480,8 +2415,6 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_library:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.library@package
@@ -2541,8 +2474,6 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_network:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.network@package
@@ -2602,8 +2533,6 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_process:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.process@package
@@ -2663,8 +2592,6 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_registry:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.registry@package
@@ -2724,8 +2651,6 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_security:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.security@package
@@ -2785,8 +2710,6 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_heartbeat:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - .logs-endpoint.heartbeat@package
@@ -2846,8 +2769,6 @@ elasticsearch:
             min_age: 30d
     so-logs-http_endpoint_x_generic:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-http_endpoint.generic@package
@@ -2896,8 +2817,6 @@ elasticsearch:
             min_age: 30d
     so-logs-httpjson_x_generic:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-httpjson.generic@package
@@ -2963,8 +2882,6 @@ elasticsearch:
               number_of_replicas: 0
     so-logs-osquery-manager_x_action_x_responses:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         _meta:
           managed: true
@@ -3036,8 +2953,6 @@ elasticsearch:
               number_of_replicas: 0
     so-logs-osquery-manager_x_result:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         _meta:
           managed: true
@@ -3090,8 +3005,6 @@ elasticsearch:
             min_age: 30d
     so-logs-soc:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -3200,8 +3113,6 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_application:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3251,8 +3162,6 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_auth:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3302,8 +3211,6 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_security:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3353,8 +3260,6 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_syslog:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3404,8 +3309,6 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_system:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3455,8 +3358,6 @@ elasticsearch:
             min_age: 30d
     so-logs-windows_x_forwarded:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-windows.forwarded@package
@@ -3504,8 +3405,6 @@ elasticsearch:
             min_age: 30d
     so-logs-windows_x_powershell:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-windows.powershell@package
@@ -3553,8 +3452,6 @@ elasticsearch:
             min_age: 30d
     so-logs-windows_x_powershell_operational:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-windows.powershell_operational@package
@@ -3602,8 +3499,6 @@ elasticsearch:
             min_age: 30d
     so-logs-windows_x_sysmon_operational:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-windows.sysmon_operational@package
@@ -3651,8 +3546,6 @@ elasticsearch:
             min_age: 30d
     so-logs-winlog_x_winlog:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-winlog.winlog@package
@@ -3701,8 +3594,6 @@ elasticsearch:
             min_age: 30d
     so-logstash:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -3818,8 +3709,6 @@ elasticsearch:
             min_age: 30d
     so-metrics-endpoint_x_metadata:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - metrics-endpoint.metadata@package
@@ -3867,8 +3756,6 @@ elasticsearch:
             min_age: 30d
     so-metrics-endpoint_x_metrics:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - metrics-endpoint.metrics@package
@@ -3916,8 +3803,6 @@ elasticsearch:
             min_age: 30d
     so-metrics-endpoint_x_policy:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - metrics-endpoint.policy@package
@@ -3965,8 +3850,6 @@ elasticsearch:
             min_age: 30d
     so-metrics-fleet_server_x_agent_status:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - metrics@tsdb-settings
@@ -3991,8 +3874,6 @@ elasticsearch:
               number_of_replicas: 0
     so-metrics-fleet_server_x_agent_versions:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - metrics@tsdb-settings
@@ -4017,8 +3898,6 @@ elasticsearch:
               number_of_replicas: 0
     so-redis:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4134,8 +4013,6 @@ elasticsearch:
             min_age: 30d
     so-strelka:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4253,8 +4130,6 @@ elasticsearch:
             min_age: 30d
     so-suricata:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4371,8 +4246,6 @@ elasticsearch:
             min_age: 30d
     so-suricata_x_alerts:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4489,8 +4362,6 @@ elasticsearch:
             min_age: 30d
     so-syslog:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4607,8 +4478,6 @@ elasticsearch:
             min_age: 30d
     so-zeek:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings

salt/elasticsearch/enabled.sls

@@ -10,6 +10,8 @@
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
+{%   set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
+{%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
 
 include:
   - ca
@@ -17,14 +19,10 @@ include:
   - elasticsearch.ssl
   - elasticsearch.config
   - elasticsearch.sostatus
-{%- if GLOBALS.role != "so-searchnode" %}
-  - elasticsearch.cluster
-{%- endif%}
 
 so-elasticsearch:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:{{ ELASTICSEARCHMERGED.version }}
-    - restart_policy: unless-stopped
     - hostname: elasticsearch
     - name: so-elasticsearch
     - user: elasticsearch
@@ -103,24 +101,134 @@ so-elasticsearch:
       - cmd: auth_users_roles_inode
       - cmd: auth_users_inode
 
-wait_for_so-elasticsearch:
-  http.wait_for_successful_query:
-    - name: "https://localhost:9200/"
-    - username: 'so_elastic'
-    - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}'
-    - ssl: True
-    - verify_ssl: False
-    - status: 200
-    - wait_for: 300
-    - request_interval: 15
-    - require:
-      - docker_container: so-elasticsearch
-
 delete_so-elasticsearch_so-status.disabled:
   file.uncomment:
     - name: /opt/so/conf/so-status/so-status.conf
     - regex: ^so-elasticsearch$
 
+{%   if GLOBALS.role != "so-searchnode" %}
+escomponenttemplates:
+  file.recurse:
+    - name: /opt/so/conf/elasticsearch/templates/component
+    - source: salt://elasticsearch/templates/component
+    - user: 930
+    - group: 939
+    - clean: True
+    - onchanges_in:
+      - file: so-elasticsearch-templates-reload
+    - show_changes: False
+      
+# Auto-generate templates from defaults file
+{%     for index, settings in ES_INDEX_SETTINGS.items() %}
+ {%      if settings.index_template is defined %}
+es_index_template_{{index}}:
+  file.managed:
+    - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json
+    - source: salt://elasticsearch/base-template.json.jinja
+    - defaults:
+      TEMPLATE_CONFIG: {{ settings.index_template }}
+    - template: jinja
+    - show_changes: False
+    - onchanges_in:
+      - file: so-elasticsearch-templates-reload
+{%       endif %}
+{%     endfor %}
+
+{%     if TEMPLATES %}
+# Sync custom templates to /opt/so/conf/elasticsearch/templates
+{%       for TEMPLATE in TEMPLATES %}
+es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
+  file.managed:
+    - source: salt://elasticsearch/templates/index/{{TEMPLATE}}
+{%         if 'jinja' in TEMPLATE.split('.')[-1] %}
+    - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}}
+    - template: jinja
+{%         else %}
+    - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1]}}
+{%         endif %}
+    - user: 930
+    - group: 939
+    - show_changes: False
+    - onchanges_in:
+      - file: so-elasticsearch-templates-reload
+{%       endfor %}
+{%     endif %}
+
+{%     if GLOBALS.role in GLOBALS.manager_roles %}
+so-es-cluster-settings:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-cluster-settings
+    - cwd: /opt/so
+    - template: jinja
+    - require:
+      - docker_container: so-elasticsearch
+      - file: elasticsearch_sbin_jinja
+{%     endif %}
+
+so-elasticsearch-ilm-policy-load:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-ilm-policy-load
+    - cwd: /opt/so
+    - require:
+      - docker_container: so-elasticsearch
+      - file: so-elasticsearch-ilm-policy-load-script
+    - onchanges:
+      - file: so-elasticsearch-ilm-policy-load-script
+
+so-elasticsearch-templates-reload:
+  file.absent:
+    - name: /opt/so/state/estemplates.txt
+
+so-elasticsearch-templates:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-templates-load
+    - cwd: /opt/so
+    - template: jinja
+    - require:
+      - docker_container: so-elasticsearch
+      - file: elasticsearch_sbin_jinja
+
+so-elasticsearch-pipelines:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }}
+    - require:
+      - docker_container: so-elasticsearch
+      - file: so-elasticsearch-pipelines-script
+
+so-elasticsearch-roles-load:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-roles-load
+    - cwd: /opt/so
+    - template: jinja
+    - require:
+      - docker_container: so-elasticsearch
+      - file: elasticsearch_sbin_jinja
+
+{%     if grains.role in ['so-managersearch', 'so-manager', 'so-managerhype'] %}
+{%       set ap = "absent" %}
+{%     endif %}
+{%     if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %}
+{%       if ELASTICSEARCHMERGED.index_clean %}
+{%         set ap = "present" %}
+{%       else %}
+{%         set ap = "absent" %}
+{%       endif %}
+{%     endif %}  
+{%     if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
+so-elasticsearch-indices-delete:
+  cron.{{ap}}:
+    - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1
+    - identifier: so-elasticsearch-indices-delete
+    - user: root
+    - minute: '*/5'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+{%     endif %}
+
+{%   endif %}
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/elasticsearch/files/ingest/common

@@ -63,8 +63,7 @@
     { "set":             { "if": "ctx.event?.dataset != null && !ctx.event.dataset.contains('.')", "field": "event.dataset", "value": "{{event.module}}.{{event.dataset}}" } },
     { "split":           { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "dataset_tag_temp" } },
     { "append":          { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}"  } },
-    { "grok":            { "if": "ctx.http?.response?.status_code instanceof String", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long}(?:\\s+%{GREEDYDATA})?"], "ignore_failure": true } },
-    { "convert":         { "if": "ctx.http?.response?.status_code != null && !(ctx.http.response.status_code instanceof Number)", "field": "http.response.status_code", "type": "long", "ignore_failure": true } },
+    { "grok":            { "if": "ctx.http?.response?.status_code != null", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long} %{GREEDYDATA}"]} },
     { "set":             { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
     { "remove":          { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } },
     { "pipeline": { "name": "global@custom", "ignore_missing_pipeline": true, "description": "[Fleet] Global pipeline for all data streams" } }

salt/elasticsearch/files/ingest/global@custom

@@ -177,84 +177,12 @@
                 "description": "Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"
             }
         },
-        {
-            "script": {
-                "description": "Snapshot event.ingested into _tmp.event_ingested_pre_fleet before .fleet_final_pipeline-1 overwrites it with ES ingest time",
-                "lang": "painless",
-                "if": "ctx.event?.ingested != null && ctx.event?.created == null",
-                "ignore_failure": true,
-                "source": "ctx.putIfAbsent('_tmp', [:]); ctx._tmp.event_ingested_pre_fleet = ctx.event.ingested;"
-            }
-        },
         {
             "pipeline": {
                 "name": ".fleet_final_pipeline-1",
                 "ignore_missing_pipeline": true
             }
         },
-        {
-            "script": {
-                "description": "Calculate time from Elastic Agent to Logstash.",
-                "lang": "painless",
-                "if": "ctx._tmp?.logstash_from_agent != null",
-                "ignore_failure": true,
-                "source": "ZonedDateTime start = ctx._tmp.event_ingested_pre_fleet != null ? ZonedDateTime.parse(ctx._tmp.event_ingested_pre_fleet) : ZonedDateTime.parse(ctx['@timestamp']); ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_elasticagent_to_logstash = ChronoUnit.SECONDS.between(start, ZonedDateTime.parse(ctx._tmp.logstash_from_agent));"
-            }
-        },
-        {
-            "script": {
-                "description": "Calculate time from Logstash to Redis",
-                "lang": "painless",
-                "if": "ctx._tmp?.logstash_from_agent != null && ctx._tmp?.logstash_to_redis != null",
-                "ignore_failure": true,
-                "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_logstash_to_redis = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_from_agent), ZonedDateTime.parse(ctx._tmp.logstash_to_redis));"
-            }
-        },
-        {
-            "script": {
-                "description": "Calculate time message spends in redis queue (logstash delay in pulling event).",
-                "lang": "painless",
-                "if": "ctx._tmp?.logstash_to_redis != null && ctx._tmp?.logstash_from_redis != null",
-                "ignore_failure": true,
-                "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_redis_to_logstash = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_to_redis), ZonedDateTime.parse(ctx._tmp.logstash_from_redis));"
-            }
-        },
-        {
-            "script": {
-                "description": "Calculate time from Logstash to Elasticsearch (after read from Redis).",
-                "lang": "painless",
-                "if": "ctx._tmp?.logstash_from_redis != null",
-                "ignore_failure": true,
-                "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_logstash_to_elasticsearch = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_from_redis), metadata().now);"
-            }
-        },
-        {
-            "script": {
-                "description": "Calculate time from Elastic Agent to Kafka.",
-                "lang": "painless",
-                "if": "ctx._tmp?.logstash_from_kafka != null && ctx._tmp?.logstash_from_agent == null",
-                "ignore_failure": true,
-                "source": "ZonedDateTime start = ctx._tmp.event_ingested_pre_fleet != null ? ZonedDateTime.parse(ctx._tmp.event_ingested_pre_fleet) : ZonedDateTime.parse(ctx['@timestamp']); ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_elasticagent_to_kafka = ChronoUnit.SECONDS.between(start, ZonedDateTime.parse(ctx._tmp.logstash_from_kafka));"
-            }
-        },
-        {
-            "script": {
-                "description": "Calculate time message spends in Kafka queue (logstash delay in pulling event).",
-                "lang": "painless",
-                "if": "ctx._tmp?.logstash_from_kafka != null && ctx.metadata?.kafka?.timestamp != null && ctx._tmp?.logstash_from_agent == null",
-                "ignore_failure": true,
-                "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_kafka_queue = ChronoUnit.SECONDS.between(ZonedDateTime.ofInstant(Instant.ofEpochMilli(Long.parseLong(ctx.metadata.kafka.timestamp.toString())), ZoneId.of('UTC')), ZonedDateTime.parse(ctx._tmp.logstash_from_kafka));"
-            }
-        },
-        {
-            "script": {
-                "description": "Calculate time from Logstash to Elasticsearch (after read from Kafka).",
-                "lang": "painless",
-                "if": "ctx._tmp?.logstash_from_kafka != null && ctx._tmp?.logstash_from_agent == null",
-                "ignore_failure": true,
-                "source": "ctx.event.putIfAbsent('ingestion', [:]); ctx.event.ingestion.latency_kafka_to_elasticsearch = ChronoUnit.SECONDS.between(ZonedDateTime.parse(ctx._tmp.logstash_from_kafka), metadata().now);"
-            }
-        },
         {
             "remove": {
                 "field": "event.agent_id_status",
@@ -274,12 +202,11 @@
                     "event.dataset_temp",
                     "dataset_tag_temp",
                     "module_temp",
-                    "datastream_dataset_temp",
-                    "_tmp"
+                    "datastream_dataset_temp"
                 ],
                 "ignore_missing": true,
                 "ignore_failure": true
             }
         }
     ]
-}
+}

salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1

@@ -10,28 +10,24 @@
   "processors": [
     {
       "set": {
-        "tag": "set_ecs_version_f5923549",
         "field": "ecs.version",
         "value": "8.17.0"
       }
     },
     {
       "set": {
-        "tag": "set_observer_vendor_ad9d35cc",
         "field": "observer.vendor",
         "value": "netgate"
       }
     },
     {
       "set": {
-        "tag": "set_observer_type_5dddf3ba",
         "field": "observer.type",
         "value": "firewall"
       }
     },
     {
       "rename": {
-        "tag": "rename_message_to_event_original_56a77271",
         "field": "message",
         "target_field": "event.original",
         "ignore_missing": true,
@@ -40,14 +36,12 @@
     },
     {
       "set": {
-        "tag": "set_event_kind_de80643c",
         "field": "event.kind",
         "value": "event"
       }
     },
     {
       "set": {
-        "tag": "set_event_timezone_4ca44cac",
         "field": "event.timezone",
         "value": "{{{_tmp.tz_offset}}}",
         "if": "ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local'"
@@ -55,7 +49,6 @@
     },
     {
       "grok": {
-        "tag": "grok_event_original_27d9c8c7",
         "description": "Parse syslog header",
         "field": "event.original",
         "patterns": [
@@ -79,7 +72,6 @@
     },
     {
       "date": {
-        "tag": "date__tmp_timestamp8601_to_timestamp_6ac9d3ce",
         "if": "ctx._tmp.timestamp8601 != null",
         "field": "_tmp.timestamp8601",
         "target_field": "@timestamp",
@@ -90,7 +82,6 @@
     },
     {
       "date": {
-        "tag": "date__tmp_timestamp_to_timestamp_f21e536e",
         "if": "ctx.event?.timezone != null && ctx._tmp?.timestamp != null",
         "field": "_tmp.timestamp",
         "target_field": "@timestamp",
@@ -104,7 +95,6 @@
     },
     {
       "grok": {
-        "tag": "grok_process_name_cef3d489",
         "description": "Set Event Provider",
         "field": "process.name",
         "patterns": [
@@ -117,83 +107,71 @@
     },
     {
       "pipeline": {
-        "tag": "pipeline_e16851a7",
-        "name": "logs-pfsense.log-1.25.2-firewall",
+        "name": "logs-pfsense.log-1.23.1-firewall",
         "if": "ctx.event.provider == 'filterlog'"
       }
     },
     {
       "pipeline": {
-        "tag": "pipeline_828590b5",
-        "name": "logs-pfsense.log-1.25.2-openvpn",
+        "name": "logs-pfsense.log-1.23.1-openvpn",
         "if": "ctx.event.provider == 'openvpn'"
       }
     },
     {
       "pipeline": {
-        "tag": "pipeline_9d37039c",
-        "name": "logs-pfsense.log-1.25.2-ipsec",
+        "name": "logs-pfsense.log-1.23.1-ipsec",
         "if": "ctx.event.provider == 'charon'"
       }
     },
     {
       "pipeline": {
-        "tag": "pipeline_ad56bbca",
-        "name": "logs-pfsense.log-1.25.2-dhcp",
-        "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\", \"dnsmasq-dhcp\"].contains(ctx.event.provider)"
+        "name": "logs-pfsense.log-1.23.1-dhcp",
+        "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)"
       }
     },
     {
       "pipeline": {
-        "tag": "pipeline_dd85553d",
-        "name": "logs-pfsense.log-1.25.2-unbound",
+        "name": "logs-pfsense.log-1.23.1-unbound",
         "if": "ctx.event.provider == 'unbound'"
       }
     },
     {
       "pipeline": {
-        "tag": "pipeline_720ed255",
-        "name": "logs-pfsense.log-1.25.2-haproxy",
+        "name": "logs-pfsense.log-1.23.1-haproxy",
         "if": "ctx.event.provider == 'haproxy'"
       }
     },
     {
       "pipeline": {
-        "tag": "pipeline_456beba5",
-        "name": "logs-pfsense.log-1.25.2-php-fpm",
+        "name": "logs-pfsense.log-1.23.1-php-fpm",
         "if": "ctx.event.provider == 'php-fpm'"
       }
     },
     {
       "pipeline": {
-        "tag": "pipeline_a0d89375",
-        "name": "logs-pfsense.log-1.25.2-squid",
+        "name": "logs-pfsense.log-1.23.1-squid",
         "if": "ctx.event.provider == 'squid'"
       }
     },
     {
       "pipeline": {
-        "tag": "pipeline_c2f1ed55",
-        "name": "logs-pfsense.log-1.25.2-snort",
+        "name": "logs-pfsense.log-1.23.1-snort",
         "if": "ctx.event.provider == 'snort'"
       }
     },
     {
       "pipeline": {
-        "tag":"pipeline_33db1c9e",
-        "name": "logs-pfsense.log-1.25.2-suricata",
+        "name": "logs-pfsense.log-1.23.1-suricata",
         "if": "ctx.event.provider == 'suricata'"
       }
     },
     {
       "drop": {
-        "tag": "drop_9d7c46f8",
-        "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dnsmasq-dhcp\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)"
+        "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)"
       }
     },
     {
       "append": {
-        "tag": "append_event_category_4780a983",
         "field": "event.category",
         "value": "network",
         "if": "ctx.network != null"
@@ -201,7 +179,6 @@
     },
     {
       "convert": {
-        "tag": "convert_source_address_to_source_ip_f5632a20",
         "field": "source.address",
         "target_field": "source.ip",
         "type": "ip",
@@ -211,7 +188,6 @@
     },
     {
       "convert": {
-        "tag": "convert_destination_address_to_destination_ip_f1388f0c",
         "field": "destination.address",
         "target_field": "destination.ip",
         "type": "ip",
@@ -221,7 +197,6 @@
     },
     {
       "set": {
-        "tag": "set_network_type_1f1d940a",
         "field": "network.type",
         "value": "ipv6",
         "if": "ctx.source?.ip != null && ctx.source.ip.contains(\":\")"
@@ -229,7 +204,6 @@
     },
     {
       "set": {
-        "tag": "set_network_type_69deca38",
         "field": "network.type",
         "value": "ipv4",
         "if": "ctx.source?.ip != null && ctx.source.ip.contains(\".\")"
@@ -237,7 +211,6 @@
     },
     {
       "geoip": {
-        "tag": "geoip_source_ip_to_source_geo_da2e41b2",
         "field": "source.ip",
         "target_field": "source.geo",
         "ignore_missing": true
@@ -245,7 +218,6 @@
     },
     {
       "geoip": {
-        "tag": "geoip_destination_ip_to_destination_geo_ab5e2968",
         "field": "destination.ip",
         "target_field": "destination.geo",
         "ignore_missing": true
@@ -253,7 +225,6 @@
     },
     {
       "geoip": {
-        "tag": "geoip_source_ip_to_source_as_28d69883",
         "ignore_missing": true,
         "database_file": "GeoLite2-ASN.mmdb",
         "field": "source.ip",
@@ -266,7 +237,6 @@
     },
     {
       "geoip": {
-        "tag": "geoip_destination_ip_to_destination_as_8a007787",
         "database_file": "GeoLite2-ASN.mmdb",
         "field": "destination.ip",
         "target_field": "destination.as",
@@ -279,7 +249,6 @@
     },
     {
       "rename": {
-        "tag": "rename_source_as_asn_to_source_as_number_a917047d",
         "field": "source.as.asn",
         "target_field": "source.as.number",
         "ignore_missing": true
@@ -287,7 +256,6 @@
     },
     {
       "rename": {
-        "tag": "rename_source_as_organization_name_to_source_as_organization_name_f1362d0b",
         "field": "source.as.organization_name",
         "target_field": "source.as.organization.name",
         "ignore_missing": true
@@ -295,7 +263,6 @@
     },
     {
       "rename": {
-        "tag": "rename_destination_as_asn_to_destination_as_number_3b459fcd",
         "field": "destination.as.asn",
         "target_field": "destination.as.number",
         "ignore_missing": true
@@ -303,7 +270,6 @@
     },
     {
       "rename": {
-        "tag": "rename_destination_as_organization_name_to_destination_as_organization_name_814bd459",
         "field": "destination.as.organization_name",
         "target_field": "destination.as.organization.name",
         "ignore_missing": true
@@ -311,14 +277,12 @@
     },
     {
       "community_id": {
-        "tag": "community_id_d2308e7a",
         "target_field": "network.community_id",
         "ignore_failure": true
       }
     },
     {
       "grok": {
-        "tag": "grok_observer_ingress_interface_name_968018d3",
         "field": "observer.ingress.interface.name",
         "patterns": [
           "%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}"
@@ -329,7 +293,6 @@
     },
     {
       "set": {
-        "tag": "set_network_vlan_id_efd4d96a",
         "field": "network.vlan.id",
         "copy_from": "observer.ingress.vlan.id",
         "ignore_empty_value": true
@@ -337,7 +300,6 @@
     },
     {
       "append": {
-        "tag": "append_related_ip_c1a6356b",
         "field": "related.ip",
         "value": "{{{destination.ip}}}",
         "allow_duplicates": false,
@@ -346,7 +308,6 @@
     },
     {
       "append": {
-        "tag": "append_related_ip_8121c591",
         "field": "related.ip",
         "value": "{{{source.ip}}}",
         "allow_duplicates": false,
@@ -355,7 +316,6 @@
     },
     {
       "append": {
-        "tag": "append_related_ip_53b62ed8",
         "field": "related.ip",
         "value": "{{{source.nat.ip}}}",
         "allow_duplicates": false,
@@ -364,7 +324,6 @@
     },
     {
       "append": {
-        "tag": "append_related_hosts_6f162628",
         "field": "related.hosts",
         "value": "{{{destination.domain}}}",
         "if": "ctx.destination?.domain != null"
@@ -372,7 +331,6 @@
     },
     {
       "append": {
-        "tag": "append_related_user_c036eec2",
         "field": "related.user",
         "value": "{{{user.name}}}",
         "if": "ctx.user?.name != null"
@@ -380,7 +338,6 @@
     },
     {
       "set": {
-        "tag": "set_network_direction_cb1e3125",
         "field": "network.direction",
         "value": "{{{network.direction}}}bound",
         "if": "ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/"
@@ -388,7 +345,6 @@
     },
     {
       "remove": {
-        "tag": "remove_a82e20f2",
         "field": [
           "_tmp"
         ],
@@ -397,21 +353,11 @@
     },
     {
       "script": {
-        "tag": "script_a7f2c062",
         "lang": "painless",
         "description": "This script processor iterates over the whole document to remove fields with null values.",
         "source": "void handleMap(Map map) {\n  for (def x : map.values()) {\n    if (x instanceof Map) {\n        handleMap(x);\n    } else if (x instanceof List) {\n        handleList(x);\n    }\n  }\n  map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n  for (def x : list) {\n      if (x instanceof Map) {\n          handleMap(x);\n      } else if (x instanceof List) {\n          handleList(x);\n      }\n  }\n}\nhandleMap(ctx);\n"
       }
     },
-    {
-      "append": {
-        "tag": "append_preserve_original_event_on_error",
-        "field": "tags",
-        "value": "preserve_original_event",
-        "allow_duplicates": false,
-        "if": "ctx.error?.message != null"
-      }
-    },
     {
       "pipeline": {
         "name": "global@custom",
@@ -459,14 +405,7 @@
     {
       "append": {
         "field": "error.message",
-        "value": "Processor '{{{ _ingest.on_failure_processor_type }}}' {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}'"
-      }
-    },
-    {
-      "append": {
-        "field": "tags",
-        "value": "preserve_original_event",
-        "allow_duplicates": false
+        "value": "{{{ _ingest.on_failure_message }}}"
       }
     }
   ]

salt/elasticsearch/files/ingest/zeek.ja4d

@@ -1,71 +0,0 @@
-{
-    "description": "zeek.ja4d",
-    "processors": [
-        {
-            "set": {
-                "field": "event.dataset",
-                "value": "ja4d"
-            }
-        },
-        {
-            "remove": {
-                "field": [
-                    "host"
-                ],
-                "ignore_failure": true
-            }
-        },
-        {
-            "json": {
-                "field": "message",
-                "target_field": "message2",
-                "ignore_failure": true
-            }
-        },
-        {
-            "rename": {
-                "field": "message2.ja4d",
-                "target_field": "hash.ja4d",
-                "ignore_missing": true,
-                "if": "ctx?.message2?.ja4d != null && ctx.message2.ja4d.length() > 0"
-            }
-        },
-        {
-            "rename": {
-                "field": "message2.client_mac",
-                "target_field": "host.mac",
-                "ignore_missing": true,
-                "if": "ctx?.message2?.client_mac != null && ctx.message2.client_mac.length() > 0"
-            }
-        },
-        {
-            "rename": {
-                "field": "message2.hostname",
-                "target_field": "host.hostname",
-                "ignore_missing": true,
-                "if": "ctx?.message2?.hostname != null && ctx.message2.hostname.length() > 0"
-            }
-        },
-        {
-            "rename": {
-                "field": "message2.requested_ip",
-                "target_field": "dhcp.requested_address",
-                "ignore_missing": true,
-                "if": "ctx?.message2?.requested_ip != null && ctx.message2.requested_ip.length() > 0"
-            }
-        },
-        {
-            "rename": {
-                "field": "message2.vendor_class_id",
-                "target_field": "zeek.ja4d.vendor_class_id",
-                "ignore_missing": true,
-                "if": "ctx?.message2?.vendor_class_id != null && ctx.message2.vendor_class_id.length() > 0"
-            }
-        },
-        {
-            "pipeline": {
-                "name": "zeek.common"
-            }
-        }
-    ]
-}

salt/elasticsearch/files/log4j2.properties

@@ -45,7 +45,3 @@ appender.rolling_json.strategy.action.condition.nested_condition.age = 1D
 rootLogger.level = info
 rootLogger.appenderRef.rolling.ref = rolling
 rootLogger.appenderRef.rolling_json.ref = rolling_json
-
-# Suppress NotEntitledException WARNs (ES 9.3.3 bug)
-logger.entitlement_security.name = org.elasticsearch.entitlement.runtime.policy.PolicyManager.x-pack-security.org.elasticsearch.security.org.elasticsearch.xpack.security
-logger.entitlement_security.level = error

salt/elasticsearch/soc_elasticsearch.yaml

@@ -4,13 +4,6 @@ elasticsearch:
     forcedType: bool
     advanced: True
     helpLink: elasticsearch
-  data_retention_method:
-    description: Method for data retention. Options are ILM or DLM. For single node deployments and most distributed grid users, DLM will be the recommended option for simplified management. Those with more complex use cases may prefer ILM. The latter allows for more granular control, but requires more management overhead.
-    options:
-    - ILM
-    - DLM
-    forcedType: string
-    global: True
   version:
     description: "This specifies the version of the following containers: so-elastic-fleet-package-registry, so-elastic-agent, so-elastic-fleet, so-kibana, so-logstash and so-elasticsearch. Modifying this value in the Elasticsearch defaults.yaml will result in catastrophic grid failure."
     readonly: True
@@ -20,7 +13,7 @@ elasticsearch:
     description: Specify the memory heap size in (m)egabytes for Elasticsearch.
     helpLink: elasticsearch
   index_clean:
-    description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, data is retained by the configured lifecycle settings. This setting only applies to EVAL, STANDALONE, and HEAVY NODE installations. Other installations use lifecycle settings only.
+    description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, indices will only be deleted by the age defined in the ILM settings. This setting only applies to EVAL, STANDALONE, and HEAVY NODE installations. Other installations can only use ILM settings.
     forcedType: bool
     helpLink: elasticsearch
   vm:
@@ -146,23 +139,6 @@ elasticsearch:
     custom010: *pipelines
   index_settings:
     global_overrides:
-      data_stream_lifecycle:
-        data_retention:
-          description: |
-            The retention period for all data streams. Retention does not define the period that the data will be removed, but the minimum time period they will be kept.
-
-            Use a number followed by a time unit, such as 7d. Leave blank for indefinite retention where supported.
-
-            Configured retention period also affects the frequency of rolling over data streams.
-              - If retention is less than or equal to 1 day, max_age will be 1 hour
-              - If retention is less than or equal to 14 days, max_age will be 1 day
-              - If retention is less than or equal to 90 days, max_age will be 7 days
-              - If retention is greater than 90 days, max_age will be 30 days
-          forcedType: string
-          allowedNodeTypes:
-            - heavynode
-          regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$
-          regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d.
       index_template:
         template:
           settings:
@@ -335,30 +311,13 @@ elasticsearch:
               forcedType: string
               global: True
               helpLink: elasticsearch
-    so-logs: &dataStreamSettings
+    so-logs: &indexSettings
       index_sorting:
         description: Sorts the index by event time, at the cost of additional processing resource consumption.
         forcedType: bool
         global: True
         advanced: True
         helpLink: elasticsearch
-      data_stream_lifecycle:
-        data_retention:
-          description: |
-            The retention period for this data stream. Retention does not define the period that the data will be removed, but the minimum time period it will be kept.
-
-            Use a number followed by a time unit, such as 7d. Leave blank for indefinite retention where supported.
-
-            Configured retention period also affects the frequency of rolling over this data stream.
-              - If retention is less than or equal to 1 day, max_age will be 1 hour
-              - If retention is less than or equal to 14 days, max_age will be 1 day
-              - If retention is less than or equal to 90 days, max_age will be 7 days
-              - If retention is greater than 90 days, max_age will be 30 days
-          forcedType: string
-          allowedNodeTypes:
-            - heavynode
-          regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$
-          regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d.
       index_template:
         index_patterns:
           description: Patterns for matching multiple indices or tables.
@@ -376,14 +335,6 @@ elasticsearch:
                 global: True
                 advanced: True
                 helpLink: elasticsearch
-              auto_expand_replicas:
-                description: Automatically expand the number of replicas based on the number of data nodes in the cluster. This can help ensure high availability as the cluster scales up or down.
-                forcedType: string
-                regex: "^(0-[1-9]|1-[2-9]|2-[3-9]|3-[4-9]|4-[5-9]|5-[6-9]|6-[7-9]|7-[89]|8-9|[0-9]-all|false)$"
-                regexFailureMessage: Must be in the format of "x-y" where x is minimum number of replicas and y is maximum number of replicas, or "0-all" to specify a minimum of 0 and no maximum, or "false" to disable automatic replica expansion.
-                global: True
-                advanced: True
-                helpLink: elasticsearch
               mapping:
                 total_fields:
                   limit:
@@ -645,349 +596,65 @@ elasticsearch:
             global: True
             advanced: True
             helpLink: elasticsearch
-    so-logs-system_x_auth: *dataStreamSettings
-    so-logs-system_x_syslog: *dataStreamSettings
-    so-logs-system_x_system: *dataStreamSettings
-    so-logs-system_x_application: *dataStreamSettings
-    so-logs-system_x_security: *dataStreamSettings
-    so-logs-windows_x_forwarded: *dataStreamSettings
-    so-logs-windows_x_powershell: *dataStreamSettings
-    so-logs-windows_x_powershell_operational: *dataStreamSettings
-    so-logs-windows_x_sysmon_operational: *dataStreamSettings
-    so-logs-winlog_x_winlog: *dataStreamSettings
-    so-logs-detections_x_alerts: *dataStreamSettings
-    so-logs-http_endpoint_x_generic: *dataStreamSettings
-    so-logs-httpjson_x_generic: *dataStreamSettings
-    so-logs-osquery-manager-actions: *dataStreamSettings
-    so-logs-osquery-manager-action_x_responses: *dataStreamSettings
-    so-logs-osquery-manager_x_action_x_responses: *dataStreamSettings
-    so-logs-osquery-manager_x_result: *dataStreamSettings
-    so-logs-elastic_agent_x_apm_server: *dataStreamSettings
-    so-logs-elastic_agent_x_auditbeat: *dataStreamSettings
-    so-logs-elastic_agent_x_cloudbeat: *dataStreamSettings
-    so-logs-elastic_agent_x_endpoint_security: *dataStreamSettings
-    so-logs-endpoint_x_alerts: *dataStreamSettings
-    so-logs-endpoint_x_events_x_api: *dataStreamSettings
-    so-logs-endpoint_x_events_x_file: *dataStreamSettings
-    so-logs-endpoint_x_events_x_library: *dataStreamSettings
-    so-logs-endpoint_x_events_x_network: *dataStreamSettings
-    so-logs-endpoint_x_events_x_process: *dataStreamSettings
-    so-logs-endpoint_x_events_x_registry: *dataStreamSettings
-    so-logs-endpoint_x_events_x_security: *dataStreamSettings
-    so-logs-elastic_agent_x_filebeat: *dataStreamSettings
-    so-logs-elastic_agent_x_fleet_server: *dataStreamSettings
-    so-logs-elastic_agent_x_heartbeat: *dataStreamSettings
-    so-logs-elastic_agent: *dataStreamSettings
-    so-logs-elastic_agent_x_metricbeat: *dataStreamSettings
-    so-logs-elastic_agent_x_osquerybeat: *dataStreamSettings
-    so-logs-elastic_agent_x_packetbeat: *dataStreamSettings
-    so-logs-elasticsearch_x_server: *dataStreamSettings
-    so-metrics-endpoint_x_metadata: *dataStreamSettings
-    so-metrics-endpoint_x_metrics: *dataStreamSettings
-    so-metrics-endpoint_x_policy: *dataStreamSettings
-    so-metrics-nginx_x_stubstatus: *dataStreamSettings
-    so-metrics-vsphere_x_datastore: *dataStreamSettings
-    so-metrics-vsphere_x_host: *dataStreamSettings
-    so-metrics-vsphere_x_virtualmachine: *dataStreamSettings
-    so-common: *dataStreamSettings
-    so-endgame: *dataStreamSettings
-    so-idh: *dataStreamSettings
-    so-suricata: *dataStreamSettings
-    so-suricata_x_alerts: *dataStreamSettings
-    so-import: *dataStreamSettings
-    so-kratos: *dataStreamSettings
-    so-hydra: *dataStreamSettings
-    so-kismet: *dataStreamSettings
-    so-logstash: *dataStreamSettings
-    so-redis: *dataStreamSettings
-    so-strelka: *dataStreamSettings
-    so-syslog: *dataStreamSettings
-    so-zeek: *dataStreamSettings
-    # Managed SOC integration annotations are inserted below this line. Referencing '*dataStreamSettings'
-    so-case: &indexSettings
-      index_sorting:
-        description: Sorts the index by event time, at the cost of additional processing resource consumption.
-        forcedType: bool
-        global: True
-        advanced: True
-        helpLink: elasticsearch
-      index_template:
-        index_patterns:
-          description: Patterns for matching multiple indices or tables.
-          forcedType: "[]string"
-          multiline: True
-          global: True
-          advanced: True
-          helpLink: elasticsearch
-        template:
-          settings:
-            index:
-              number_of_replicas:
-                description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs.
-                forcedType: int
-                global: True
-                advanced: True
-                helpLink: elasticsearch
-              auto_expand_replicas:
-                description: Automatically expand the number of replicas based on the number of data nodes in the cluster. This can help ensure high availability as the cluster scales up or down.
-                forcedType: string
-                regex: "^(0-[1-9]|1-[2-9]|2-[3-9]|3-[4-9]|4-[5-9]|5-[6-9]|6-[7-9]|7-[89]|8-9|[0-9]-all|false)$"
-                regexFailureMessage: Must be in the format of "x-y" where x is minimum number of replicas and y is maximum number of replicas, or "0-all" to specify a minimum of 0 and no maximum, or "false" to disable automatic replica expansion.
-                global: True
-                advanced: True
-                helpLink: elasticsearch
-              mapping:
-                total_fields:
-                  limit:
-                    description: Max number of fields that can exist on a single index. Larger values will consume more resources.
-                    global: True
-                    advanced: True
-                    helpLink: elasticsearch
-              refresh_interval:
-                description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation.
-                global: True
-                advanced: True
-                helpLink: elasticsearch
-              number_of_shards:
-                description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs.
-                global: True
-                advanced: True
-                helpLink: elasticsearch
-              sort:
-                field:
-                  description: The field to sort by. Must set index_sorting to True.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-                order:
-                  description: The order to sort by. Must set index_sorting to True.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-          mappings:
-            _meta:
-              package:
-                name:
-                  description: Meta settings for the mapping.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-              managed_by:
-                  description: Meta settings for the mapping.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-              managed:
-                  description: Meta settings for the mapping.
-                  forcedType: bool
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-        composed_of:
-          description: The index template is composed of these component templates.
-          forcedType: "[]string"
-          global: True
-          advanced: True
-          helpLink: elasticsearch
-        priority:
-          description: The priority of the index template.
-          forcedType: int
-          global: True
-          advanced: True
-          helpLink: elasticsearch
-      policy:
-        phases:
-          hot:
-            min_age:
-              description: Minimum age of index. This determines when the index should be moved to the hot tier.
-              global: True
-              advanced: True
-              helpLink: elasticsearch
-            actions:
-              set_priority:
-                priority:
-                  description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
-                  forcedType: int
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-              rollover:
-                max_age:
-                  description: Maximum age of index.  Once an index reaches this limit, it will be rolled over into a new index.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-                max_primary_shard_size:
-                  description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-              shrink:
-                method:
-                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
-                  options:
-                  - COUNT
-                  - SIZE
-                  global: True
-                  advanced: True
-                  forcedType: string
-                number_of_shards:
-                  title: shard count
-                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
-                  global: True
-                  forcedType: int
-                  advanced: True
-                max_primary_shard_size:
-                  title: max shard size
-                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
-                  regex: ^[0-9]+(?:gb|tb|pb)$
-                  global: True
-                  forcedType: string
-                  advanced: True
-                allow_write_after_shrink:
-                  description: Allow writes after shrink.
-                  global: True
-                  forcedType: bool
-                  default: False
-                  advanced: True
-              forcemerge:
-                max_num_segments:
-                  description: Reduce the number of segments in each index shard and clean up deleted documents.
-                  global: True
-                  forcedType: int
-                  advanced: True
-                index_codec:
-                  title: compression
-                  description: Use higher compression for stored fields at the cost of slower performance.
-                  forcedType: bool
-                  global: True
-                  default: False
-                  advanced: True
-          warm:
-            min_age:
-              description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
-              regex: ^[0-9]{1,5}d$
-              forcedType: string
-              global: True
-              advanced: True
-              helpLink: elasticsearch
-            actions:
-              set_priority:
-                priority:
-                  description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
-                  forcedType: int
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-              rollover:
-                max_age:
-                  description: Maximum age of index.  Once an index reaches this limit, it will be rolled over into a new index.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-                max_primary_shard_size:
-                  description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-              shrink:
-                method:
-                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
-                  options:
-                  - COUNT
-                  - SIZE
-                  global: True
-                  advanced: True
-                number_of_shards:
-                  title: shard count
-                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
-                  global: True
-                  forcedType: int
-                  advanced: True
-                max_primary_shard_size:
-                  title: max shard size
-                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
-                  regex: ^[0-9]+(?:gb|tb|pb)$
-                  global: True
-                  forcedType: string
-                  advanced: True
-                allow_write_after_shrink:
-                  description: Allow writes after shrink.
-                  global: True
-                  forcedType: bool
-                  default: False
-                  advanced: True
-              forcemerge:
-                max_num_segments:
-                  description: Reduce the number of segments in each index shard and clean up deleted documents.
-                  global: True
-                  forcedType: int
-                  advanced: True
-                index_codec:
-                  title: compression
-                  description: Use higher compression for stored fields at the cost of slower performance.
-                  forcedType: bool
-                  global: True
-                  default: False
-                  advanced: True
-              allocate:
-                number_of_replicas:
-                  description: Set the number of replicas. Remains the same as the previous phase by default.
-                  forcedType: int
-                  global: True
-                  advanced: True
-          cold:
-            min_age:
-              description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
-              regex: ^[0-9]{1,5}d$
-              forcedType: string
-              global: True
-              advanced: True
-              helpLink: elasticsearch
-            actions:
-              set_priority:
-                priority:
-                  description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
-                  forcedType: int
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-              allocate:
-                number_of_replicas:
-                  description: Set the number of replicas. Remains the same as the previous phase by default.
-                  forcedType: int
-                  global: True
-                  advanced: True
-          delete:
-            min_age:
-              description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.
-              regex: ^[0-9]{1,5}d$
-              forcedType: string
-              global: True
-              advanced: True
-              helpLink: elasticsearch
-        _meta:
-          package:
-            name:
-              description: Meta settings for the mapping.
-              global: True
-              advanced: True
-              helpLink: elasticsearch
-          managed_by:
-            description: Meta settings for the mapping.
-            global: True
-            advanced: True
-            helpLink: elasticsearch
-          managed:
-            description: Meta settings for the mapping.
-            forcedType: bool
-            global: True
-            advanced: True
-            helpLink: elasticsearch
-    sos-backup: *indexSettings
-    so-detection: *indexSettings
-    so-assistant-chat: *indexSettings
-    so-assistant-session: *indexSettings
+    so-logs-system_x_auth: *indexSettings
+    so-logs-system_x_syslog: *indexSettings
+    so-logs-system_x_system: *indexSettings
+    so-logs-system_x_application: *indexSettings
+    so-logs-system_x_security: *indexSettings
+    so-logs-windows_x_forwarded: *indexSettings
+    so-logs-windows_x_powershell: *indexSettings
+    so-logs-windows_x_powershell_operational: *indexSettings
+    so-logs-windows_x_sysmon_operational: *indexSettings
+    so-logs-winlog_x_winlog: *indexSettings
+    so-logs-detections_x_alerts: *indexSettings
+    so-logs-http_endpoint_x_generic: *indexSettings
+    so-logs-httpjson_x_generic: *indexSettings
+    so-logs-osquery-manager-actions: *indexSettings
+    so-logs-osquery-manager-action_x_responses: *indexSettings
+    so-logs-osquery-manager_x_action_x_responses: *indexSettings
+    so-logs-osquery-manager_x_result: *indexSettings
+    so-logs-elastic_agent_x_apm_server: *indexSettings
+    so-logs-elastic_agent_x_auditbeat: *indexSettings
+    so-logs-elastic_agent_x_cloudbeat: *indexSettings
+    so-logs-elastic_agent_x_endpoint_security: *indexSettings
+    so-logs-endpoint_x_alerts: *indexSettings
+    so-logs-endpoint_x_events_x_api: *indexSettings
+    so-logs-endpoint_x_events_x_file: *indexSettings
+    so-logs-endpoint_x_events_x_library: *indexSettings
+    so-logs-endpoint_x_events_x_network: *indexSettings
+    so-logs-endpoint_x_events_x_process: *indexSettings
+    so-logs-endpoint_x_events_x_registry: *indexSettings
+    so-logs-endpoint_x_events_x_security: *indexSettings
+    so-logs-elastic_agent_x_filebeat: *indexSettings
+    so-logs-elastic_agent_x_fleet_server: *indexSettings
+    so-logs-elastic_agent_x_heartbeat: *indexSettings
+    so-logs-elastic_agent: *indexSettings
+    so-logs-elastic_agent_x_metricbeat: *indexSettings
+    so-logs-elastic_agent_x_osquerybeat: *indexSettings
+    so-logs-elastic_agent_x_packetbeat: *indexSettings
+    so-logs-elasticsearch_x_server: *indexSettings
+    so-metrics-endpoint_x_metadata: *indexSettings
+    so-metrics-endpoint_x_metrics: *indexSettings
+    so-metrics-endpoint_x_policy: *indexSettings
+    so-metrics-nginx_x_stubstatus: *indexSettings
+    so-metrics-vsphere_x_datastore: *indexSettings
+    so-metrics-vsphere_x_host: *indexSettings
+    so-metrics-vsphere_x_virtualmachine: *indexSettings
+    so-case: *indexSettings
+    so-common: *indexSettings
+    so-endgame: *indexSettings
+    so-idh: *indexSettings
+    so-suricata: *indexSettings
+    so-suricata_x_alerts: *indexSettings
+    so-import: *indexSettings
+    so-kratos: *indexSettings
+    so-hydra: *indexSettings
+    so-kismet: *indexSettings
+    so-logstash: *indexSettings
+    so-redis: *indexSettings
+    so-strelka: *indexSettings
+    so-syslog: *indexSettings
+    so-zeek: *indexSettings
     so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings
       index_sorting:
         description: Sorts the index by event time, at the cost of additional processing resource consumption.

salt/elasticsearch/template.map.jinja

@@ -4,11 +4,7 @@
    Elastic License 2.0. #}
 
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
-{# ELASTICSEARCHMERGED only used here to collect data_retention_method. This file intentionally works with ELASTICSEARCHDEFAULTS #}
-{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
-
 {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %}
-{% set DATA_RETENTION_METHOD = ELASTICSEARCHMERGED.data_retention_method %}
 
 {% set PILLAR_GLOBAL_OVERRIDES = {} %}
 {% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings', {}) %}
@@ -18,42 +14,15 @@
 
 {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %}
 
-{% set ALL_ADDON_INTEGRATION_DEFAULTS = {} %}
-{% set ALL_ADDON_SETTINGS_ORIG = {} %}
-{% set ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES = {} %}
-{% set ALL_ADDON_SETTINGS = {} %}
 {# start generation of integration default index_settings #}
-{% if salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %}
-{#   import integration type defaults #}
-{%   if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') %}
-{%     set check_integration_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %}
-{%     if check_integration_package_components.size > 1 %}
-{%       from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}
-{%       do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INTEGRATION_DEFAULTS) %}
-{%     endif %}
-{%   endif %}
-
-{#   import input type defaults #}
-{%   if salt['file.file_exists']('/opt/so/state/esfleet_input_package_components.json') %}
-{%     set check_input_package_components = salt['file.stats']('/opt/so/state/esfleet_input_package_components.json') %}
-{%     if check_input_package_components.size > 1 %}
-{%       from 'elasticfleet/input-defaults.map.jinja' import ADDON_INPUT_INTEGRATION_DEFAULTS %}
-{%       do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INPUT_INTEGRATION_DEFAULTS) %}
-{%     endif %}
-{%   endif %}
-
-{#   import content type defaults #}
-{%   if salt['file.file_exists']('/opt/so/state/esfleet_content_package_components.json') %}
-{%     set check_content_package_components = salt['file.stats']('/opt/so/state/esfleet_content_package_components.json') %}
-{%     if check_content_package_components.size > 1 %}
-{%       from 'elasticfleet/content-defaults.map.jinja' import ADDON_CONTENT_INTEGRATION_DEFAULTS %}
-{%       do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_CONTENT_INTEGRATION_DEFAULTS) %}
-{%     endif %}
-{%   endif %}
-
-{%   for index, settings in ALL_ADDON_INTEGRATION_DEFAULTS.items() %}
-{%     do ALL_ADDON_SETTINGS_ORIG.update({index: settings}) %}
-{%   endfor %}
+{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') and salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %}
+{%   set check_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %}
+{%   if check_package_components.size > 1 %}
+{%    from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}
+{%    for index, settings in ADDON_INTEGRATION_DEFAULTS.items() %}
+{%      do ES_INDEX_SETTINGS_ORIG.update({index: settings}) %}
+{%    endfor %}
+{%   endif%}
 {% endif %}
 {# end generation of integration default index_settings #}
 
@@ -62,43 +31,25 @@
 {%   do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}
 {% endfor %}
 
-{% if ALL_ADDON_SETTINGS_ORIG.keys() | length > 0 %}
-{%   for index in ALL_ADDON_SETTINGS_ORIG.keys() %}
-{%     do ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ALL_ADDON_SETTINGS_ORIG[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}
-{#     Explicitly excluding addon indices from ES_INDEX_SETTINGS_ORIG
-         When manager.soc_managed_annotations runs, new entries are added to the salt/elasticsearch/defaults.yaml file to support 'revert to default' functionality.
-         Subsequent map renders will then incorrectly include 'integration X' in 'ES_INDEX_SETTINGS_ORIG' due to being in the defaults.yaml file. #}
-{%     if index in ES_INDEX_SETTINGS_ORIG.keys() %}
-{%       do ES_INDEX_SETTINGS_ORIG.pop(index) %}
-{%     endif %}
-{%   endfor %}
-{% endif %}
-
 {% set ES_INDEX_SETTINGS = {} %}
-{% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS, EXCLUDE_INDICES=[]) %}
-
-{% do GLOBAL_OVERRIDES.update(salt['defaults.merge'](GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
-{% for index, settings in GLOBAL_OVERRIDES.items() %}
-
-{%   if index in EXCLUDE_INDICES %}
-{%     continue %}
-{%   endif %}
+{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
+{% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %}
 
 {#   prevent this action from being performed on custom defined indices. #}
 {#   the custom defined index is not present in either of the dictionaries and fails to reder. #}
-{%   if index in DEFINED_SETTINGS and index in GLOBAL_OVERRIDES %}
+{%   if index in ES_INDEX_SETTINGS_ORIG and index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES %}
 
 {#     dont merge policy from the global_overrides if policy isn't defined in the original index settingss #}
 {#     this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #}
-{%     if not DEFINED_SETTINGS[index].policy is defined and GLOBAL_OVERRIDES[index].policy is defined %}
-{%       do GLOBAL_OVERRIDES[index].pop('policy') %}
+{%     if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
+{%       do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %}
 {%     endif %}
 
 {#     this prevents and index from inderiting a policy phase from global overrides if it wasnt defined in the defaults. #}
-{%     if GLOBAL_OVERRIDES[index].policy is defined %}
-{%       for phase in GLOBAL_OVERRIDES[index].policy.phases.copy() %}
-{%         if DEFINED_SETTINGS[index].policy.phases[phase] is not defined %}
-{%           do GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %}
+{%     if ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %}
+{%       for phase in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.copy() %}
+{%         if ES_INDEX_SETTINGS_ORIG[index].policy.phases[phase] is not defined %}
+{%           do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %}
 {%         endif %}
 {%       endfor %}
 {%     endif %}
@@ -109,17 +60,6 @@
 {%     if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %}
 {%       do settings.index_template.template.settings.index.pop('sort') %}
 {%     endif %}
-{%     if DATA_RETENTION_METHOD == 'DLM' and settings.index_template.data_stream is defined and settings.data_stream_lifecycle is defined %}
-{%       if settings.data_stream_lifecycle.data_retention is defined and settings.data_stream_lifecycle.data_retention %}
-{%         do settings.index_template.template.update({'lifecycle': {'data_retention': settings.data_stream_lifecycle.data_retention}}) %}
-{%       else %}
-{%         do settings.index_template.template.update({'lifecycle': {}}) %}
-{%       endif %}
-{%       if settings.index_template.template.settings.index.lifecycle is not defined %}
-{%         do settings.index_template.template.settings.index.update({'lifecycle': {}}) %}
-{%       endif %}
-{%       do settings.index_template.template.settings.index.lifecycle.update({'prefer_ilm': false}) %}
-{%     endif %}
 {%   endif %}
 
 {# advanced ilm actions #}
@@ -171,23 +111,5 @@
 {%     endfor %}
 {%   endif %}
 
-{%   do FINAL_INDEX_SETTINGS.update({index | replace("_x_", "."): GLOBAL_OVERRIDES[index]}) %}
-{% endfor %}
-{% endmacro %}
-
-{# Exclude addon integrations from final ES_INDEX_SETTINGS #}
-{{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS, ALL_ADDON_SETTINGS_ORIG.keys() | list ) }}
-
-{# Exclude SO managed indices, otherwise ALL_ADDON_SETTINGS will include pillar values
-  of core integrations without merging defaults, resulting in an overlapping, but bad index template being generated. #}
-{{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS, ES_INDEX_SETTINGS_ORIG.keys() | list ) }}
-
-{% set SO_MANAGED_INDICES = [] %}
-{% for index, settings in ES_INDEX_SETTINGS.items() %}
-{%   do SO_MANAGED_INDICES.append(index) %}
-{% endfor %}
-
-{% set ADDON_INDICES = [] %}
-{% for index, settings in ALL_ADDON_SETTINGS.items() %}
-{%   do ADDON_INDICES.append(index) %}
+{%   do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %}
 {% endfor %}

salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list

@@ -6,19 +6,8 @@
 # Elastic License 2.0.
 
 . /usr/sbin/so-common
-
-if [[ -z "$1" ]]; then
-    if output=$(so-elasticsearch-query "_component_template" --retry 3 --retry-delay 1 --fail); then
-        jq '[.component_templates[] | .name] | sort' <<< "$output"
-    else
-        echo "Failed to retrieve component templates from Elasticsearch."
-        exit 1
-    fi
+if [ "$1" == "" ]; then
+    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template | jq '.component_templates[] |.name'| sort
 else
-    if output=$(so-elasticsearch-query "_component_template/$1" --retry 3 --retry-delay 1 --fail); then
-        jq <<< "$output"
-    else
-        echo "Failed to retrieve component template '$1' from Elasticsearch."
-        exit 1
-    fi
-fi
+    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template/$1 | jq
+fi

salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load

@@ -1,268 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-
-SO_STATEFILE_SUCCESS=/opt/so/state/estemplates.txt
-ADDON_STATEFILE_SUCCESS=/opt/so/state/addon_estemplates.txt
-ELASTICSEARCH_TEMPLATES_DIR="/opt/so/conf/elasticsearch/templates"
-SO_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/index"
-ADDON_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/addon-index"
-SO_LOAD_FAILURES=0
-ADDON_LOAD_FAILURES=0
-SO_LOAD_FAILURES_NAMES=()
-ADDON_LOAD_FAILURES_NAMES=()
-IS_HEAVYNODE="false"
-FORCE="false"
-VERBOSE="false"
-SHOULD_EXIT_ON_FAILURE="true"
-
-# If soup is running, ignore errors
-pgrep soup >/dev/null && SHOULD_EXIT_ON_FAILURE="false"
-
-while [[ $# -gt 0 ]]; do
-    case "$1" in
-    --heavynode)
-        IS_HEAVYNODE="true"
-        ;;
-    --force)
-        FORCE="true"
-        ;;
-    --verbose)
-        VERBOSE="true"
-        ;;
-    *)
-        echo "Usage: $0 [options]"
-        echo "Options:"
-        echo "  --heavynode     Only loads index templates specific to heavynodes"
-        echo "  --force     Force reload all templates regardless of statefiles (default: false)"
-        echo "  --verbose     Enable verbose output"
-        exit 1
-        ;;
-    esac
-    shift
-done
-
-load_template() {
-    local uri="$1"
-    local file="$2"
-
-    echo "Loading template file $file"
-    if ! output=$(retry 3 3 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"); then
-        echo "$output"
-
-        return 1
-
-    elif [[ "$VERBOSE" == "true" ]]; then
-        echo "$output"
-    fi
-
-}
-
-check_required_component_template_exists() {
-    local required
-    local missing
-    local file=$1
-
-    required=$(jq '[((.composed_of //[]) - (.ignore_missing_component_templates // []))[]]' "$file")
-    missing=$(jq -n --argjson required "$required" --argjson component_templates "$component_templates" '(($required) - ($component_templates))')
-
-    if [[ $(jq length <<<"$missing") -gt 0 ]]; then
-
-        return 1
-    fi
-}
-
-check_heavynode_compatiable_index_template() {
-    # The only templates that are relevant to heavynodes are from datasets defined in elasticagent/files/elastic-agent.yml.jinja.
-    # Heavynodes do not have fleet server packages installed and do not support elastic agents reporting directly to them.
-    local -A heavynode_index_templates=(
-        ["so-import"]=1
-        ["so-syslog"]=1
-        ["so-logs-soc"]=1
-        ["so-suricata"]=1
-        ["so-suricata.alerts"]=1
-        ["so-zeek"]=1
-        ["so-strelka"]=1
-    )
-
-    local template_name="$1"
-
-    if [[ ! -v heavynode_index_templates["$template_name"] ]]; then
-
-        return 1
-    fi
-
-}
-
-load_component_templates() {
-    local printed_name="$1"
-    local pattern="${ELASTICSEARCH_TEMPLATES_DIR}/component/$2"
-    local append_mappings="${3:-"false"}"
-
-    echo -e "\nLoading $printed_name component templates...\n"
-
-    if ! compgen -G "${pattern}/*.json" > /dev/null; then
-        echo "No $printed_name component templates found in ${pattern}, skipping."
-        return
-    fi
-
-    for component in "$pattern"/*.json; do
-        tmpl_name=$(basename "${component%.json}")
-
-        if [[ "$append_mappings" == "true" ]]; then
-            # avoid duplicating "-mappings" if it already exists in the component template filename
-            tmpl_name="${tmpl_name%-mappings}-mappings"
-        fi
-
-        if ! load_template "_component_template/${tmpl_name}" "$component"; then
-            SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
-            SO_LOAD_FAILURES_NAMES+=("$component")
-        fi
-    done
-}
-
-index_templates_exist() {
-    local templates_dir="$1"
-
-    if [[ ! -d "$templates_dir" ]]; then
-        return 1
-    fi
-
-    compgen -G "${templates_dir}/*.json" > /dev/null
-}
-
-should_load_addon_templates() {
-    if [[ "$IS_HEAVYNODE" == "true" ]]; then
-        return 1
-    fi
-
-    # Skip statefile checks when forcing template load
-    if [[ "$FORCE" != "true" ]]; then
-        if [[ ! -f "$SO_STATEFILE_SUCCESS" || -f "$ADDON_STATEFILE_SUCCESS" ]]; then
-            return 1
-        fi
-    fi
-
-    index_templates_exist "$ADDON_TEMPLATES_DIR"
-}
-
-if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]] && index_templates_exist "$SO_TEMPLATES_DIR"; then
-    check_elasticsearch_responsive
-
-    if [[ "$IS_HEAVYNODE" == "false" ]]; then
-        # TODO: Better way to check if fleet server is installed vs checking for Elastic Defend component template.
-        fleet_check="logs-endpoint.alerts@package"
-        if ! so-elasticsearch-query "_component_template/$fleet_check" --output /dev/null --retry 5 --retry-delay 3 --fail; then
-            # This check prevents so-elasticsearch-templates-load from running before so-elastic-fleet-setup has run.
-            echo -e "\nPackage $fleet_check not yet installed. Fleet Server may not be fully configured yet."
-            # Fleet Server is required because some SO index templates depend on components installed via
-            #  specific integrations eg Elastic Defend. These are components that we do not manually create / manage
-            #  via /opt/so/saltstack/salt/elasticsearch/templates/component/
-
-            exit 0
-        fi
-    fi
-
-    # load_component_templates "Name" "directory" "append '-mappings'?"
-    load_component_templates "ECS" "ecs" "true"
-    load_component_templates "Elastic Agent" "elastic-agent"
-    load_component_templates "Security Onion" "so"
-
-    component_templates=$(so-elasticsearch-component-templates-list)
-    echo -e "Loading Security Onion index templates...\n"
-    for so_idx_tmpl in "${SO_TEMPLATES_DIR}"/*.json; do
-        tmpl_name=$(basename "${so_idx_tmpl%-template.json}")
-
-        if [[ "$IS_HEAVYNODE" == "true" ]]; then
-            # TODO: Better way to load only heavynode specific templates
-            if ! check_heavynode_compatiable_index_template "$tmpl_name"; then
-                if [[ "$VERBOSE" == "true" ]]; then
-                    echo "Skipping over $so_idx_tmpl, template is not a heavynode specific index template."
-                fi
-
-                continue
-            fi
-        fi
-
-        if check_required_component_template_exists "$so_idx_tmpl"; then
-            if ! load_template "_index_template/$tmpl_name" "$so_idx_tmpl"; then
-                SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
-                SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl")
-            fi
-        else
-            echo "Skipping over $so_idx_tmpl due to missing required component template(s)."
-            SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
-            SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl")
-
-            continue
-        fi
-    done
-
-    if [[ $SO_LOAD_FAILURES -eq 0 ]]; then
-        echo "All Security Onion core templates loaded successfully."
-
-        touch "$SO_STATEFILE_SUCCESS"
-    else
-        echo "Encountered $SO_LOAD_FAILURES failure(s) loading templates:"
-        for failed_template in "${SO_LOAD_FAILURES_NAMES[@]}"; do
-            echo "  - $failed_template"
-        done
-        if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then
-            fail "Failed to load all Security Onion core templates successfully."
-        fi
-    fi
-elif ! index_templates_exist "$SO_TEMPLATES_DIR"; then
-    echo "No Security Onion core index templates found in ${SO_TEMPLATES_DIR}, skipping."
-elif [[ -f "$SO_STATEFILE_SUCCESS" ]]; then
-    echo "Security Onion core templates already loaded"
-fi
-
-# Start loading addon templates
-if should_load_addon_templates; then
-
-    check_elasticsearch_responsive
-
-    echo -e "\nLoading addon integration index templates...\n"
-    component_templates=$(so-elasticsearch-component-templates-list)
-
-    for addon_idx_tmpl in "${ADDON_TEMPLATES_DIR}"/*.json; do
-        tmpl_name=$(basename "${addon_idx_tmpl%-template.json}")
-
-        if check_required_component_template_exists "$addon_idx_tmpl"; then
-            if ! load_template "_index_template/${tmpl_name}" "$addon_idx_tmpl"; then
-                ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1))
-                ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl")
-            fi
-        else
-            echo "Skipping over $addon_idx_tmpl due to missing required component template(s)."
-            ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1))
-            ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl")
-
-            continue
-        fi
-    done
-
-    if [[ $ADDON_LOAD_FAILURES -eq 0 ]]; then
-        echo "All addon integration templates loaded successfully."
-
-        touch "$ADDON_STATEFILE_SUCCESS"
-    else
-        echo "Encountered $ADDON_LOAD_FAILURES failure(s) loading addon integration templates:"
-        for failed_template in "${ADDON_LOAD_FAILURES_NAMES[@]}"; do
-            echo "  - $failed_template"
-        done
-        if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then
-            fail "Failed to load all addon integration templates successfully."
-        fi
-    fi
-
-elif [[ ! -f "$SO_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" ]]; then
-    echo "Skipping loading addon integration templates until Security Onion core templates have been loaded."
-
-elif [[ -f "$ADDON_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" && "$FORCE" == "false" ]]; then
-    echo "Addon integration templates already loaded"
-fi

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply

@@ -1,175 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-
-{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
-{%- set DATA_RETENTION_METHOD = ELASTICSEARCHMERGED.data_retention_method %}
-
-ELASTICSEARCH_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR:-/opt/so/conf/elasticsearch/templates}"
-TEMPLATE_DIRS=(
-    "${ELASTICSEARCH_TEMPLATES_DIR}/index"
-    "${ELASTICSEARCH_TEMPLATES_DIR}/addon-index"
-)
-DATA_RETENTION_METHOD=$(cat <<'EOF'
-{{ DATA_RETENTION_METHOD }}
-EOF
-)
-DLM_FAILURES=0
-DLM_FAILURE_NAMES=()
-
-if [[ "$DATA_RETENTION_METHOD" != "DLM" && "$DATA_RETENTION_METHOD" != "ILM" ]]; then
-    echo "Unsupported data retention method $DATA_RETENTION_METHOD. Expected DLM or ILM."
-    exit 1
-fi
-
-validate_template_file() {
-    local template_file="$1"
-
-    if ! jq -e 'type == "object" and (.data_stream == null or (.data_stream | type == "object")) and (.template.lifecycle == null or (.template.lifecycle | type == "object")) and (.template.lifecycle.data_retention == null or (.template.lifecycle.data_retention | type == "string"))' >/dev/null 2>&1 "$template_file"; then
-        echo "Invalid index template JSON: $template_file"
-        return 1
-    fi
-}
-
-is_data_stream_template() {
-    jq -e '.data_stream | type == "object"' >/dev/null 2>&1 "$1"
-}
-
-has_data_stream_lifecycle() {
-    jq -e '.template.lifecycle | type == "object"' >/dev/null 2>&1 "$1"
-}
-
-get_data_retention() {
-    jq -r '.template.lifecycle.data_retention // ""' "$1"
-}
-
-find_template_file() {
-    local template="$1"
-    local template_dir
-    local template_file
-
-    for template_dir in "${TEMPLATE_DIRS[@]}"; do
-        template_file="${template_dir}/${template}-template.json"
-
-        if [[ -f "$template_file" ]]; then
-            echo "$template_file"
-            return 0
-        fi
-    done
-
-    return 1
-}
-
-set_data_stream_lifecycle() {
-    local data_stream="$1"
-    local data_retention="$2"
-    local body
-    local output
-
-    if [[ -n "$data_retention" ]]; then
-        if jq -e --arg data_stream "$data_stream" --arg data_retention "$data_retention" '.data_streams[]? | select(.name == $data_stream and .lifecycle.enabled == true and .lifecycle.data_retention == $data_retention)' >/dev/null 2>&1 <<< "$data_streams"; then
-            echo "DLM lifecycle already set for $data_stream with data_retention $data_retention, skipping."
-            return 0
-        fi
-    elif jq -e --arg data_stream "$data_stream" '.data_streams[]? | select(.name == $data_stream and .lifecycle.enabled == true and (.lifecycle.data_retention == null))' >/dev/null 2>&1 <<< "$data_streams"; then
-        echo "DLM lifecycle already set for $data_stream with indefinite retention, skipping."
-        return 0
-    fi
-
-    if [[ -n "$data_retention" ]]; then
-        body=$(jq -cn --arg data_retention "$data_retention" '{data_retention: $data_retention}')
-    else
-        # Setting indefinite retention
-        body='{}'
-    fi
-
-    if ! output=$(so-elasticsearch-query "_data_stream/${data_stream}/_lifecycle" -XPUT -d "$body" --retry 3 --retry-delay 5 --fail); then
-        echo "Failed to set data stream lifecycle for $data_stream."
-        return 1
-    fi
-
-    if [[ -n "$data_retention" ]]; then
-        echo "Set DLM lifecycle for $data_stream with data_retention $data_retention."
-    else
-        echo "Set DLM lifecycle for $data_stream with indefinite retention."
-    fi
-}
-
-disable_data_stream_lifecycle() {
-    local data_stream="$1"
-    local body='{"enabled":false}'
-    local output
-
-    if ! jq -e --arg data_stream "$data_stream" '.data_streams[]? | select(.name == $data_stream and .lifecycle != null and .lifecycle.enabled != false)' >/dev/null 2>&1 <<< "$data_streams"; then
-        # No action needed
-        return 0
-    fi
-
-    if ! output=$(so-elasticsearch-query "_data_stream/${data_stream}/_lifecycle" -XPUT -d "$body" --retry 3 --retry-delay 5 --fail); then
-        echo "Failed to disable data stream lifecycle for $data_stream."
-        return 1
-    fi
-
-    echo "Disabled DLM lifecycle for $data_stream."
-}
-
-process_data_stream() {
-    local data_stream="$1"
-    local data_retention="$2"
-
-    if [[ "$DATA_RETENTION_METHOD" == "DLM" ]]; then
-        set_data_stream_lifecycle "$data_stream" "$data_retention"
-    else
-        disable_data_stream_lifecycle "$data_stream"
-    fi
-}
-
-check_elasticsearch_responsive
-
-if ! data_streams=$(so-elasticsearch-query "_data_stream?format=json" --retry 3 --retry-delay 5 --fail); then
-    echo "Failed to retrieve data streams."
-    exit 1
-fi
-
-while read -r data_stream_config; do
-    data_stream=$(jq -r '.name' <<< "$data_stream_config")
-    template=$(jq -r '.template' <<< "$data_stream_config")
-
-    if ! template_file=$(find_template_file "$template"); then
-        echo "Skipping $data_stream: index template file not found for $template."
-        continue
-    fi
-
-    validate_template_file "$template_file" || exit 1
-
-    if ! is_data_stream_template "$template_file"; then
-        echo "Skipping $data_stream: $template_file is not a data stream template."
-        continue
-    fi
-
-    if [[ "$DATA_RETENTION_METHOD" == "DLM" ]] && ! has_data_stream_lifecycle "$template_file"; then
-        echo "Skipping $data_stream: $template_file does not define data stream lifecycle."
-        continue
-    fi
-
-    data_retention=$(get_data_retention "$template_file")
-
-    if ! process_data_stream "$data_stream" "$data_retention"; then
-        DLM_FAILURES=$((DLM_FAILURES + 1))
-        DLM_FAILURE_NAMES+=("$data_stream")
-    fi
-done < <(jq -c '.data_streams[]' <<< "$data_streams")
-
-if [[ $DLM_FAILURES -eq 0 ]]; then
-    echo "Data stream lifecycle updates completed successfully."
-else
-    echo "Encountered $DLM_FAILURES failure(s) updating data stream lifecycle:"
-    for failed_data_stream in "${DLM_FAILURE_NAMES[@]}"; do
-        echo "  - $failed_data_stream"
-    done
-    exit 1
-fi

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load

@@ -7,9 +7,6 @@
 . /usr/sbin/so-common
 
 {%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
-{%- if GLOBALS.role != "so-heavynode" %}
-{%-   from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
-{%- endif %}
 
 {%- for index, settings in ES_INDEX_SETTINGS.items() %}
 {%-   if settings.policy is defined %}
@@ -36,13 +33,3 @@
 {%-   endif %}
 {%- endfor %}
 echo
-{%- if GLOBALS.role != "so-heavynode" %}
-{%-   for index, settings in ALL_ADDON_SETTINGS.items() %}
-{%-     if settings.policy is defined %}
-  echo
-  echo "Setting up {{ index }}-logs policy..."
-  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
-  echo
-{%-     endif %}
-{%-   endfor %}
-{%- endif %}

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load

@@ -0,0 +1,165 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+{%  from 'vars/globals.map.jinja' import GLOBALS %}
+
+STATE_FILE_INITIAL=/opt/so/state/estemplates_initial_load_attempt.txt
+STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt
+
+if [[ -f $STATE_FILE_INITIAL ]]; then
+  # The initial template load has already run. As this is a subsequent load, all dependencies should 
+  # already be satisified. Therefore, immediately exit/abort this script upon any template load failure
+  # since this is an unrecoverable failure.
+  should_exit_on_failure=1
+else
+  # This is the initial template load, and there likely are some components not yet setup in Elasticsearch.
+  # Therefore load as many templates as possible at this time and if an error occurs proceed to the next 
+  # template. But if at least one template fails to load do not mark the templates as having been loaded.
+  # This will allow the next load to resume the load of the templates that failed to load initially.
+  should_exit_on_failure=0
+  echo "This is the initial template load"
+fi
+
+# If soup is running, ignore errors
+pgrep soup > /dev/null && should_exit_on_failure=0
+
+load_failures=0
+
+load_template() {
+  uri=$1
+  file=$2
+
+  echo "Loading template file $i" 
+  if ! retry 3 1 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"; then
+    if [[ $should_exit_on_failure -eq 1 ]]; then
+      fail "Could not load template file: $file"
+    else
+      load_failures=$((load_failures+1))
+      echo "Incremented load failure counter: $load_failures"
+    fi
+  fi
+}
+
+if [ ! -f $STATE_FILE_SUCCESS ]; then
+  echo "State file $STATE_FILE_SUCCESS not found. Running so-elasticsearch-templates-load."
+
+  . /usr/sbin/so-common
+
+  {% if GLOBALS.role != 'so-heavynode' %}
+  if [ -f /usr/sbin/so-elastic-fleet-common ]; then
+    . /usr/sbin/so-elastic-fleet-common
+  fi
+  {% endif %}
+
+  default_conf_dir=/opt/so/conf
+
+  # Define a default directory to load pipelines from
+  ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/"
+
+  {% if GLOBALS.role == 'so-heavynode' %}
+  file="/opt/so/conf/elasticsearch/templates/index/so-common-template.json"
+  {% else %}
+  file="/usr/sbin/so-elastic-fleet-common"
+  {% endif %}
+
+  if [ -f "$file" ]; then
+    # Wait for ElasticSearch to initialize
+    echo -n "Waiting for ElasticSearch..."
+    retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
+    {% if GLOBALS.role != 'so-heavynode' %}
+    TEMPLATE="logs-endpoint.alerts@package"
+    INSTALLED=$(so-elasticsearch-query _component_template/$TEMPLATE | jq -r .component_templates[0].name)
+    if [ "$INSTALLED" != "$TEMPLATE" ]; then
+      echo
+      echo "Packages not yet installed."
+      echo
+      exit 0
+    fi
+    {% endif %}
+
+    touch $STATE_FILE_INITIAL
+
+    cd ${ELASTICSEARCH_TEMPLATES}/component/ecs
+
+    echo "Loading ECS component templates..."
+    for i in *; do 
+      TEMPLATE=$(echo $i | cut -d '.' -f1) 
+      load_template "_component_template/${TEMPLATE}-mappings" "$i"
+    done
+    echo
+
+    cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent
+
+    echo "Loading Elastic Agent component templates..."
+    {% if GLOBALS.role == 'so-heavynode' %}
+    component_pattern="so-*"
+    {% else %}
+    component_pattern="*"
+    {% endif %}
+    for i in $component_pattern; do 
+      TEMPLATE=${i::-5} 
+      load_template "_component_template/$TEMPLATE" "$i"
+    done
+    echo
+	    
+    # Load SO-specific component templates
+    cd ${ELASTICSEARCH_TEMPLATES}/component/so
+
+    echo "Loading Security Onion component templates..."
+    for i in *; do 
+      TEMPLATE=$(echo $i | cut -d '.' -f1); 
+      load_template "_component_template/$TEMPLATE" "$i"
+    done
+    echo
+
+    # Load SO index templates
+    cd ${ELASTICSEARCH_TEMPLATES}/index
+
+    echo "Loading Security Onion index templates..."
+    shopt -s extglob
+    {% if GLOBALS.role == 'so-heavynode' %}
+    pattern="!(*1password*|*aws*|*azure*|*cloudflare*|*elastic_agent*|*fim*|*github*|*google*|*osquery*|*system*|*windows*|*endpoint*|*elasticsearch*|*generic*|*fleet_server*|*soc*)"
+    {% else %}
+    pattern="*"
+    {% endif %}
+    # Index templates will be skipped if the following conditions are met:
+    # 1. The template is part of the "so-logs-" template group
+    # 2. The template name does not correlate to at least one existing component template
+    # In this situation, the script will treat the skipped template as a temporary failure
+    # and allow the templates to be loaded again on the next run or highstate, whichever 
+    # comes first.
+    COMPONENT_LIST=$(so-elasticsearch-component-templates-list)
+    for i in $pattern; do
+      TEMPLATE=${i::-14}
+      COMPONENT_PATTERN=${TEMPLATE:3}
+      MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery")
+      if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ \.generic|logs-winlog\.winlog ]]; then
+        load_failures=$((load_failures+1))
+        echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures"
+      else
+        load_template "_index_template/$TEMPLATE" "$i"
+      fi
+    done
+  else
+    {% if GLOBALS.role == 'so-heavynode' %}
+    echo "Common template does not exist. Exiting..."
+    {% else %}
+    echo "Elastic Fleet not configured. Exiting..."
+    {% endif %}
+    exit 0
+  fi
+  
+  cd - >/dev/null
+
+  if [[ $load_failures -eq 0 ]]; then
+    echo "All templates loaded successfully"
+    touch $STATE_FILE_SUCCESS
+  else
+    echo "Encountered $load_failures templates that were unable to load, likely due to missing dependencies that will be available later; will retry on next highstate"
+  fi
+else
+  echo "Templates already loaded"
+fi

salt/firewall/containers.map.jinja

@@ -11,7 +11,6 @@
      'so-kratos',
      'so-hydra',
      'so-nginx',
-     'so-postgres',
      'so-redis',
      'so-soc',
      'so-strelka-coordinator',
@@ -35,7 +34,6 @@
      'so-hydra',
      'so-logstash',
      'so-nginx',
-     'so-postgres',
      'so-redis',
      'so-soc',
      'so-strelka-coordinator',
@@ -79,7 +77,6 @@
      'so-kratos',
      'so-hydra',
      'so-nginx',
-     'so-postgres',
      'so-soc'
 ] %}
 

salt/firewall/defaults.yaml

@@ -98,10 +98,6 @@ firewall:
       tcp:
         - 8086
       udp: []
-    postgres:
-      tcp:
-        - 5432
-      udp: []
     kafka_controller:
       tcp:
         - 9093
@@ -197,7 +193,6 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
-                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - localrules
@@ -384,7 +379,6 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
-                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - docker_registry
@@ -398,7 +392,6 @@ firewall:
                 - elasticsearch_rest
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -411,7 +404,6 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -429,7 +421,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
             searchnode:
               portgroups:
@@ -440,7 +431,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -454,7 +444,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -464,7 +453,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -498,7 +486,6 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - elastic_agent_control
@@ -509,7 +496,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -604,7 +590,6 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
-                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - docker_registry
@@ -618,7 +603,6 @@ firewall:
                 - elasticsearch_rest
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -631,7 +615,6 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -649,7 +632,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
             searchnode:
               portgroups:
@@ -660,7 +642,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -674,7 +655,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -684,7 +664,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -716,7 +695,6 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - elastic_agent_control
@@ -727,7 +705,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -822,7 +799,6 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
-                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - docker_registry
@@ -836,7 +812,6 @@ firewall:
                 - elasticsearch_rest
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -849,7 +824,6 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -867,7 +841,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
             searchnode:
               portgroups:
@@ -877,7 +850,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -890,7 +862,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -900,7 +871,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -934,7 +904,6 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - elastic_agent_control
@@ -945,7 +914,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -1043,7 +1011,6 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
-                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - docker_registry
@@ -1064,7 +1031,6 @@ firewall:
                 - elasticsearch_rest
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -1077,7 +1043,6 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -1089,7 +1054,6 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - beats_5044
@@ -1101,7 +1065,6 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - redis
@@ -1111,7 +1074,6 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - redis
@@ -1122,7 +1084,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -1159,7 +1120,6 @@ firewall:
               portgroups:
                 - docker_registry
                 - influxdb
-                - postgres
                 - sensoroni
                 - yum
                 - elastic_agent_control
@@ -1170,7 +1130,6 @@ firewall:
                 - yum
                 - docker_registry
                 - influxdb
-                - postgres
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
@@ -1514,7 +1473,6 @@ firewall:
                 - kibana
                 - redis
                 - influxdb
-                - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
                 - elastic_agent_control

salt/global/defaults.yaml

@@ -1,10 +1,3 @@
 global:
   pcapengine: SURICATA
-  pipeline: REDIS
-  push:
-    enabled: true
-    highstate_interval_hours: 2
-    debounce_seconds: 30
-    drain_interval: 15
-    batch: '25%'
-    batch_wait: 15
+  pipeline: REDIS

salt/global/soc_global.yaml

@@ -11,14 +11,18 @@ global:
     regexFailureMessage: You must enter a valid IP address or CIDR.
   mdengine:
     description: Which engine to use for meta data generation. Options are ZEEK and SURICATA.
+    regex: ^(ZEEK|SURICATA)$
     options:
       - ZEEK
       - SURICATA
+    regexFailureMessage: You must enter either ZEEK or SURICATA.
     global: True
   pcapengine:
     description: Which engine to use for generating pcap. Currently only SURICATA is supported.
+    regex: ^(SURICATA)$
     options:
       - SURICATA
+    regexFailureMessage: You must enter either SURICATA.
     global: True
   ids:
     description: Which IDS engine to use. Currently only Suricata is supported.
@@ -38,9 +42,11 @@ global:
     advanced: True
   pipeline:
     description: Sets which pipeline technology for events to use. The use of Kafka requires a Security Onion Pro license.
+    regex: ^(REDIS|KAFKA)$
     options:
       - REDIS
       - KAFKA
+    regexFailureMessage: You must enter either REDIS or KAFKA.
     global: True
     advanced: True
   repo_host:
@@ -59,41 +65,4 @@ global:
     description: Allows use of Endgame with Security Onion. This feature requires a license from Endgame.
     global: True
     advanced: True
-  push:
-    enabled:
-      description: Master kill-switch for the active push feature. When disabled, rule and pillar changes are picked up at the next scheduled highstate instead of being pushed immediately.
-      forcedType: bool
-      helpLink: push
-      global: True
-    highstate_interval_hours:
-      description: How often every minion in the grid runs a scheduled state.highstate, in hours. Lower values keep minions closer in sync at the cost of more load; higher values reduce load but increase worst-case latency for non-pushed changes. The salt-minion health check restarts a minion if its last highstate is older than this value plus one hour.
-      forcedType: int
-      helpLink: push
-      global: True
-      advanced: True
-    debounce_seconds:
-      description: Trailing-edge debounce window in seconds. A push intent must be quiet for this long before the drainer dispatches. Rapid bursts of edits within this window coalesce into one dispatch.
-      forcedType: int
-      helpLink: push
-      global: True
-      advanced: True
-    drain_interval:
-      description: How often the push drainer checks for ready intents, in seconds. Small values lower dispatch latency at the cost of more background work on the manager.
-      forcedType: int
-      helpLink: push
-      global: True
-      advanced: True
-    batch:
-      description: "Host batch size for push orchestrations. A number (e.g. '10') or a percentage (e.g. '25%'). Limits how many minions run the push state at once so large fleets don't thundering-herd."
-      helpLink: push
-      global: True
-      advanced: True
-      regex: '^([0-9]+%?)$'
-      regexFailureMessage: Enter a whole number or a whole-number percentage (e.g. 10 or 25%).
-    batch_wait:
-      description: Seconds to wait between host batches in a push orchestration. Gives the fleet time to breathe between waves.
-      forcedType: int
-      helpLink: push
-      global: True
-      advanced: True
 

salt/hydra/enabled.sls

@@ -58,7 +58,6 @@ so-hydra:
       - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
     {%   endfor %}
     {% endif %}
-    # Intentionally unless-stopped -- matches the fleet default.
     - restart_policy: unless-stopped
     - watch:
       - file: hydraconfig

salt/idh/enabled.sls

@@ -15,7 +15,6 @@ include:
 so-idh:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-idh:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - name: so-idh
     - detach: True
     - network_mode: host

salt/influxdb/enabled.sls

@@ -18,7 +18,6 @@ include:
 so-influxdb:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-influxdb:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: influxdb
     - networks:
       - sobridge:

salt/influxdb/soc_influxdb.yaml

@@ -85,10 +85,7 @@ influxdb:
       description: The log level to use for outputting log statements. Allowed values are debug, info, or error.
       global: True
       advanced: false
-      options:
-      - info
-      - debug
-      - error
+      regex: ^(info|debug|error)$
       helpLink: influxdb
     metrics-disabled:
       description: If true, the HTTP endpoint that exposes internal InfluxDB metrics will be inaccessible.
@@ -143,9 +140,7 @@ influxdb:
       description: Determines the type of storage used for secrets. Allowed values are bolt or vault.
       global: True
       advanced: True
-      options:
-      - bolt
-      - vault
+      regex: ^(bolt|vault)$
       helpLink: influxdb
     session-length:
       description: Number of minutes that a user login session can remain authenticated. 
@@ -265,9 +260,7 @@ influxdb:
       description: The type of data store to use for HTTP resources. Allowed values are disk or memory. Memory should not be used for production Security Onion installations.
       global: True
       advanced: True
-      options:
-      - disk
-      - memory
+      regex: ^(disk|memory)$
       helpLink: influxdb
     tls-cert: 
       description: The container path to the certificate to use for TLS encryption of the HTTP requests and responses.

salt/kafka/enabled.sls

@@ -27,13 +27,12 @@ include:
 so-kafka:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kafka:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: so-kafka
     - name: so-kafka
     - networks:
       - sobridge:
         - ipv4_address: {{ DOCKERMERGED.containers['so-kafka'].ip }}
-    - user: "960"
+    - user: kafka
     - environment:
         KAFKA_HEAP_OPTS: -Xmx2G -Xms1G
         KAFKA_OPTS: "-javaagent:/opt/jolokia/agents/jolokia-agent-jvm-javaagent.jar=port=8778,host={{ DOCKERMERGED.containers['so-kafka'].ip }},policyLocation=file:/opt/jolokia/jolokia.xml {%- if KAFKA_EXTERNAL_ACCESS %} -Djava.security.auth.login.config=/opt/kafka/config/kafka_server_jaas.conf {% endif -%}"

salt/kafka/soc_kafka.yaml

@@ -128,13 +128,10 @@ kafka:
         title: ssl.keystore.password
         sensitive: True
         helpLink: kafka
-      ssl_x_keystore_x_type:
+      ssl_x_keystore_x_type: 
         description: The key store file format.
         title: ssl.keystore.type
-        options:
-        - JKS
-        - PKCS12
-        - PEM
+        regex: ^(JKS|PKCS12|PEM)$
         helpLink: kafka
       ssl_x_truststore_x_location:
         description: The trust store file location within the Docker container.
@@ -163,11 +160,7 @@ kafka:
       security_x_protocol:
         description: 'Broker communication protocol. Options are: SASL_SSL, PLAINTEXT, SSL, SASL_PLAINTEXT'
         title: security.protocol
-        options:
-        - SASL_SSL
-        - PLAINTEXT
-        - SSL
-        - SASL_PLAINTEXT
+        regex: ^(SASL_SSL|PLAINTEXT|SSL|SASL_PLAINTEXT)
         helpLink: kafka
       ssl_x_keystore_x_location:
         description: The key store file location within the Docker container.
@@ -181,10 +174,7 @@ kafka:
       ssl_x_keystore_x_type:
         description: The key store file format.
         title: ssl.keystore.type
-        options:
-        - JKS
-        - PKCS12
-        - PEM
+        regex: ^(JKS|PKCS12|PEM)$
         helpLink: kafka
       ssl_x_truststore_x_location:
         description: The trust store file location within the Docker container.

salt/kibana/defaults.yaml

@@ -22,7 +22,7 @@ kibana:
           - default
           - file
     migrations:
-      discardCorruptObjects: "9.3.3"
+      discardCorruptObjects: "8.18.8"
     telemetry:
       enabled: False
     xpack:

salt/kibana/enabled.sls

@@ -6,7 +6,6 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
-{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 
 include:
@@ -17,9 +16,8 @@ include:
 so-kibana:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kibana:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: kibana
-    - user: "932:0"
+    - user: kibana
     - networks:
       - sobridge:
         - ipv4_address: {{ DOCKERMERGED.containers['so-kibana'].ip }}
@@ -62,19 +60,6 @@ so-kibana:
     - watch:
       - file: kibanaconfig
 
-wait_for_so-kibana:
-  http.wait_for_successful_query:
-    - name: "http://localhost:5601/api/status"
-    - username: 'so_elastic'
-    - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}'
-    - ssl: True
-    - verify_ssl: False
-    - status: 200
-    - wait_for: 300
-    - request_interval: 15
-    - require:
-      - docker_container: so-kibana
-
 delete_so-kibana_so-status.disabled:
   file.uncomment:
     - name: /opt/so/conf/so-status/so-status.conf

salt/kibana/tools/sbin_jinja/so-kibana-space-defaults

@@ -9,5 +9,5 @@ SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http:
 # Disable certain Features from showing up in the Kibana UI
 echo
 echo "Setting up default Kibana Space:"
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV1","entityManager","streams","cloudConnect","slo"]} ' >> /opt/so/log/kibana/misc.log
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","entityManager"]} ' >> /opt/so/log/kibana/misc.log
 echo

salt/kratos/enabled.sls

@@ -51,7 +51,6 @@ so-kratos:
       - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
     {%   endfor %}
     {% endif %}
-    # Intentionally unless-stopped -- matches the fleet default.
     - restart_policy: unless-stopped
     - watch:
       - file: kratosschema

salt/kratos/soc_kratos.yaml

@@ -3,8 +3,8 @@ kratos:
     description: Enables or disables the Kratos authentication system. WARNING - Disabling this process will cause the grid to malfunction. Re-enabling this setting will require manual effort via SSH.
     forcedType: bool
     advanced: True
-    readonly: True
     helpLink: kratos
+
   oidc:
     enabled:
       description: Set to True to enable OIDC / Single Sign-On (SSO) to SOC. Requires a valid Security Onion license key.
@@ -21,12 +21,8 @@ kratos:
         description: "Specify the provider type. Required. Valid values are: auth0, generic, github, google, microsoft"
         global: True
         forcedType: string
-        options:
-        - auth0
-        - generic
-        - github
-        - google
-        - microsoft
+        regex: "auth0|generic|github|google|microsoft"
+        regexFailureMessage: "Valid values are: auth0, generic, github, google, microsoft"
         helpLink: oidc
       client_id: 
         description: Specify the client ID, also referenced as the application ID. Required.
@@ -47,9 +43,8 @@ kratos:
         description: The source of the subject identifier. Typically 'userinfo'. Only used when provider is 'microsoft'.
         global: True
         forcedType: string
-        options:
-        - me
-        - userinfo
+        regex: me|userinfo
+        regexFailureMessage: "Valid values are: me, userinfo"
         helpLink: oidc
       auth_url: 
         description: Provider's auth URL. Required when provider is 'generic'.
@@ -103,7 +98,7 @@ kratos:
   config:
     session:
       lifespan: 
-        description: Defines the length of a login session before it will timeout, and require a new login.
+        description: Defines the length of a login session.
         global: True
         helpLink: kratos
       whoami:

salt/logstash/defaults.yaml

@@ -26,12 +26,12 @@ logstash:
     manager:
       - so/0011_input_endgame.conf
       - so/0012_input_elastic_agent.conf.jinja
-      - so/0013_input_lumberjack_fleet.conf.jinja
+      - so/0013_input_lumberjack_fleet.conf
       - so/9999_output_redis.conf.jinja
     receiver:
       - so/0011_input_endgame.conf
       - so/0012_input_elastic_agent.conf.jinja
-      - so/0013_input_lumberjack_fleet.conf.jinja
+      - so/0013_input_lumberjack_fleet.conf
       - so/9999_output_redis.conf.jinja
     search:
       - so/0900_input_redis.conf.jinja
@@ -69,5 +69,4 @@ logstash:
     pipeline_x_batch_x_size: 125
     pipeline_x_ecs_compatibility: disabled
   dmz_nodes: []
-  latency_metrics: False
 

salt/logstash/enabled.sls

@@ -28,13 +28,12 @@ include:
 so-logstash:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-logstash:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: so-logstash
     - name: so-logstash
     - networks:
       - sobridge:
         - ipv4_address: {{ DOCKERMERGED.containers['so-logstash'].ip }}
-    - user: "931:0"
+    - user: logstash
     - extra_hosts:
     {% for node in LOGSTASH_NODES %}
     {%   for hostname, ip in node.items() %}

salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf.jinja

@@ -1,4 +1,3 @@
-{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
 input {
   elastic_agent {
     port => 5055
@@ -12,15 +11,10 @@ input {
   }
 }
 filter {
-  {% if LOGSTASH_MERGED.get('latency_metrics', False) %}
-  ruby {
-    code => "event.set('[_tmp][logstash_from_agent]', Time.now().utc.iso8601(3));"
-  }
-  {% endif %}
-  if ![metadata] {
-    mutate {
-      rename => {"@metadata" => "metadata"}
-    }
+if ![metadata] {
+  mutate {
+    rename => {"@metadata" => "metadata"}
   }
 }
+}
 

salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf

@@ -0,0 +1,23 @@
+input {
+  elastic_agent {
+    port => 5056
+    tags => [ "elastic-agent", "fleet-lumberjack-input" ]
+    ssl_enabled => true
+    ssl_certificate => "/usr/share/logstash/elasticfleet-lumberjack.crt"
+    ssl_key => "/usr/share/logstash/elasticfleet-lumberjack.key"
+    ecs_compatibility => v8
+    id => "fleet-lumberjack-in"  
+    codec => "json"
+  }
+}
+
+
+filter {
+if ![metadata] {
+  mutate {
+    rename => {"@metadata" => "metadata"}
+  }
+}
+}
+
+

salt/logstash/pipelines/config/so/0013_input_lumberjack_fleet.conf.jinja

@@ -1,26 +0,0 @@
-{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
-input {
-  elastic_agent {
-    port => 5056
-    tags => [ "elastic-agent", "fleet-lumberjack-input" ]
-    ssl_enabled => true
-    ssl_certificate => "/usr/share/logstash/elasticfleet-lumberjack.crt"
-    ssl_key => "/usr/share/logstash/elasticfleet-lumberjack.key"
-    ecs_compatibility => v8
-    id => "fleet-lumberjack-in"
-    codec => "json"
-  }
-}
-
-filter {
-  {% if LOGSTASH_MERGED.get('latency_metrics', False) %}
-  ruby {
-    code => "event.set('[_tmp][logstash_from_fleet]', Time.now().utc.iso8601(3));"
-  }
-  {% endif %}
-  if ![metadata] {
-    mutate {
-      rename => {"@metadata" => "metadata"}
-    }
-  }
-}

salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja

@@ -1,4 +1,3 @@
-{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
 {%- set kafka_password = salt['pillar.get']('kafka:config:password') %}
 {%- set kafka_trustpass = salt['pillar.get']('kafka:config:trustpass') %}
 {%- set kafka_brokers = salt['pillar.get']('kafka:nodes', {}) %}
@@ -31,11 +30,6 @@ input {
     }
 }
 filter {
-  {% if LOGSTASH_MERGED.get('latency_metrics', False) %}
-  ruby {
-    code => "event.set('[_tmp][logstash_from_kafka]', Time.now().utc.iso8601(3));"
-  }
-  {% endif %}
   if ![metadata] {
     mutate {
       rename => { "@metadata" => "metadata" }

salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja

@@ -1,4 +1,4 @@
-{%- from 'logstash/map.jinja' import LOGSTASH_REDIS_NODES, LOGSTASH_MERGED %}
+{%- from 'logstash/map.jinja' import LOGSTASH_REDIS_NODES with context %}
 {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %}
 
 {%- for index in range(LOGSTASH_REDIS_NODES|length) %}
@@ -18,10 +18,3 @@ input {
 }
 {%   endfor %}
 {% endfor -%}
-filter {
-  {% if LOGSTASH_MERGED.get('latency_metrics', False) %}
-  ruby {
-    code => "event.set('[_tmp][logstash_from_redis]', Time.now().utc.iso8601(3));"
-  }
-  {% endif %}
-}

salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja

@@ -1,11 +1,3 @@
-{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
-{% if LOGSTASH_MERGED.get('latency_metrics', False) %}
-filter {
-  ruby {
-    code => "event.set('[_tmp][logstash_to_elasticsearch]', Time.now().utc.iso8601(3));"
-  }
-}
-{% endif %}
 output {
   if "elastic-agent" in [tags] and "so-ip-mappings" in [tags] {
     elasticsearch {

salt/logstash/pipelines/config/so/9806_output_lumberjack_fleet.conf.jinja

@@ -13,20 +13,13 @@ filter {
                     add_tag => "fleet-lumberjack-{{ GLOBALS.hostname }}"
           }
   }
-{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
-{% if LOGSTASH_MERGED.get('latency_metrics', False) %}
-filter {
-  ruby {
-    code => "event.set('[_tmp][fleet_to_logstash]', Time.now().utc.iso8601(3));"
-  }
-}
-{% endif %}
-output {
-    lumberjack {
-        codec => json
+
+output { 
+    lumberjack { 
+        codec => json 
         hosts => {{  FAILOVER_LOGSTASH_NODES }}
         ssl_certificate => "/usr/share/filebeat/ca.crt"
-        port => 5056
+        port => 5056 
         id => "fleet-lumberjack-{{ GLOBALS.hostname }}"
-        }
+        } 
     }

salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja

@@ -1,17 +1,10 @@
-{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
 {%- if grains.role in ['so-heavynode', 'so-receiver'] %}
   {%- set HOST = GLOBALS.hostname %}
 {%- else %}
   {%- set HOST = GLOBALS.manager %}
 {%- endif %}
 {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %}
-{% if LOGSTASH_MERGED.get('latency_metrics', False) %}
-filter {
-  ruby {
-    code => "event.set('[_tmp][logstash_to_redis]', Time.now().utc.iso8601(3));"
-  }
-}
-{% endif %}
+
 output {
 	redis {
 		host => '{{ HOST }}'

salt/logstash/soc_logstash.yaml

@@ -86,8 +86,3 @@ logstash:
     multiline: True
     advanced: True
     forcedType: "[]string"
-  latency_metrics:
-    description: Enable latency metrics within events processed by logstash. Useful for pinpointing log ingest delay.
-    forcedType: bool
-    global: False
-    advanced: True

salt/manager/beacons.sls

@@ -1,21 +0,0 @@
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-{% from 'global/map.jinja' import GLOBALMERGED %}
-
-include:
-  - salt.minion
-
-{% if GLOBALS.is_manager and GLOBALMERGED.push.enabled %}
-salt_beacons_pushstate:
-  file.managed:
-    - name: /etc/salt/minion.d/beacons_pushstate.conf
-    - source: salt://manager/files/beacons_pushstate.conf.jinja
-    - template: jinja
-    - watch_in:
-      - service: salt_minion_service
-{% else %}
-salt_beacons_pushstate:
-  file.absent:
-    - name: /etc/salt/minion.d/beacons_pushstate.conf
-    - watch_in:
-      - service: salt_minion_service
-{% endif %}

salt/manager/files/beacons_pushstate.conf.jinja

@@ -1,41 +0,0 @@
-{% from 'global/map.jinja' import GLOBALMERGED %}
-beacons:
-  pillar_db:
-    - interval: {{ GLOBALMERGED.push.drain_interval }}
-    - disable_during_state_run: True
-  inotify:
-    - disable_during_state_run: True
-    - coalesce: True
-    - files:
-        /opt/so/saltstack/local/salt/suricata/rules:
-          mask:
-            - close_write
-            - moved_to
-            - delete
-          recurse: True
-          auto_add: True
-          exclude:
-            - '\.sw[a-z]$':
-                regex: True
-            - '~$':
-                regex: True
-            - '/4913$':
-                regex: True
-            - '/\.#':
-                regex: True
-        /opt/so/saltstack/local/salt/strelka/rules/compiled:
-          mask:
-            - close_write
-            - moved_to
-            - delete
-          recurse: True
-          auto_add: True
-          exclude:
-            - '\.sw[a-z]$':
-                regex: True
-            - '~$':
-                regex: True
-            - '/4913$':
-                regex: True
-            - '/\.#':
-                regex: True