set default DLM retention 90d

salt/common/init.sls

@@ -130,17 +130,6 @@ common_sbin:
       - so-pcap-import
 {% endif %}
 
-# Pin physical NIC names by MAC (run-once) so a kernel upgrade can't renumber the
-# interfaces SO binds by name. The marker keeps it a one-time setup; an admin can
-# pre-create the marker to opt out.
-pin_nic_names:
-  cmd.run:
-    - name: /usr/sbin/so-nic-pin
-    - unless: 'test -e /opt/so/state/nic_names_pinned'
-    - require:
-      - file: common_sbin
-      - file: statedir
-
 common_sbin_jinja:
   file.recurse:
     - name: /usr/sbin

salt/common/tools/sbin/so-common

@@ -142,6 +142,11 @@ check_elastic_license() {
 	fi  
 }
 
+check_elasticsearch_responsive() {
+    retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" ||
+        fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot."
+}
+
 check_salt_master_status() {
 	local count=0
     local attempts="${1:- 10}"

salt/common/tools/sbin/so-nic-pin

@@ -1,76 +0,0 @@
-#!/bin/bash
-#
-# so-nic-pin — pin physical NIC names by permanent MAC via classic by-MAC udev
-#              rules, so a kernel upgrade can't renumber them.
-#
-# Security Onion binds its management and monitor interfaces BY NAME in pillar
-# (host:mainint, sensor:mainint, and bond0 is built on a specific physical NIC).
-# A kernel upgrade can change the kernel/systemd-udevd predictable-naming output
-# and renumber those NICs (e.g. enp1s0 -> enp2s0), which breaks the grid: the
-# pillar references a name that no longer exists and bond/bridge bring-up fails.
-#
-# This writes /etc/udev/rules.d/70-persistent-net.rules pinning each PHYSICAL NIC
-# to its CURRENT name by its PERMANENT MAC, freezing the names across future kernel
-# changes. It only writes the rules file; it does NOT live-trigger a rename (the
-# rules apply on the next boot/kernel, and a live rename would be disruptive).
-#
-# Run-once: gated by the drop file /opt/so/state/nic_names_pinned. If the marker is
-# present the script does nothing, so an admin can pre-create it to opt out. Invoked
-# from the common state on every highstate; the marker keeps it a one-time setup.
-
-NET_RULES_FILE="/etc/udev/rules.d/70-persistent-net.rules"
-MARKER="/opt/so/state/nic_names_pinned"
-
-log() { echo -e "[so-nic-pin] $*"; }
-
-# Echo "<name> <permanent-mac>" for every PHYSICAL NIC. A physical NIC is backed by a
-# real device (has device/driver), which excludes bond0/sobridge/docker0/veth*/lo whose
-# MACs are dynamic and must never be pinned. The PERMANENT MAC is used (ethtool -P, with
-# fallbacks), not the current one: an enslaved bond member's current MAC is rewritten to
-# the bond's, so matching on it would be wrong/ambiguous.
-physical_nics() {
-    local path n mac
-    for path in /sys/class/net/*; do
-        n="${path##*/}"
-        [ "$n" = "lo" ] && continue
-        [ -e "${path}/device/driver" ] || continue          # real device only
-        mac="$(ethtool -P "$n" 2>/dev/null | awk '/Permanent address/{print $NF}')"
-        case "$mac" in ""|00:00:00:00:00:00) mac="$(cat "${path}/bonding_slave/perm_hwaddr" 2>/dev/null)" ;; esac
-        case "$mac" in ""|00:00:00:00:00:00) mac="$(cat "${path}/address" 2>/dev/null)" ;; esac
-        case "$mac" in ""|00:00:00:00:00:00) continue ;; esac
-        echo "$n $mac"
-    done
-}
-
-# Turn "<name> <mac>" lines on stdin into classic by-MAC persistent-net udev rules.
-render_net_rules() {
-    echo "# Generated by so-nic-pin: pin NIC names by MAC so kernel upgrades can't renumber them."
-    echo "# Security Onion binds its management/monitor interfaces by name; do not hand-edit."
-    local n mac
-    while read -r n mac; do
-        [ -n "$n" ] || continue
-        printf 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="%s", NAME="%s"\n' \
-            "$mac" "$n"
-    done
-}
-
-[ "$(id -u)" -eq 0 ] || exit 0                   # salt runs us as root; bail quietly otherwise
-[ -e "${MARKER}" ] && exit 0                      # run-once guard (mirrors the state's unless)
-
-nics="$(physical_nics)"
-if [ -z "${nics}" ]; then
-    log "no physical NICs detected — nothing to pin (will retry on next highstate)"
-    exit 0                                         # do NOT drop the marker; let it retry later
-fi
-
-log "pinning physical NICs by permanent MAC:"
-echo "${nics}" | sed 's/^/    /'
-
-[ -f "${NET_RULES_FILE}" ] && cp -f "${NET_RULES_FILE}" "${NET_RULES_FILE}.bak"
-echo "${nics}" | render_net_rules > "${NET_RULES_FILE}" || {
-    log "ERROR: failed to write ${NET_RULES_FILE}"
-    exit 1
-}
-
-mkdir -p "$(dirname "${MARKER}")" && touch "${MARKER}"
-log "wrote ${NET_RULES_FILE} ($(grep -c '^SUBSYSTEM' "${NET_RULES_FILE}") NIC(s) pinned); dropped ${MARKER}"

salt/elasticfleet/content-defaults.map.jinja

@@ -9,7 +9,6 @@
 
 {% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
 {% set ADDON_CONTENT_INTEGRATION_DEFAULTS = {} %}
-{% set DEBUG_STUFF = {} %}
 
 {% for pkg in ADDON_CONTENT_PACKAGE_COMPONENTS %}
 {%   if pkg.name in CORE_ESFLEET_PACKAGES %}

salt/elasticfleet/enabled.sls

@@ -101,17 +101,6 @@ so-elastic-fleet:
       - file: trusttheca
       - x509: etc_elasticfleet_key
       - x509: etc_elasticfleet_crt
-
-wait_for_so-elastic-fleet:
-  http.wait_for_successful_query:
-    - name: "https://localhost:8220/api/status"
-    - ssl: True
-    - verify_ssl: False
-    - status: 200
-    - wait_for: 300
-    - request_interval: 15
-    - require:
-      - docker_container: so-elastic-fleet
 {%   endif %}
 
 delete_so-elastic-fleet_so-status.disabled:

salt/elasticfleet/input-defaults.map.jinja

@@ -9,7 +9,6 @@
 
 {% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
 {% set ADDON_INPUT_INTEGRATION_DEFAULTS = {} %}
-{% set DEBUG_STUFF = {} %}
 
 {% for pkg in ADDON_INPUT_PACKAGE_COMPONENTS %}
 {%   if pkg.name in CORE_ESFLEET_PACKAGES %}
@@ -116,7 +115,6 @@
 
 
 {%         do ADDON_INPUT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
-{%         do DEBUG_STUFF.update({integration_key: "Generating defaults for "+ pkg.name })%}
 {%       endfor %}
 {%     endif %}
 {%   endif %}

salt/elasticfleet/manager.sls

@@ -9,7 +9,6 @@
 
 include:
   - elasticfleet.config
-  - kibana.enabled
 
 # If enabled, automatically update Fleet Logstash Outputs
 {% if ELASTICFLEETMERGED.config.server.enable_auto_configuration %}
@@ -20,8 +19,6 @@ so-elastic-fleet-auto-configure-logstash-outputs:
     - retry:
         attempts: 4
         interval: 30
-    - require:
-      - http: wait_for_so-kibana
 {%   endif %}
 
 # If enabled, automatically update Fleet Server URLs & ES Connection
@@ -31,8 +28,6 @@ so-elastic-fleet-auto-configure-server-urls:
     - retry:
         attempts: 4
         interval: 30
-    - require:
-      - http: wait_for_so-kibana
 {% endif %}
 
 # Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
@@ -42,8 +37,6 @@ so-elastic-fleet-auto-configure-elasticsearch-urls:
     - retry:
         attempts: 4
         interval: 30
-    - require:
-      - http: wait_for_so-kibana
 
 so-elastic-fleet-auto-configure-artifact-urls:
   cmd.run:
@@ -51,8 +44,6 @@ so-elastic-fleet-auto-configure-artifact-urls:
     - retry:
         attempts: 4
         interval: 30
-    - require:
-      - http: wait_for_so-kibana
 
 so-elastic-fleet-package-statefile:
   file.managed:
@@ -64,9 +55,7 @@ so-elastic-fleet-package-upgrade:
     - name: /usr/sbin/so-elastic-fleet-package-upgrade
     - retry:
         attempts: 3
-        interval: 30
-    - require:
-      - http: wait_for_so-kibana
+        interval: 10
     - onchanges:
       - file: /opt/so/state/elastic_fleet_packages.txt
 
@@ -76,8 +65,6 @@ so-elastic-fleet-integrations:
     - retry:
         attempts: 3
         interval: 10
-    - require:
-      - http: wait_for_so-kibana
 
 so-elastic-agent-grid-upgrade:
   cmd.run:
@@ -85,8 +72,6 @@ so-elastic-agent-grid-upgrade:
     - retry:
         attempts: 12
         interval: 5
-    - require:
-      - http: wait_for_so-kibana
 
 so-elastic-fleet-integration-upgrade:
   cmd.run:
@@ -94,22 +79,16 @@ so-elastic-fleet-integration-upgrade:
     - retry:
         attempts: 3
         interval: 10
-    - require:
-      - http: wait_for_so-kibana
 
 {# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
 so-elastic-fleet-addon-integrations:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
-    - require:
-      - http: wait_for_so-kibana
 
 {% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
 so-elastic-defend-manage-filters-file-watch:
   cmd.run:
     - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log
-    - require:
-      - http: wait_for_so-kibana
     - onchanges:
       - file: elasticdefendcustom
       - file: elasticdefenddisabled

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -108,12 +108,9 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
   done
 
   # Only create the state file if all policies were created/updated successfully
-  if [[ $RETURN_CODE -eq 0 ]]; then
+  if [[ "$RETURN_CODE" != "1" ]]; then
     touch /opt/so/state/eaintegrations.txt
-  else
-    exit 1
   fi
 else
-  echo "Fleet integration policies already loaded."
-  exit 0
+  exit $RETURN_CODE
 fi

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade

@@ -8,33 +8,18 @@
 
 . /usr/sbin/so-elastic-fleet-common
 
-PKG_LOAD_FAILURES=0
-PKG_LOAD_FAILURES_NAMES=()
-
 {%- for PACKAGE in SUPPORTED_PACKAGES %}
 echo "Upgrading {{ PACKAGE }} package..."
 if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
     if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
-        PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
-        PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
+        # exit 1 on failure to upgrade a default package, allow salt to handle retries
+        echo -e "\nERROR: Failed to upgrade $PACKAGE to version: $VERSION"
+        exit 1
     fi
 else
-    PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
-    PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
+    echo -e "\nERROR: Failed to get version information for integration $PACKAGE"
 fi
 echo
 {%- endfor %}
-
-if [ $PKG_LOAD_FAILURES -gt 0 ]; then
-    echo "ERROR: Failed to upgrade $PKG_LOAD_FAILURES package(s):"
-    for PKG in "${PKG_LOAD_FAILURES_NAMES[@]}"; do
-        echo " - $PKG"
-    done
-    # exit 1 on failure to upgrade a default package, allow salt to handle retries
-    exit 1
-else
-    echo "Successfully upgraded all packages."
-fi
-
 echo
 /usr/sbin/so-elasticsearch-templates-load

salt/elasticsearch/cluster.sls

@@ -133,6 +133,18 @@ so-elasticsearch-templates:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
 
+so-elasticsearch-dlm-apply:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-dlm-apply
+    - cwd: /opt/so
+    - require:
+      - docker_container: so-elasticsearch
+      - file: elasticsearch_sbin_jinja
+      - cmd: so-elasticsearch-templates
+    - retry:
+        attempts: 3
+        interval: 10
+
 so-elasticsearch-pipelines:
   cmd.run:
     - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }}

salt/elasticsearch/defaults.yaml

@@ -2,6 +2,7 @@ elasticsearch:
   enabled: false
   version: 9.3.3
   index_clean: true
+  data_retention_method: DLM
   vm:
     max_map_count: 1048576
   config:
@@ -18,9 +19,18 @@ elasticsearch:
               flood_stage: 90%
               high: 85%
               low: 80%
+    # don't want to set retention here since it will make ES restart with every update +
+    #   potentially case where we could unintentially fall back to retention 7d and cause data loss
+    # data_streams:
+    #   lifecycle:
+    #     retention:
+    #       default: 7d
     indices:
       id_field_data:
         enabled: false
+    # index:
+      # lifecycle:
+        # prefer_ilm: true
     logger:
       org:
         elasticsearch:
@@ -63,6 +73,9 @@ elasticsearch:
             verification_mode: none
   index_settings:
     global_overrides:
+    # Tie this into cluster setting for data_streams.lifecycle.retention.default
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         template:
           settings:
@@ -143,6 +156,8 @@ elasticsearch:
                 order: desc
     so-common:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -304,6 +319,8 @@ elasticsearch:
               number_of_shards: 1
     so-assistant-chat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: ""
       index_template:
         composed_of:
         - assistant-chat-mappings
@@ -344,6 +361,8 @@ elasticsearch:
             min_age: 0ms
     so-assistant-session:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: ""
       index_template:
         composed_of:
         - assistant-session-mappings
@@ -497,6 +516,8 @@ elasticsearch:
             min_age: 30d
     so-idh:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -605,6 +626,8 @@ elasticsearch:
             min_age: 30d
     so-import:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -787,6 +810,8 @@ elasticsearch:
             min_age: 0ms
     so-kismet:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - kismet-mappings
@@ -836,6 +861,8 @@ elasticsearch:
             min_age: 30d
     so-kratos:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -904,6 +931,8 @@ elasticsearch:
             min_age: 30d
     so-hydra:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -1049,6 +1078,8 @@ elasticsearch:
             min_age: 0ms
     so-logs:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - so-data-streams-mappings
@@ -1129,6 +1160,8 @@ elasticsearch:
             min_age: 30d
     so-logs-detections_x_alerts:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - so-data-streams-mappings
@@ -1192,6 +1225,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1307,6 +1342,8 @@ elasticsearch:
             min_age: 30d
     so-elastic-agent-monitor:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1369,6 +1406,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_apm_server:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.apm_server@package
@@ -1433,6 +1472,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_auditbeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.auditbeat@package
@@ -1497,6 +1538,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_cloudbeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.cloudbeat@package
@@ -1561,6 +1604,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_endpoint_security:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1620,6 +1665,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_filebeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1679,6 +1726,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_fleet_server:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1735,6 +1784,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_heartbeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.heartbeat@package
@@ -1799,6 +1850,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_metricbeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1858,6 +1911,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_osquerybeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1917,6 +1972,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_packetbeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.packetbeat@package
@@ -1981,6 +2038,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elasticsearch_x_server:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-elasticsearch.server@package
@@ -2045,6 +2104,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_actions:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - .logs-endpoint.actions@package
@@ -2104,6 +2165,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_action_x_responses:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - .logs-endpoint.action.responses@package
@@ -2163,6 +2226,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_alerts:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.alerts@package
@@ -2222,6 +2287,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_diagnostic_x_collection:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - .logs-endpoint.diagnostic.collection@package
@@ -2297,6 +2364,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_api:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.api@package
@@ -2356,6 +2425,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_file:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.file@package
@@ -2415,6 +2486,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_library:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.library@package
@@ -2474,6 +2547,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_network:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.network@package
@@ -2533,6 +2608,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_process:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.process@package
@@ -2592,6 +2669,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_registry:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.registry@package
@@ -2651,6 +2730,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_security:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.security@package
@@ -2710,6 +2791,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_heartbeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - .logs-endpoint.heartbeat@package
@@ -2769,6 +2852,8 @@ elasticsearch:
             min_age: 30d
     so-logs-http_endpoint_x_generic:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-http_endpoint.generic@package
@@ -2817,6 +2902,8 @@ elasticsearch:
             min_age: 30d
     so-logs-httpjson_x_generic:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-httpjson.generic@package
@@ -2882,6 +2969,8 @@ elasticsearch:
               number_of_replicas: 0
     so-logs-osquery-manager_x_action_x_responses:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         _meta:
           managed: true
@@ -2953,6 +3042,8 @@ elasticsearch:
               number_of_replicas: 0
     so-logs-osquery-manager_x_result:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         _meta:
           managed: true
@@ -3005,6 +3096,8 @@ elasticsearch:
             min_age: 30d
     so-logs-soc:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -3113,6 +3206,8 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_application:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3162,6 +3257,8 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_auth:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3211,6 +3308,8 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_security:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3260,6 +3359,8 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_syslog:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3309,6 +3410,8 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_system:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3358,6 +3461,8 @@ elasticsearch:
             min_age: 30d
     so-logs-windows_x_forwarded:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-windows.forwarded@package
@@ -3405,6 +3510,8 @@ elasticsearch:
             min_age: 30d
     so-logs-windows_x_powershell:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-windows.powershell@package
@@ -3452,6 +3559,8 @@ elasticsearch:
             min_age: 30d
     so-logs-windows_x_powershell_operational:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-windows.powershell_operational@package
@@ -3499,6 +3608,8 @@ elasticsearch:
             min_age: 30d
     so-logs-windows_x_sysmon_operational:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-windows.sysmon_operational@package
@@ -3546,6 +3657,8 @@ elasticsearch:
             min_age: 30d
     so-logs-winlog_x_winlog:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-winlog.winlog@package
@@ -3594,6 +3707,8 @@ elasticsearch:
             min_age: 30d
     so-logstash:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -3709,6 +3824,8 @@ elasticsearch:
             min_age: 30d
     so-metrics-endpoint_x_metadata:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - metrics-endpoint.metadata@package
@@ -3756,6 +3873,8 @@ elasticsearch:
             min_age: 30d
     so-metrics-endpoint_x_metrics:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - metrics-endpoint.metrics@package
@@ -3803,6 +3922,8 @@ elasticsearch:
             min_age: 30d
     so-metrics-endpoint_x_policy:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - metrics-endpoint.policy@package
@@ -3850,6 +3971,8 @@ elasticsearch:
             min_age: 30d
     so-metrics-fleet_server_x_agent_status:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - metrics@tsdb-settings
@@ -3874,6 +3997,8 @@ elasticsearch:
               number_of_replicas: 0
     so-metrics-fleet_server_x_agent_versions:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - metrics@tsdb-settings
@@ -3898,6 +4023,8 @@ elasticsearch:
               number_of_replicas: 0
     so-redis:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -3958,13 +4085,10 @@ elasticsearch:
         - vulnerability-mappings
         - common-settings
         - common-dynamic-mappings
-        - logs-redis.log@package
-        - logs-redis.log@custom
         data_stream:
           allow_custom_routing: false
           hidden: false
-        ignore_missing_component_templates:
-        - logs-redis.log@custom
+        ignore_missing_component_templates: []
         index_patterns:
         - logs-redis.log*
         priority: 501
@@ -4016,6 +4140,8 @@ elasticsearch:
             min_age: 30d
     so-strelka:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4133,6 +4259,8 @@ elasticsearch:
             min_age: 30d
     so-suricata:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4249,6 +4377,8 @@ elasticsearch:
             min_age: 30d
     so-suricata_x_alerts:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4365,6 +4495,8 @@ elasticsearch:
             min_age: 30d
     so-syslog:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4481,6 +4613,8 @@ elasticsearch:
             min_age: 30d
     so-zeek:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings

salt/elasticsearch/soc_elasticsearch.yaml

@@ -4,6 +4,13 @@ elasticsearch:
     forcedType: bool
     advanced: True
     helpLink: elasticsearch
+  data_retention_method:
+    description: Method for data retention. Options are ILM or DLM. For single node deployments and most distributed grid users, DLM will be the recommended option for simplified management. Those with more complex use cases may prefer ILM. The latter allows for more granular control, but requires more management overhead.
+    options:
+    - ILM
+    - DLM
+    forcedType: string
+    global: True
   version:
     description: "This specifies the version of the following containers: so-elastic-fleet-package-registry, so-elastic-agent, so-elastic-fleet, so-kibana, so-logstash and so-elasticsearch. Modifying this value in the Elasticsearch defaults.yaml will result in catastrophic grid failure."
     readonly: True
@@ -13,7 +20,7 @@ elasticsearch:
     description: Specify the memory heap size in (m)egabytes for Elasticsearch.
     helpLink: elasticsearch
   index_clean:
-    description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, indices will only be deleted by the age defined in the ILM settings. This setting only applies to EVAL, STANDALONE, and HEAVY NODE installations. Other installations can only use ILM settings.
+    description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, data is retained by the configured lifecycle settings. This setting only applies to EVAL, STANDALONE, and HEAVY NODE installations. Other installations use lifecycle settings only.
     forcedType: bool
     helpLink: elasticsearch
   vm:
@@ -139,6 +146,21 @@ elasticsearch:
     custom010: *pipelines
   index_settings:
     global_overrides:
+      data_stream_lifecycle:
+        data_retention:
+          description: |
+            The retention period for all data streams. Retention does not define the period that the data will be removed, but the minimum time period they will be kept.
+
+            Use a number followed by a time unit, such as 7d. Leave blank for indefinite retention where supported.
+
+            Configured retention period also affects the frequency of rolling over data streams.
+              - If retention is less than or equal to 1 day, max_age will be 1 hour
+              - If retention is less than or equal to 14 days, max_age will be 1 day
+              - If retention is less than or equal to 90 days, max_age will be 7 days
+              - If retention is greater than 90 days, max_age will be 30 days
+          forcedType: string
+          regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$
+          regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d.
       index_template:
         template:
           settings:
@@ -311,13 +333,28 @@ elasticsearch:
               forcedType: string
               global: True
               helpLink: elasticsearch
-    so-logs: &indexSettings
+    so-logs: &dataStreamSettings
       index_sorting:
         description: Sorts the index by event time, at the cost of additional processing resource consumption.
         forcedType: bool
         global: True
         advanced: True
         helpLink: elasticsearch
+      data_stream_lifecycle:
+        data_retention:
+          description: |
+            The retention period for this data stream. Retention does not define the period that the data will be removed, but the minimum time period it will be kept.
+
+            Use a number followed by a time unit, such as 7d. Leave blank for indefinite retention where supported.
+
+            Configured retention period also affects the frequency of rolling over this data stream.
+              - If retention is less than or equal to 1 day, max_age will be 1 hour
+              - If retention is less than or equal to 14 days, max_age will be 1 day
+              - If retention is less than or equal to 90 days, max_age will be 7 days
+              - If retention is greater than 90 days, max_age will be 30 days
+          forcedType: string
+          regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$
+          regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d.
       index_template:
         index_patterns:
           description: Patterns for matching multiple indices or tables.
@@ -335,6 +372,14 @@ elasticsearch:
                 global: True
                 advanced: True
                 helpLink: elasticsearch
+              auto_expand_replicas:
+                description: Automatically expand the number of replicas based on the number of data nodes in the cluster. This can help ensure high availability as the cluster scales up or down.
+                forcedType: string
+                regex: "^(0-[1-9]|1-[2-9]|2-[3-9]|3-[4-9]|4-[5-9]|5-[6-9]|6-[7-9]|7-[89]|8-9|[0-9]-all|false)$"
+                regexFailureMessage: Must be in the format of "x-y" where x is minimum number of replicas and y is maximum number of replicas, or "0-all" to specify a minimum of 0 and no maximum, or "false" to disable automatic replica expansion.
+                global: True
+                advanced: True
+                helpLink: elasticsearch
               mapping:
                 total_fields:
                   limit:
@@ -596,65 +641,349 @@ elasticsearch:
             global: True
             advanced: True
             helpLink: elasticsearch
-    so-logs-system_x_auth: *indexSettings
-    so-logs-system_x_syslog: *indexSettings
-    so-logs-system_x_system: *indexSettings
-    so-logs-system_x_application: *indexSettings
-    so-logs-system_x_security: *indexSettings
-    so-logs-windows_x_forwarded: *indexSettings
-    so-logs-windows_x_powershell: *indexSettings
-    so-logs-windows_x_powershell_operational: *indexSettings
-    so-logs-windows_x_sysmon_operational: *indexSettings
-    so-logs-winlog_x_winlog: *indexSettings
-    so-logs-detections_x_alerts: *indexSettings
-    so-logs-http_endpoint_x_generic: *indexSettings
-    so-logs-httpjson_x_generic: *indexSettings
-    so-logs-osquery-manager-actions: *indexSettings
-    so-logs-osquery-manager-action_x_responses: *indexSettings
-    so-logs-osquery-manager_x_action_x_responses: *indexSettings
-    so-logs-osquery-manager_x_result: *indexSettings
-    so-logs-elastic_agent_x_apm_server: *indexSettings
-    so-logs-elastic_agent_x_auditbeat: *indexSettings
-    so-logs-elastic_agent_x_cloudbeat: *indexSettings
-    so-logs-elastic_agent_x_endpoint_security: *indexSettings
-    so-logs-endpoint_x_alerts: *indexSettings
-    so-logs-endpoint_x_events_x_api: *indexSettings
-    so-logs-endpoint_x_events_x_file: *indexSettings
-    so-logs-endpoint_x_events_x_library: *indexSettings
-    so-logs-endpoint_x_events_x_network: *indexSettings
-    so-logs-endpoint_x_events_x_process: *indexSettings
-    so-logs-endpoint_x_events_x_registry: *indexSettings
-    so-logs-endpoint_x_events_x_security: *indexSettings
-    so-logs-elastic_agent_x_filebeat: *indexSettings
-    so-logs-elastic_agent_x_fleet_server: *indexSettings
-    so-logs-elastic_agent_x_heartbeat: *indexSettings
-    so-logs-elastic_agent: *indexSettings
-    so-logs-elastic_agent_x_metricbeat: *indexSettings
-    so-logs-elastic_agent_x_osquerybeat: *indexSettings
-    so-logs-elastic_agent_x_packetbeat: *indexSettings
-    so-logs-elasticsearch_x_server: *indexSettings
-    so-metrics-endpoint_x_metadata: *indexSettings
-    so-metrics-endpoint_x_metrics: *indexSettings
-    so-metrics-endpoint_x_policy: *indexSettings
-    so-metrics-nginx_x_stubstatus: *indexSettings
-    so-metrics-vsphere_x_datastore: *indexSettings
-    so-metrics-vsphere_x_host: *indexSettings
-    so-metrics-vsphere_x_virtualmachine: *indexSettings
-    so-case: *indexSettings
-    so-common: *indexSettings
-    so-endgame: *indexSettings
-    so-idh: *indexSettings
-    so-suricata: *indexSettings
-    so-suricata_x_alerts: *indexSettings
-    so-import: *indexSettings
-    so-kratos: *indexSettings
-    so-hydra: *indexSettings
-    so-kismet: *indexSettings
-    so-logstash: *indexSettings
-    so-redis: *indexSettings
-    so-strelka: *indexSettings
-    so-syslog: *indexSettings
-    so-zeek: *indexSettings
+    so-logs-system_x_auth: *dataStreamSettings
+    so-logs-system_x_syslog: *dataStreamSettings
+    so-logs-system_x_system: *dataStreamSettings
+    so-logs-system_x_application: *dataStreamSettings
+    so-logs-system_x_security: *dataStreamSettings
+    so-logs-windows_x_forwarded: *dataStreamSettings
+    so-logs-windows_x_powershell: *dataStreamSettings
+    so-logs-windows_x_powershell_operational: *dataStreamSettings
+    so-logs-windows_x_sysmon_operational: *dataStreamSettings
+    so-logs-winlog_x_winlog: *dataStreamSettings
+    so-logs-detections_x_alerts: *dataStreamSettings
+    so-logs-http_endpoint_x_generic: *dataStreamSettings
+    so-logs-httpjson_x_generic: *dataStreamSettings
+    so-logs-osquery-manager-actions: *dataStreamSettings
+    so-logs-osquery-manager-action_x_responses: *dataStreamSettings
+    so-logs-osquery-manager_x_action_x_responses: *dataStreamSettings
+    so-logs-osquery-manager_x_result: *dataStreamSettings
+    so-logs-elastic_agent_x_apm_server: *dataStreamSettings
+    so-logs-elastic_agent_x_auditbeat: *dataStreamSettings
+    so-logs-elastic_agent_x_cloudbeat: *dataStreamSettings
+    so-logs-elastic_agent_x_endpoint_security: *dataStreamSettings
+    so-logs-endpoint_x_alerts: *dataStreamSettings
+    so-logs-endpoint_x_events_x_api: *dataStreamSettings
+    so-logs-endpoint_x_events_x_file: *dataStreamSettings
+    so-logs-endpoint_x_events_x_library: *dataStreamSettings
+    so-logs-endpoint_x_events_x_network: *dataStreamSettings
+    so-logs-endpoint_x_events_x_process: *dataStreamSettings
+    so-logs-endpoint_x_events_x_registry: *dataStreamSettings
+    so-logs-endpoint_x_events_x_security: *dataStreamSettings
+    so-logs-elastic_agent_x_filebeat: *dataStreamSettings
+    so-logs-elastic_agent_x_fleet_server: *dataStreamSettings
+    so-logs-elastic_agent_x_heartbeat: *dataStreamSettings
+    so-logs-elastic_agent: *dataStreamSettings
+    so-logs-elastic_agent_x_metricbeat: *dataStreamSettings
+    so-logs-elastic_agent_x_osquerybeat: *dataStreamSettings
+    so-logs-elastic_agent_x_packetbeat: *dataStreamSettings
+    so-logs-elasticsearch_x_server: *dataStreamSettings
+    so-metrics-endpoint_x_metadata: *dataStreamSettings
+    so-metrics-endpoint_x_metrics: *dataStreamSettings
+    so-metrics-endpoint_x_policy: *dataStreamSettings
+    so-metrics-nginx_x_stubstatus: *dataStreamSettings
+    so-metrics-vsphere_x_datastore: *dataStreamSettings
+    so-metrics-vsphere_x_host: *dataStreamSettings
+    so-metrics-vsphere_x_virtualmachine: *dataStreamSettings
+    so-common: *dataStreamSettings
+    so-endgame: *dataStreamSettings
+    so-idh: *dataStreamSettings
+    so-suricata: *dataStreamSettings
+    so-suricata_x_alerts: *dataStreamSettings
+    so-import: *dataStreamSettings
+    so-kratos: *dataStreamSettings
+    so-hydra: *dataStreamSettings
+    so-kismet: *dataStreamSettings
+    so-logstash: *dataStreamSettings
+    so-redis: *dataStreamSettings
+    so-strelka: *dataStreamSettings
+    so-syslog: *dataStreamSettings
+    so-zeek: *dataStreamSettings
+    # Managed SOC integration annotations are inserted below this line. Referencing '*dataStreamSettings'
+    so-case: &indexSettings
+      index_sorting:
+        description: Sorts the index by event time, at the cost of additional processing resource consumption.
+        forcedType: bool
+        global: True
+        advanced: True
+        helpLink: elasticsearch
+      index_template:
+        index_patterns:
+          description: Patterns for matching multiple indices or tables.
+          forcedType: "[]string"
+          multiline: True
+          global: True
+          advanced: True
+          helpLink: elasticsearch
+        template:
+          settings:
+            index:
+              number_of_replicas:
+                description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs.
+                forcedType: int
+                global: True
+                advanced: True
+                helpLink: elasticsearch
+              auto_expand_replicas:
+                description: Automatically expand the number of replicas based on the number of data nodes in the cluster. This can help ensure high availability as the cluster scales up or down.
+                forcedType: string
+                regex: "^(0-[1-9]|1-[2-9]|2-[3-9]|3-[4-9]|4-[5-9]|5-[6-9]|6-[7-9]|7-[89]|8-9|[0-9]-all|false)$"
+                regexFailureMessage: Must be in the format of "x-y" where x is minimum number of replicas and y is maximum number of replicas, or "0-all" to specify a minimum of 0 and no maximum, or "false" to disable automatic replica expansion.
+                global: True
+                advanced: True
+                helpLink: elasticsearch
+              mapping:
+                total_fields:
+                  limit:
+                    description: Max number of fields that can exist on a single index. Larger values will consume more resources.
+                    global: True
+                    advanced: True
+                    helpLink: elasticsearch
+              refresh_interval:
+                description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation.
+                global: True
+                advanced: True
+                helpLink: elasticsearch
+              number_of_shards:
+                description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs.
+                global: True
+                advanced: True
+                helpLink: elasticsearch
+              sort:
+                field:
+                  description: The field to sort by. Must set index_sorting to True.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+                order:
+                  description: The order to sort by. Must set index_sorting to True.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+          mappings:
+            _meta:
+              package:
+                name:
+                  description: Meta settings for the mapping.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+              managed_by:
+                  description: Meta settings for the mapping.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+              managed:
+                  description: Meta settings for the mapping.
+                  forcedType: bool
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+        composed_of:
+          description: The index template is composed of these component templates.
+          forcedType: "[]string"
+          global: True
+          advanced: True
+          helpLink: elasticsearch
+        priority:
+          description: The priority of the index template.
+          forcedType: int
+          global: True
+          advanced: True
+          helpLink: elasticsearch
+      policy:
+        phases:
+          hot:
+            min_age:
+              description: Minimum age of index. This determines when the index should be moved to the hot tier.
+              global: True
+              advanced: True
+              helpLink: elasticsearch
+            actions:
+              set_priority:
+                priority:
+                  description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
+                  forcedType: int
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+              rollover:
+                max_age:
+                  description: Maximum age of index.  Once an index reaches this limit, it will be rolled over into a new index.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+                max_primary_shard_size:
+                  description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+              shrink:
+                method:
+                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
+                  options:
+                  - COUNT
+                  - SIZE
+                  global: True
+                  advanced: True
+                  forcedType: string
+                number_of_shards:
+                  title: shard count
+                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                max_primary_shard_size:
+                  title: max shard size
+                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
+                  regex: ^[0-9]+(?:gb|tb|pb)$
+                  global: True
+                  forcedType: string
+                  advanced: True
+                allow_write_after_shrink:
+                  description: Allow writes after shrink.
+                  global: True
+                  forcedType: bool
+                  default: False
+                  advanced: True
+              forcemerge:
+                max_num_segments:
+                  description: Reduce the number of segments in each index shard and clean up deleted documents.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                index_codec:
+                  title: compression
+                  description: Use higher compression for stored fields at the cost of slower performance.
+                  forcedType: bool
+                  global: True
+                  default: False
+                  advanced: True
+          warm:
+            min_age:
+              description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
+              regex: ^[0-9]{1,5}d$
+              forcedType: string
+              global: True
+              advanced: True
+              helpLink: elasticsearch
+            actions:
+              set_priority:
+                priority:
+                  description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
+                  forcedType: int
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+              rollover:
+                max_age:
+                  description: Maximum age of index.  Once an index reaches this limit, it will be rolled over into a new index.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+                max_primary_shard_size:
+                  description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+              shrink:
+                method:
+                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
+                  options:
+                  - COUNT
+                  - SIZE
+                  global: True
+                  advanced: True
+                number_of_shards:
+                  title: shard count
+                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                max_primary_shard_size:
+                  title: max shard size
+                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
+                  regex: ^[0-9]+(?:gb|tb|pb)$
+                  global: True
+                  forcedType: string
+                  advanced: True
+                allow_write_after_shrink:
+                  description: Allow writes after shrink.
+                  global: True
+                  forcedType: bool
+                  default: False
+                  advanced: True
+              forcemerge:
+                max_num_segments:
+                  description: Reduce the number of segments in each index shard and clean up deleted documents.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                index_codec:
+                  title: compression
+                  description: Use higher compression for stored fields at the cost of slower performance.
+                  forcedType: bool
+                  global: True
+                  default: False
+                  advanced: True
+              allocate:
+                number_of_replicas:
+                  description: Set the number of replicas. Remains the same as the previous phase by default.
+                  forcedType: int
+                  global: True
+                  advanced: True
+          cold:
+            min_age:
+              description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
+              regex: ^[0-9]{1,5}d$
+              forcedType: string
+              global: True
+              advanced: True
+              helpLink: elasticsearch
+            actions:
+              set_priority:
+                priority:
+                  description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
+                  forcedType: int
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+              allocate:
+                number_of_replicas:
+                  description: Set the number of replicas. Remains the same as the previous phase by default.
+                  forcedType: int
+                  global: True
+                  advanced: True
+          delete:
+            min_age:
+              description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.
+              regex: ^[0-9]{1,5}d$
+              forcedType: string
+              global: True
+              advanced: True
+              helpLink: elasticsearch
+        _meta:
+          package:
+            name:
+              description: Meta settings for the mapping.
+              global: True
+              advanced: True
+              helpLink: elasticsearch
+          managed_by:
+            description: Meta settings for the mapping.
+            global: True
+            advanced: True
+            helpLink: elasticsearch
+          managed:
+            description: Meta settings for the mapping.
+            forcedType: bool
+            global: True
+            advanced: True
+            helpLink: elasticsearch
+    sos-backup: *indexSettings
+    so-detection: *indexSettings
+    so-assistant-chat: *indexSettings
+    so-assistant-session: *indexSettings
     so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings
       index_sorting:
         description: Sorts the index by event time, at the cost of additional processing resource consumption.

salt/elasticsearch/template.map.jinja

@@ -5,6 +5,7 @@
 
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
 {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %}
+{% set DATA_RETENTION_METHOD = salt['pillar.get']('elasticsearch:data_retention_method', ELASTICSEARCHDEFAULTS.elasticsearch.get('data_retention_method', 'ILM')) %}
 
 {% set PILLAR_GLOBAL_OVERRIDES = {} %}
 {% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings', {}) %}
@@ -105,6 +106,17 @@
 {%     if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %}
 {%       do settings.index_template.template.settings.index.pop('sort') %}
 {%     endif %}
+{%     if DATA_RETENTION_METHOD == 'DLM' and settings.index_template.data_stream is defined and settings.data_stream_lifecycle is defined %}
+{%       if settings.data_stream_lifecycle.data_retention is defined and settings.data_stream_lifecycle.data_retention %}
+{%         do settings.index_template.template.update({'lifecycle': {'data_retention': settings.data_stream_lifecycle.data_retention}}) %}
+{%       else %}
+{%         do settings.index_template.template.update({'lifecycle': {}}) %}
+{%       endif %}
+{%       if settings.index_template.template.settings.index.lifecycle is not defined %}
+{%         do settings.index_template.template.settings.index.update({'lifecycle': {}}) %}
+{%       endif %}
+{%       do settings.index_template.template.settings.index.lifecycle.update({'prefer_ilm': false}) %}
+{%     endif %}
 {%   endif %}
 
 {# advanced ilm actions #}

salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load

@@ -125,14 +125,6 @@ load_component_templates() {
     done
 }
 
-check_elasticsearch_responsive() {
-    # Cannot load templates if Elasticsearch is not responding.
-    #  NOTE: Slightly faster exit w/ failure than previous "retry 240 1" if there is a problem with Elasticsearch the
-    #    script should exit sooner rather than hang at the 'so-elasticsearch-templates' salt state.
-    retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" ||
-        fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot."
-}
-
 index_templates_exist() {
     local templates_dir="$1"
 

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply

@@ -0,0 +1,178 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+{%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
+
+{%- set DATA_RETENTION_METHOD = salt['pillar.get']('elasticsearch:data_retention_method', ELASTICSEARCHDEFAULTS.elasticsearch.get('data_retention_method', 'ILM')) %}
+
+ELASTICSEARCH_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR:-/opt/so/conf/elasticsearch/templates}"
+TEMPLATE_DIRS=(
+    "${ELASTICSEARCH_TEMPLATES_DIR}/index"
+    "${ELASTICSEARCH_TEMPLATES_DIR}/addon-index"
+)
+DATA_RETENTION_METHOD=$(cat <<'EOF'
+{{ DATA_RETENTION_METHOD }}
+EOF
+)
+DLM_FAILURES=0
+DLM_FAILURE_NAMES=()
+
+if [[ "$DATA_RETENTION_METHOD" != "DLM" && "$DATA_RETENTION_METHOD" != "ILM" ]]; then
+    echo "Unsupported data retention method $DATA_RETENTION_METHOD. Expected DLM or ILM."
+    exit 1
+fi
+
+validate_template_file() {
+    local template_file="$1"
+
+    if ! jq -e 'type == "object" and (.data_stream == null or (.data_stream | type == "object")) and (.template.lifecycle == null or (.template.lifecycle | type == "object")) and (.template.lifecycle.data_retention == null or (.template.lifecycle.data_retention | type == "string"))' >/dev/null 2>&1 "$template_file"; then
+        echo "Invalid index template JSON: $template_file"
+        return 1
+    fi
+}
+
+is_data_stream_template() {
+    jq -e '.data_stream | type == "object"' >/dev/null 2>&1 "$1"
+}
+
+has_data_stream_lifecycle() {
+    jq -e '.template.lifecycle | type == "object"' >/dev/null 2>&1 "$1"
+}
+
+get_data_retention() {
+    jq -r '.template.lifecycle.data_retention // ""' "$1"
+}
+
+find_template_file() {
+    local template="$1"
+    local template_dir
+    local template_file
+
+    for template_dir in "${TEMPLATE_DIRS[@]}"; do
+        template_file="${template_dir}/${template}-template.json"
+
+        if [[ -f "$template_file" ]]; then
+            echo "$template_file"
+            return 0
+        fi
+    done
+
+    return 1
+}
+
+set_data_stream_lifecycle() {
+    local data_stream="$1"
+    local data_retention="$2"
+    local body
+    local output
+
+    if [[ -n "$data_retention" ]]; then
+        if jq -e --arg data_stream "$data_stream" --arg data_retention "$data_retention" '.data_streams[]? | select(.name == $data_stream and .lifecycle.enabled == true and .lifecycle.data_retention == $data_retention)' >/dev/null 2>&1 <<< "$data_streams"; then
+            echo "DLM lifecycle already set for $data_stream with data_retention $data_retention, skipping."
+            return 0
+        fi
+    elif jq -e --arg data_stream "$data_stream" '.data_streams[]? | select(.name == $data_stream and .lifecycle.enabled == true and (.lifecycle.data_retention == null))' >/dev/null 2>&1 <<< "$data_streams"; then
+        echo "DLM lifecycle already set for $data_stream with indefinite retention, skipping."
+        return 0
+    fi
+
+    if [[ -n "$data_retention" ]]; then
+        body=$(jq -cn --arg data_retention "$data_retention" '{data_retention: $data_retention}')
+    else
+        # Setting indefinite retention
+        body='{}'
+    fi
+
+    if ! output=$(so-elasticsearch-query "_data_stream/${data_stream}/_lifecycle" -XPUT -d "$body" --retry 3 --retry-delay 5 --fail); then
+        echo "Failed to set data stream lifecycle for $data_stream."
+        echo "$output"
+        return 1
+    fi
+
+    if [[ -n "$data_retention" ]]; then
+        echo "Set DLM lifecycle for $data_stream with data_retention $data_retention."
+    else
+        echo "Set DLM lifecycle for $data_stream with indefinite retention."
+    fi
+}
+
+disable_data_stream_lifecycle() {
+    local data_stream="$1"
+    local body='{"enabled":false}'
+    local output
+
+    if ! jq -e --arg data_stream "$data_stream" '.data_streams[]? | select(.name == $data_stream and .lifecycle != null and .lifecycle.enabled != false)' >/dev/null 2>&1 <<< "$data_streams"; then
+        # No action needed
+        return 0
+    fi
+
+    if ! output=$(so-elasticsearch-query "_data_stream/${data_stream}/_lifecycle" -XPUT -d "$body" --retry 3 --retry-delay 5 --fail); then
+        echo "Failed to disable data stream lifecycle for $data_stream."
+        echo "$output"
+        return 1
+    fi
+
+    echo "Disabled DLM lifecycle for $data_stream."
+}
+
+process_data_stream() {
+    local data_stream="$1"
+    local data_retention="$2"
+
+    if [[ "$DATA_RETENTION_METHOD" == "DLM" ]]; then
+        set_data_stream_lifecycle "$data_stream" "$data_retention"
+    else
+        disable_data_stream_lifecycle "$data_stream"
+    fi
+}
+
+check_elasticsearch_responsive
+
+if ! data_streams=$(so-elasticsearch-query "_data_stream?format=json" --retry 3 --retry-delay 5 --fail); then
+    echo "Failed to retrieve data streams."
+    exit 1
+fi
+
+while read -r data_stream_config; do
+    data_stream=$(jq -r '.name' <<< "$data_stream_config")
+    template=$(jq -r '.template' <<< "$data_stream_config")
+
+    if ! template_file=$(find_template_file "$template"); then
+        echo "Skipping $data_stream: index template file not found for $template."
+        continue
+    fi
+
+    validate_template_file "$template_file" || exit 1
+
+    if ! is_data_stream_template "$template_file"; then
+        echo "Skipping $data_stream: $template_file is not a data stream template."
+        continue
+    fi
+
+    if [[ "$DATA_RETENTION_METHOD" == "DLM" ]] && ! has_data_stream_lifecycle "$template_file"; then
+        echo "Skipping $data_stream: $template_file does not define data stream lifecycle."
+        continue
+    fi
+
+    data_retention=$(get_data_retention "$template_file")
+
+    if ! process_data_stream "$data_stream" "$data_retention"; then
+        DLM_FAILURES=$((DLM_FAILURES + 1))
+        DLM_FAILURE_NAMES+=("$data_stream")
+    fi
+done < <(jq -c '.data_streams[]' <<< "$data_streams")
+
+if [[ $DLM_FAILURES -eq 0 ]]; then
+    echo "Data stream lifecycle updates completed successfully."
+else
+    echo "Encountered $DLM_FAILURES failure(s) updating data stream lifecycle:"
+    for failed_data_stream in "${DLM_FAILURE_NAMES[@]}"; do
+        echo "  - $failed_data_stream"
+    done
+    exit 1
+fi

salt/kibana/enabled.sls

@@ -6,7 +6,6 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
-{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 
 include:
@@ -61,19 +60,6 @@ so-kibana:
     - watch:
       - file: kibanaconfig
 
-wait_for_so-kibana:
-  http.wait_for_successful_query:
-    - name: "http://localhost:5601/api/status"
-    - username: 'so_elastic'
-    - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}'
-    - ssl: True
-    - verify_ssl: False
-    - status: 200
-    - wait_for: 300
-    - request_interval: 15
-    - require:
-      - docker_container: so-kibana
-
 delete_so-kibana_so-status.disabled:
   file.uncomment:
     - name: /opt/so/conf/so-status/so-status.conf

salt/manager/managed_soc_annotations.sls

@@ -16,40 +16,35 @@
 {%       endif %}
 {%     endfor %}
 {%   endfor %}
+{%   set soc_annotation_lines = [] %}
+{%   set defaults_lines = [] %}
+{%   for k in matched_integration_names %}
+{%     do soc_annotation_lines.append('    ' ~ k ~ ': *dataStreamSettings') %}
+{%     do defaults_lines.append('    ' ~ k ~ ':') %}
+{%     set defaults_yaml = salt['slsutil.serialize']('yaml', ADDON_INTEGRATION_DEFAULTS[k], default_flow_style=False).strip() %}
+{%     for line in defaults_yaml.splitlines() %}
+{%       do defaults_lines.append('      ' ~ line) %}
+{%     endfor %}
+{%   endfor %}
 {%   set es_soc_annotations = '/opt/so/saltstack/default/salt/elasticsearch/soc_elasticsearch.yaml' %}
-{{   es_soc_annotations }}:
-     file.serialize:
-       - dataset:
-           {% set data = salt['file.read'](es_soc_annotations) | load_yaml %}
-           {% set es = data.get('elasticsearch', {}) %}
-           {% set index_settings = es.get('index_settings', {}) %}
-           {% set input = index_settings.get('so-logs', {}) %}
-           {% for k in matched_integration_names %}
-           {%   do index_settings.update({k: input}) %}
-           {% endfor %}
-           {% for k in addon_integration_keys %}
-           {%   if k not in matched_integration_names and k in index_settings %}
-           {%     do index_settings.pop(k) %}
-           {%   endif %}
-           {% endfor %}
-           {{ data }}
+manage_soc_annotations:
+  file.blockreplace:
+    - name: {{ es_soc_annotations }}
+    - marker_start: '    # START managed SOC integration annotations'
+    - marker_end: '    # END managed SOC integration annotations'
+    - content: {{ soc_annotation_lines | join('\n') | tojson }}
+    - insert_after_match: '^    # Managed SOC integration annotations are inserted below this line\.'
+    - append_if_not_found: False
+    - show_changes: True
 
 {#   Managed elasticsearch/defaults.yaml file for enabling 'Revert to default' via SOC UI for newly added config items #}
 {%   set es_defaults = '/opt/so/saltstack/default/salt/elasticsearch/defaults.yaml' %}
 {{   es_defaults }}:
-     file.serialize:
-       - dataset:
-           {% set data = salt['file.read'](es_defaults) | load_yaml %}
-           {% set es = data.get('elasticsearch', {}) %}
-           {% set index_settings = es.get('index_settings', {}) %}
-           {% for k in matched_integration_names %}
-           {%   set input = ADDON_INTEGRATION_DEFAULTS[k] %}
-           {%     do index_settings.update({k: input})%}
-           {% endfor %}
-           {% for k in addon_integration_keys %}
-           {%   if k not in matched_integration_names and k in index_settings %}
-           {%     do index_settings.pop(k) %}
-           {%   endif %}
-           {% endfor %}
-           {{ data }}
-{% endif %}
+  file.blockreplace:
+    - marker_start: '    # START managed SOC integration defaults'
+    - marker_end: '    # END managed SOC integration defaults'
+    - content: {{ defaults_lines | join('\n') | tojson }}
+    - insert_after_match: '^  index_settings:$'
+    - append_if_not_found: False
+    - show_changes: True
+{% endif %}

salt/manager/tools/sbin/so-boot-mine-update

@@ -1,117 +0,0 @@
-#!/bin/bash
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Runs once per boot on managers (via so-boot-mine-update.service), before
-# so-boot-highstate.service. Waits for the responsive minion set to settle, pushes
-# mine.update, waits until every up minion has actually reported to the mine, then
-# warms the master's per-minion pillar cache so the mine-backed node pillars (node
-# IPs, ES/Redis/Logstash/hypervisor discovery -- some glob- and some pillar/grain-
-# targeted) are complete before the boot highstate renders them. Otherwise a node
-# that is up but not yet fully reported gets dropped from those pillars and torn
-# out of the configs they build (e.g. so-elasticsearch ExtraHosts -> container recreate).
-
-MAX_WAIT=${MINE_UPDATE_MAX_WAIT:-180}   # hard backstop only
-INTERVAL=10
-STABLE_CHECKS=3                          # up-count must hold steady this many polls
-elapsed=0
-prev=-1
-stable=0
-up=0
-
-# Wait for the *reachable* minion set to settle rather than for every accepted
-# key to report up: an operator may accept a minion's key and then intentionally
-# power off that host, so requiring up >= accepted would never be satisfied and
-# we'd always burn the full MAX_WAIT. Once the responsive count stops growing we
-# stop waiting and run mine.update against whoever is up.
-while [ "$elapsed" -lt "$MAX_WAIT" ]; do
-  up=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null \
-    | python3 -c 'import sys,json; print(len(json.load(sys.stdin)))' 2>/dev/null)
-  up=${up:-0}
-  if [ "$up" -gt 0 ] && [ "$up" -eq "$prev" ]; then
-    stable=$((stable + 1))
-    [ "$stable" -ge "$STABLE_CHECKS" ] && break
-  else
-    stable=0
-  fi
-  prev=$up
-  sleep "$INTERVAL"
-  elapsed=$((elapsed + INTERVAL))
-done
-
-echo "so-boot-mine-update: ${up} minions up (settled after ${elapsed}s); running mine.update"
-/usr/bin/salt '*' mine.update --out=txt
-
-# A node that is up but has not yet re-reported network.ip_addrs to the mine is
-# silently dropped from mine-backed pillars (elasticsearch:nodes, node_data, ...)
-# when highstate recompiles them -- which e.g. removes it from so-elasticsearch
-# ExtraHosts and forces a container recreate. After the broad mine.update above,
-# wait until every up minion actually has network.ip_addrs in the mine, re-pushing
-# mine.update to stragglers, before releasing the boot highstate. Bounded by the
-# same MAX_WAIT backstop so a slow/down node never blocks boot indefinitely.
-missing=""
-while [ "$elapsed" -lt "$MAX_WAIT" ]; do
-  up_json=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null)
-  mine_json=$(/usr/bin/salt-run mine.get '*' network.ip_addrs tgt_type=glob --out=json 2>/dev/null)
-  missing=$(printf '%s' "$up_json" | python3 -c '
-import sys, json
-up = set(json.load(sys.stdin) or [])
-mine = {k for k, v in (json.loads(sys.argv[1]) or {}).items() if v}
-print("\n".join(sorted(up - mine)))
-' "$mine_json" 2>/dev/null)
-  if [ -z "$missing" ]; then
-    echo "so-boot-mine-update: mine complete for all up minions after ${elapsed}s"
-    break
-  fi
-  echo "so-boot-mine-update: mine missing up minion(s): $(echo $missing); re-running mine.update"
-  for m in $missing; do /usr/bin/salt "$m" mine.update --out=txt; done
-  sleep "$INTERVAL"
-  elapsed=$((elapsed + INTERVAL))
-done
-[ -n "$missing" ] && echo "so-boot-mine-update: WARNING ${MAX_WAIT}s backstop hit; up minion(s) still absent from mine: $(echo $missing); highstate may drop them from configs"
-
-# The pillar/compound-targeted node pillars (elasticsearch:nodes, redis:nodes,
-# logstash:nodes, hypervisor:nodes) resolve their target against the master's
-# per-minion data cache (grains+pillar in .../minions/<id>/data.p), populated only
-# when a minion's pillar is (re)compiled -- separately from the mine. A freshly
-# booted node can be in the mine (glob/node_data sees it) yet absent from that
-# cache, so it is dropped from those pillars and from the configs they build (e.g.
-# so-elasticsearch ExtraHosts). Force a synchronous pillar refresh so the master
-# caches every up node's pillar; refresh_pillar wait=True returns only once the
-# pillar is recompiled (and thus cached for matching). Retry stragglers <= MAX_WAIT.
-echo "so-boot-mine-update: warming master pillar cache for pillar/grain-targeted node pillars"
-/usr/bin/salt '*' saltutil.refresh_pillar wait=True --out=txt
-missing=""
-while [ "$elapsed" -lt "$MAX_WAIT" ]; do
-  up_json=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null)
-  cached_json=$(/usr/bin/salt-run cache.pillar tgt='*' --out=json 2>/dev/null)
-  missing=$(printf '%s' "$up_json" | python3 -c '
-import sys, json
-up = set(json.load(sys.stdin) or [])
-cached = {k for k, v in (json.loads(sys.argv[1]) or {}).items() if v}
-print("\n".join(sorted(up - cached)))
-' "$cached_json" 2>/dev/null)
-  if [ -z "$missing" ]; then
-    echo "so-boot-mine-update: pillar cache warm for all up minions after ${elapsed}s"
-    break
-  fi
-  echo "so-boot-mine-update: pillar not yet cached for: $(echo $missing); refreshing"
-  for m in $missing; do /usr/bin/salt "$m" saltutil.refresh_pillar wait=True --out=txt; done
-  sleep "$INTERVAL"
-  elapsed=$((elapsed + INTERVAL))
-done
-[ -n "$missing" ] && echo "so-boot-mine-update: WARNING ${MAX_WAIT}s backstop hit; pillar not cached for: $(echo $missing); pillar-targeted pillars may drop them"
-
-# Log what the mine-backed pillars render so the boot-time state is inspectable.
-/usr/bin/salt-call saltutil.refresh_pillar >/dev/null 2>&1
-sleep 2
-for key in node_data elasticsearch:nodes; do
-  rendered=$(/usr/bin/salt-call --out=json pillar.get "$key" 2>/dev/null \
-    | python3 -c 'import sys,json; print(json.dumps(json.load(sys.stdin).get("local"), indent=2, sort_keys=True))' 2>/dev/null)
-  echo "so-boot-mine-update: ${key} rendered as:"
-  echo "${rendered:-null}"
-done
-exit 0

salt/manager/tools/sbin/soup

@@ -761,9 +761,57 @@ bootstrap_so_soc_database() {
   echo "so_soc bootstrap complete."
 }
 
+# Existing grids should keep ILM unless an admin explicitly opts in to DLM.
+pin_elasticsearch_data_retention_method() {
+  local elasticsearch_file=/opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls
+  mkdir -p "$(dirname "$elasticsearch_file")"
+  [[ -f "$elasticsearch_file" ]] || touch "$elasticsearch_file"
+
+  if so-yaml.py get -r "$elasticsearch_file" elasticsearch.data_retention_method >/dev/null 2>&1; then
+    echo "elasticsearch.data_retention_method already set; leaving as-is."
+    return 0
+  fi
+
+  echo "Pinning existing grid to ILM data retention."
+  so-yaml.py add "$elasticsearch_file" elasticsearch.data_retention_method ILM
+  chown socore:socore "$elasticsearch_file"
+}
+
+# Addes auto_expand_replicas setting to .kibana_streams index template
+#
+# In Kibana 9.3.3 the auto_expand_replicas setting was not added to the .kibana_streams index template. Causing single node deployments to be stuck in yellow state (unable to assign replica). Here we update the template in place using the so_kibana system user (system managed index template) to include the auto_expand_replicas setting
+#
+# Reference: https://github.com/elastic/kibana/issues/263048
+kibana_backport_streams_index_template() {
+    local current_template updated_template
+    current_template=$(so-elasticsearch-query "_index_template/.kibana_streams" --retry 3 --retry-delay 5 --fail)
+
+    if [[ -z "$current_template" ]]; then
+        echo "Unable to retrieve current .kibana_streams index template, skipping backport."
+        return 0
+    fi
+
+    updated_template=$(jq '.index_templates[0].index_template | .template.settings += {"index.auto_expand_replicas": "0-1"} | del(.created_date_millis, .modified_date_millis)' <<< "$current_template")
+
+    if ! kibana_user_pass=$(/usr/sbin/so-yaml.py get -r /opt/so/saltstack/local/pillar/elasticsearch/auth.sls elasticsearch.auth.users.so_kibana_user.pass); then
+        echo "Unable to retrieve so_kibana_user password, skipping .kibana_streams index template backport."
+        return 0
+    fi
+
+    if ! so-elasticsearch-query "_index_template/.kibana_streams" -XPUT -d "$updated_template" -u "so_kibana:$kibana_user_pass" --retry 3 --retry-delay 5 --fail; then
+        echo "Unable to automatically update .kibana_streams index template"
+        return 0
+    fi
+
+    ## NOTE: Should really add a check here for existing .kibana_streams index and then update its config in place
+
+}
+
 up_to_3.2.0() {
   fix_logstash_0013_lumberjack_pipeline_name
 
+  pin_elasticsearch_data_retention_method
+
   INSTALLEDVERSION=3.2.0
 }
 
@@ -774,6 +822,8 @@ post_to_3.2.0() {
   echo "Regenerating Elastic Agent Installers"
   /sbin/so-elastic-agent-gen-installers
 
+  kibana_backport_streams_index_template
+
   POSTVERSION=3.2.0
 }
 

salt/salt/master.sls

@@ -14,7 +14,6 @@
 
 include:
   - salt.minion
-  - salt.master.boot_mine_update
 {%   if 'vrt' in salt['pillar.get']('features', []) %}
   - salt.cloud
   - salt.cloud.reactor_config_hypervisor

salt/salt/master/boot_mine_update.sls

@@ -1,29 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Manages /etc/systemd/system/so-boot-mine-update.service, a manager-only
-# Type=oneshot unit that pushes `salt '*' mine.update` once per boot, ordered
-# before so-boot-highstate.service so mine-backed pillars (node IPs, ES/Redis/
-# Logstash discovery) are fresh before the boot highstate renders them.
-
-include:
-  - systemd.reload
-
-so_boot_mine_update_unit_file:
-  file.managed:
-    - name: /etc/systemd/system/so-boot-mine-update.service
-    - source: salt://salt/service/so-boot-mine-update.service
-    - onchanges_in:
-      - module: systemd_reload
-
-# Only enable once setup is complete. Until then the gate file is missing and
-# the unit's own ConditionPathExists would no-op it anyway.
-so_boot_mine_update_service:
-  service.enabled:
-    - name: so-boot-mine-update.service
-    - onlyif: test -e /opt/so/state/setup-complete
-    - require:
-      - file: so_boot_mine_update_unit_file
-      - module: systemd_reload

salt/salt/service/so-boot-mine-update.service

@@ -1,15 +0,0 @@
-[Unit]
-Description=Security Onion boot-time grid mine.update (managers, runs once per boot before highstate)
-After=salt-master.service salt-minion.service network-online.target
-Wants=network-online.target
-Requires=salt-master.service salt-minion.service
-Before=so-boot-highstate.service
-ConditionPathExists=/opt/so/state/setup-complete
-
-[Service]
-Type=oneshot
-RemainAfterExit=yes
-ExecStart=/usr/sbin/so-boot-mine-update
-
-[Install]
-WantedBy=multi-user.target

setup/so-functions

@@ -980,8 +980,6 @@ docker_seed_registry() {
 		docker_seed_update_percent=25
 
 		update_docker_containers 'netinstall' '' 'docker_seed_update' '/dev/stdout' 2>&1 | tee -a "$setup_log"
-        # Use pipe exit status of 'update_docker_containers' for return code
-		return ${PIPESTATUS[0]}
 	fi
 }
 

setup/so-setup

@@ -223,8 +223,6 @@ if [ -n "$test_profile" ]; then
 	WEBPASSWD1=0n10nus3r
 	WEBPASSWD2=0n10nus3r
 	NODE_DESCRIPTION="${HOSTNAME} - ${install_type} - ${MSRVIP_OFFSET}"
-	# opt out of telemetry for automated testing
-	telemetry=1
 
 	update_sudoers_for_testing
 fi
@@ -769,10 +767,7 @@ if ! [[ -f $install_opt_file ]]; then
 		title "Applying the registry state"
 		logCmd "salt-call state.apply -l info registry"
 		title "Seeding the docker registry"
-		if ! docker_seed_registry; then
-			error "Failed to seed the docker registry"
-			fail_setup
-		fi
+		docker_seed_registry
 		title "Applying the manager state"
 		logCmd "salt-call state.apply -l info manager"
 		logCmd "salt-call state.apply influxdb -l info"