From eb82f9ea9d07aec323fc1dccfadbd4a197a11278 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Mon, 8 Jun 2026 16:53:35 -0400
Subject: [PATCH 01/28] kilo version

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 944880fa1..03e153fda 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-3.2.0
+3.0.0-kilo

From 9aa9ea3255f3b53bbea6293d743f170d9b908ec2 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Mon, 8 Jun 2026 15:32:03 -0500
Subject: [PATCH 02/28] Iniitial DLM support

---
 salt/common/tools/sbin/so-common              |   5 +
 salt/elasticfleet/content-defaults.map.jinja  |   1 -
 salt/elasticfleet/input-defaults.map.jinja    |   2 -
 salt/elasticsearch/cluster.sls                |  12 +
 salt/elasticsearch/defaults.yaml              | 142 +++++-
 salt/elasticsearch/soc_elasticsearch.yaml     | 453 +++++++++++++++---
 salt/elasticsearch/template.map.jinja         |  12 +
 .../sbin/so-elasticsearch-templates-load      |   8 -
 .../sbin_jinja/so-elasticsearch-dlm-apply     | 135 ++++++
 salt/manager/managed_soc_annotations.sls      |  59 ++-
 salt/manager/tools/sbin/soup                  |  50 ++
 11 files changed, 771 insertions(+), 108 deletions(-)
 create mode 100644 salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index aca8496f5..812c1bb10 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -142,6 +142,11 @@ check_elastic_license() {
 	fi  
 }
 
+check_elasticsearch_responsive() {
+    retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" ||
+        fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot."
+}
+
 check_salt_master_status() {
 	local count=0
     local attempts="${1:- 10}"
diff --git a/salt/elasticfleet/content-defaults.map.jinja b/salt/elasticfleet/content-defaults.map.jinja
index f4237d6d1..7f40f7a21 100644
--- a/salt/elasticfleet/content-defaults.map.jinja
+++ b/salt/elasticfleet/content-defaults.map.jinja
@@ -9,7 +9,6 @@
 
 {% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
 {% set ADDON_CONTENT_INTEGRATION_DEFAULTS = {} %}
-{% set DEBUG_STUFF = {} %}
 
 {% for pkg in ADDON_CONTENT_PACKAGE_COMPONENTS %}
 {%   if pkg.name in CORE_ESFLEET_PACKAGES %}
diff --git a/salt/elasticfleet/input-defaults.map.jinja b/salt/elasticfleet/input-defaults.map.jinja
index a02844330..6b94d0581 100644
--- a/salt/elasticfleet/input-defaults.map.jinja
+++ b/salt/elasticfleet/input-defaults.map.jinja
@@ -9,7 +9,6 @@
 
 {% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
 {% set ADDON_INPUT_INTEGRATION_DEFAULTS = {} %}
-{% set DEBUG_STUFF = {} %}
 
 {% for pkg in ADDON_INPUT_PACKAGE_COMPONENTS %}
 {%   if pkg.name in CORE_ESFLEET_PACKAGES %}
@@ -116,7 +115,6 @@
 
 
 {%         do ADDON_INPUT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
-{%         do DEBUG_STUFF.update({integration_key: "Generating defaults for "+ pkg.name })%}
 {%       endfor %}
 {%     endif %}
 {%   endif %}
diff --git a/salt/elasticsearch/cluster.sls b/salt/elasticsearch/cluster.sls
index d20ee45ca..efa50285e 100644
--- a/salt/elasticsearch/cluster.sls
+++ b/salt/elasticsearch/cluster.sls
@@ -133,6 +133,18 @@ so-elasticsearch-templates:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
 
+so-elasticsearch-dlm-apply:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-dlm-apply
+    - cwd: /opt/so
+    - require:
+      - docker_container: so-elasticsearch
+      - file: elasticsearch_sbin_jinja
+      - cmd: so-elasticsearch-templates
+    - retry:
+        attempts: 3
+        interval: 10
+
 so-elasticsearch-pipelines:
   cmd.run:
     - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }}
diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml
index 52964b9cf..7b49074e8 100644
--- a/salt/elasticsearch/defaults.yaml
+++ b/salt/elasticsearch/defaults.yaml
@@ -2,6 +2,7 @@ elasticsearch:
   enabled: false
   version: 9.3.3
   index_clean: true
+  data_retention_method: DLM
   vm:
     max_map_count: 1048576
   config:
@@ -18,9 +19,18 @@ elasticsearch:
               flood_stage: 90%
               high: 85%
               low: 80%
+    # don't want to set retention here since it will make ES restart with every update +
+    #   potentially case where we could unintentially fall back to retention 7d and cause data loss
+    # data_streams:
+    #   lifecycle:
+    #     retention:
+    #       default: 7d
     indices:
       id_field_data:
         enabled: false
+    # index:
+      # lifecycle:
+        # prefer_ilm: true
     logger:
       org:
         elasticsearch:
@@ -63,6 +73,9 @@ elasticsearch:
             verification_mode: none
   index_settings:
     global_overrides:
+    # Tie this into cluster setting for data_streams.lifecycle.retention.default
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         template:
           settings:
@@ -143,6 +156,8 @@ elasticsearch:
                 order: desc
     so-common:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - agent-mappings
@@ -304,6 +319,8 @@ elasticsearch:
               number_of_shards: 1
     so-assistant-chat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: ""
       index_template:
         composed_of:
         - assistant-chat-mappings
@@ -344,6 +361,8 @@ elasticsearch:
             min_age: 0ms
     so-assistant-session:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: ""
       index_template:
         composed_of:
         - assistant-session-mappings
@@ -497,6 +516,8 @@ elasticsearch:
             min_age: 30d
     so-idh:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - agent-mappings
@@ -605,6 +626,8 @@ elasticsearch:
             min_age: 30d
     so-import:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - agent-mappings
@@ -787,6 +810,8 @@ elasticsearch:
             min_age: 0ms
     so-kismet:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - kismet-mappings
@@ -836,6 +861,8 @@ elasticsearch:
             min_age: 30d
     so-kratos:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - agent-mappings
@@ -904,6 +931,8 @@ elasticsearch:
             min_age: 30d
     so-hydra:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - agent-mappings
@@ -1049,6 +1078,8 @@ elasticsearch:
             min_age: 0ms
     so-logs:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - so-data-streams-mappings
@@ -1129,6 +1160,8 @@ elasticsearch:
             min_age: 30d
     so-logs-detections_x_alerts:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - so-data-streams-mappings
@@ -1192,6 +1225,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - event-mappings
@@ -1307,6 +1342,8 @@ elasticsearch:
             min_age: 30d
     so-elastic-agent-monitor:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - event-mappings
@@ -1369,6 +1406,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_apm_server:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - logs-elastic_agent.apm_server@package
@@ -1433,6 +1472,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_auditbeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - logs-elastic_agent.auditbeat@package
@@ -1497,6 +1538,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_cloudbeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - logs-elastic_agent.cloudbeat@package
@@ -1561,6 +1604,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_endpoint_security:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - event-mappings
@@ -1620,6 +1665,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_filebeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - event-mappings
@@ -1679,6 +1726,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_fleet_server:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - event-mappings
@@ -1735,6 +1784,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_heartbeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - logs-elastic_agent.heartbeat@package
@@ -1799,6 +1850,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_metricbeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - event-mappings
@@ -1858,6 +1911,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_osquerybeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - event-mappings
@@ -1917,6 +1972,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_packetbeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - logs-elastic_agent.packetbeat@package
@@ -1981,6 +2038,8 @@ elasticsearch:
             min_age: 30d
     so-logs-elasticsearch_x_server:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - logs-elasticsearch.server@package
@@ -2045,6 +2104,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_actions:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - .logs-endpoint.actions@package
@@ -2104,6 +2165,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_action_x_responses:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - .logs-endpoint.action.responses@package
@@ -2163,6 +2226,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_alerts:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - logs-endpoint.alerts@package
@@ -2222,6 +2287,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_diagnostic_x_collection:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - .logs-endpoint.diagnostic.collection@package
@@ -2297,6 +2364,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_api:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - logs-endpoint.events.api@package
@@ -2356,6 +2425,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_file:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - logs-endpoint.events.file@package
@@ -2415,6 +2486,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_library:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - logs-endpoint.events.library@package
@@ -2474,6 +2547,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_network:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - logs-endpoint.events.network@package
@@ -2533,6 +2608,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_process:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - logs-endpoint.events.process@package
@@ -2592,6 +2669,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_registry:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - logs-endpoint.events.registry@package
@@ -2651,6 +2730,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_security:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - logs-endpoint.events.security@package
@@ -2710,6 +2791,8 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_heartbeat:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - .logs-endpoint.heartbeat@package
@@ -2769,6 +2852,8 @@ elasticsearch:
             min_age: 30d
     so-logs-http_endpoint_x_generic:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - logs-http_endpoint.generic@package
@@ -2817,6 +2902,8 @@ elasticsearch:
             min_age: 30d
     so-logs-httpjson_x_generic:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - logs-httpjson.generic@package
@@ -2882,6 +2969,8 @@ elasticsearch:
               number_of_replicas: 0
     so-logs-osquery-manager_x_action_x_responses:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         _meta:
           managed: true
@@ -2953,6 +3042,8 @@ elasticsearch:
               number_of_replicas: 0
     so-logs-osquery-manager_x_result:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         _meta:
           managed: true
@@ -3005,6 +3096,8 @@ elasticsearch:
             min_age: 30d
     so-logs-soc:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - agent-mappings
@@ -3113,6 +3206,8 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_application:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - event-mappings
@@ -3162,6 +3257,8 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_auth:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - event-mappings
@@ -3211,6 +3308,8 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_security:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - event-mappings
@@ -3260,6 +3359,8 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_syslog:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - event-mappings
@@ -3309,6 +3410,8 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_system:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - event-mappings
@@ -3358,6 +3461,8 @@ elasticsearch:
             min_age: 30d
     so-logs-windows_x_forwarded:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - logs-windows.forwarded@package
@@ -3405,6 +3510,8 @@ elasticsearch:
             min_age: 30d
     so-logs-windows_x_powershell:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - logs-windows.powershell@package
@@ -3452,6 +3559,8 @@ elasticsearch:
             min_age: 30d
     so-logs-windows_x_powershell_operational:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - logs-windows.powershell_operational@package
@@ -3499,6 +3608,8 @@ elasticsearch:
             min_age: 30d
     so-logs-windows_x_sysmon_operational:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - logs-windows.sysmon_operational@package
@@ -3546,6 +3657,8 @@ elasticsearch:
             min_age: 30d
     so-logs-winlog_x_winlog:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - logs-winlog.winlog@package
@@ -3594,6 +3707,8 @@ elasticsearch:
             min_age: 30d
     so-logstash:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - agent-mappings
@@ -3709,6 +3824,8 @@ elasticsearch:
             min_age: 30d
     so-metrics-endpoint_x_metadata:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - metrics-endpoint.metadata@package
@@ -3756,6 +3873,8 @@ elasticsearch:
             min_age: 30d
     so-metrics-endpoint_x_metrics:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - metrics-endpoint.metrics@package
@@ -3803,6 +3922,8 @@ elasticsearch:
             min_age: 30d
     so-metrics-endpoint_x_policy:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - metrics-endpoint.policy@package
@@ -3850,6 +3971,8 @@ elasticsearch:
             min_age: 30d
     so-metrics-fleet_server_x_agent_status:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - metrics@tsdb-settings
@@ -3874,6 +3997,8 @@ elasticsearch:
               number_of_replicas: 0
     so-metrics-fleet_server_x_agent_versions:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - metrics@tsdb-settings
@@ -3898,6 +4023,8 @@ elasticsearch:
               number_of_replicas: 0
     so-redis:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - agent-mappings
@@ -3958,13 +4085,10 @@ elasticsearch:
         - vulnerability-mappings
         - common-settings
         - common-dynamic-mappings
-        - logs-redis.log@package
-        - logs-redis.log@custom
         data_stream:
           allow_custom_routing: false
           hidden: false
-        ignore_missing_component_templates:
-        - logs-redis.log@custom
+        ignore_missing_component_templates: []
         index_patterns:
         - logs-redis.log*
         priority: 501
@@ -4016,6 +4140,8 @@ elasticsearch:
             min_age: 30d
     so-strelka:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - agent-mappings
@@ -4133,6 +4259,8 @@ elasticsearch:
             min_age: 30d
     so-suricata:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - agent-mappings
@@ -4249,6 +4377,8 @@ elasticsearch:
             min_age: 30d
     so-suricata_x_alerts:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - agent-mappings
@@ -4365,6 +4495,8 @@ elasticsearch:
             min_age: 30d
     so-syslog:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - agent-mappings
@@ -4481,6 +4613,8 @@ elasticsearch:
             min_age: 30d
     so-zeek:
       index_sorting: false
+      data_stream_lifecycle:
+        data_retention: 7d
       index_template:
         composed_of:
         - agent-mappings
diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml
index b96c58dbe..251a0c1d8 100644
--- a/salt/elasticsearch/soc_elasticsearch.yaml
+++ b/salt/elasticsearch/soc_elasticsearch.yaml
@@ -4,6 +4,13 @@ elasticsearch:
     forcedType: bool
     advanced: True
     helpLink: elasticsearch
+  data_retention_method:
+    description: Method for data retention. Options are ILM or DLM. For single node deployments and most distributed grid users, DLM will be the recommended option for simplified management. Those with more complex use cases may prefer ILM. The latter allows for more granular control, but requires more management overhead.
+    options:
+    - ILM
+    - DLM
+    forcedType: string
+    global: True
   version:
     description: "This specifies the version of the following containers: so-elastic-fleet-package-registry, so-elastic-agent, so-elastic-fleet, so-kibana, so-logstash and so-elasticsearch. Modifying this value in the Elasticsearch defaults.yaml will result in catastrophic grid failure."
     readonly: True
@@ -13,7 +20,7 @@ elasticsearch:
     description: Specify the memory heap size in (m)egabytes for Elasticsearch.
     helpLink: elasticsearch
   index_clean:
-    description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, indices will only be deleted by the age defined in the ILM settings. This setting only applies to EVAL, STANDALONE, and HEAVY NODE installations. Other installations can only use ILM settings.
+    description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, data is retained by the configured lifecycle settings. This setting only applies to EVAL, STANDALONE, and HEAVY NODE installations. Other installations use lifecycle settings only.
     forcedType: bool
     helpLink: elasticsearch
   vm:
@@ -139,6 +146,22 @@ elasticsearch:
     custom010: *pipelines
   index_settings:
     global_overrides:
+      data_stream_lifecycle:
+        data_retention:
+          description: |
+            The retention period for all data streams. Retention does not define the period that the data will be removed, but the minimum time period they will be kept.
+
+            Use a number followed by a time unit, such as 7d. Leave blank for indefinite retention where supported.
+
+            Configured retention period also affects the frequency of rolling over data streams.
+              - If retention is less than or equal to 1 day, max_age will be 1 hour
+              - If retention is less than or equal to 14 days, max_age will be 1 day
+              - If retention is less than or equal to 90 days, max_age will be 7 days
+              - If retention is greater than 90 days, max_age will be 30 days
+          global: True
+          forcedType: string
+          regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$
+          regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d.
       index_template:
         template:
           settings:
@@ -311,13 +334,29 @@ elasticsearch:
               forcedType: string
               global: True
               helpLink: elasticsearch
-    so-logs: &indexSettings
+    so-logs: &dataStreamSettings
       index_sorting:
         description: Sorts the index by event time, at the cost of additional processing resource consumption.
         forcedType: bool
         global: True
         advanced: True
         helpLink: elasticsearch
+      data_stream_lifecycle:
+        data_retention:
+          description: |
+            The retention period for this data stream. Retention does not define the period that the data will be removed, but the minimum time period it will be kept.
+
+            Use a number followed by a time unit, such as 7d. Leave blank for indefinite retention where supported.
+
+            Configured retention period also affects the frequency of rolling over this data stream.
+              - If retention is less than or equal to 1 day, max_age will be 1 hour
+              - If retention is less than or equal to 14 days, max_age will be 1 day
+              - If retention is less than or equal to 90 days, max_age will be 7 days
+              - If retention is greater than 90 days, max_age will be 30 days
+          global: True
+          forcedType: string
+          regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$
+          regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d.
       index_template:
         index_patterns:
           description: Patterns for matching multiple indices or tables.
@@ -335,6 +374,14 @@ elasticsearch:
                 global: True
                 advanced: True
                 helpLink: elasticsearch
+              auto_expand_replicas:
+                description: Automatically expand the number of replicas based on the number of data nodes in the cluster. This can help ensure high availability as the cluster scales up or down.
+                forcedType: string
+                regex: "^(0-[1-9]|1-[2-9]|2-[3-9]|3-[4-9]|4-[5-9]|5-[6-9]|6-[7-9]|7-[89]|8-9|[0-9]-all|false)$"
+                regexFailureMessage: Must be in the format of "x-y" where x is minimum number of replicas and y is maximum number of replicas, or "0-all" to specify a minimum of 0 and no maximum, or "false" to disable automatic replica expansion.
+                global: True
+                advanced: True
+                helpLink: elasticsearch
               mapping:
                 total_fields:
                   limit:
@@ -596,65 +643,349 @@ elasticsearch:
             global: True
             advanced: True
             helpLink: elasticsearch
-    so-logs-system_x_auth: *indexSettings
-    so-logs-system_x_syslog: *indexSettings
-    so-logs-system_x_system: *indexSettings
-    so-logs-system_x_application: *indexSettings
-    so-logs-system_x_security: *indexSettings
-    so-logs-windows_x_forwarded: *indexSettings
-    so-logs-windows_x_powershell: *indexSettings
-    so-logs-windows_x_powershell_operational: *indexSettings
-    so-logs-windows_x_sysmon_operational: *indexSettings
-    so-logs-winlog_x_winlog: *indexSettings
-    so-logs-detections_x_alerts: *indexSettings
-    so-logs-http_endpoint_x_generic: *indexSettings
-    so-logs-httpjson_x_generic: *indexSettings
-    so-logs-osquery-manager-actions: *indexSettings
-    so-logs-osquery-manager-action_x_responses: *indexSettings
-    so-logs-osquery-manager_x_action_x_responses: *indexSettings
-    so-logs-osquery-manager_x_result: *indexSettings
-    so-logs-elastic_agent_x_apm_server: *indexSettings
-    so-logs-elastic_agent_x_auditbeat: *indexSettings
-    so-logs-elastic_agent_x_cloudbeat: *indexSettings
-    so-logs-elastic_agent_x_endpoint_security: *indexSettings
-    so-logs-endpoint_x_alerts: *indexSettings
-    so-logs-endpoint_x_events_x_api: *indexSettings
-    so-logs-endpoint_x_events_x_file: *indexSettings
-    so-logs-endpoint_x_events_x_library: *indexSettings
-    so-logs-endpoint_x_events_x_network: *indexSettings
-    so-logs-endpoint_x_events_x_process: *indexSettings
-    so-logs-endpoint_x_events_x_registry: *indexSettings
-    so-logs-endpoint_x_events_x_security: *indexSettings
-    so-logs-elastic_agent_x_filebeat: *indexSettings
-    so-logs-elastic_agent_x_fleet_server: *indexSettings
-    so-logs-elastic_agent_x_heartbeat: *indexSettings
-    so-logs-elastic_agent: *indexSettings
-    so-logs-elastic_agent_x_metricbeat: *indexSettings
-    so-logs-elastic_agent_x_osquerybeat: *indexSettings
-    so-logs-elastic_agent_x_packetbeat: *indexSettings
-    so-logs-elasticsearch_x_server: *indexSettings
-    so-metrics-endpoint_x_metadata: *indexSettings
-    so-metrics-endpoint_x_metrics: *indexSettings
-    so-metrics-endpoint_x_policy: *indexSettings
-    so-metrics-nginx_x_stubstatus: *indexSettings
-    so-metrics-vsphere_x_datastore: *indexSettings
-    so-metrics-vsphere_x_host: *indexSettings
-    so-metrics-vsphere_x_virtualmachine: *indexSettings
-    so-case: *indexSettings
-    so-common: *indexSettings
-    so-endgame: *indexSettings
-    so-idh: *indexSettings
-    so-suricata: *indexSettings
-    so-suricata_x_alerts: *indexSettings
-    so-import: *indexSettings
-    so-kratos: *indexSettings
-    so-hydra: *indexSettings
-    so-kismet: *indexSettings
-    so-logstash: *indexSettings
-    so-redis: *indexSettings
-    so-strelka: *indexSettings
-    so-syslog: *indexSettings
-    so-zeek: *indexSettings
+    so-logs-system_x_auth: *dataStreamSettings
+    so-logs-system_x_syslog: *dataStreamSettings
+    so-logs-system_x_system: *dataStreamSettings
+    so-logs-system_x_application: *dataStreamSettings
+    so-logs-system_x_security: *dataStreamSettings
+    so-logs-windows_x_forwarded: *dataStreamSettings
+    so-logs-windows_x_powershell: *dataStreamSettings
+    so-logs-windows_x_powershell_operational: *dataStreamSettings
+    so-logs-windows_x_sysmon_operational: *dataStreamSettings
+    so-logs-winlog_x_winlog: *dataStreamSettings
+    so-logs-detections_x_alerts: *dataStreamSettings
+    so-logs-http_endpoint_x_generic: *dataStreamSettings
+    so-logs-httpjson_x_generic: *dataStreamSettings
+    so-logs-osquery-manager-actions: *dataStreamSettings
+    so-logs-osquery-manager-action_x_responses: *dataStreamSettings
+    so-logs-osquery-manager_x_action_x_responses: *dataStreamSettings
+    so-logs-osquery-manager_x_result: *dataStreamSettings
+    so-logs-elastic_agent_x_apm_server: *dataStreamSettings
+    so-logs-elastic_agent_x_auditbeat: *dataStreamSettings
+    so-logs-elastic_agent_x_cloudbeat: *dataStreamSettings
+    so-logs-elastic_agent_x_endpoint_security: *dataStreamSettings
+    so-logs-endpoint_x_alerts: *dataStreamSettings
+    so-logs-endpoint_x_events_x_api: *dataStreamSettings
+    so-logs-endpoint_x_events_x_file: *dataStreamSettings
+    so-logs-endpoint_x_events_x_library: *dataStreamSettings
+    so-logs-endpoint_x_events_x_network: *dataStreamSettings
+    so-logs-endpoint_x_events_x_process: *dataStreamSettings
+    so-logs-endpoint_x_events_x_registry: *dataStreamSettings
+    so-logs-endpoint_x_events_x_security: *dataStreamSettings
+    so-logs-elastic_agent_x_filebeat: *dataStreamSettings
+    so-logs-elastic_agent_x_fleet_server: *dataStreamSettings
+    so-logs-elastic_agent_x_heartbeat: *dataStreamSettings
+    so-logs-elastic_agent: *dataStreamSettings
+    so-logs-elastic_agent_x_metricbeat: *dataStreamSettings
+    so-logs-elastic_agent_x_osquerybeat: *dataStreamSettings
+    so-logs-elastic_agent_x_packetbeat: *dataStreamSettings
+    so-logs-elasticsearch_x_server: *dataStreamSettings
+    so-metrics-endpoint_x_metadata: *dataStreamSettings
+    so-metrics-endpoint_x_metrics: *dataStreamSettings
+    so-metrics-endpoint_x_policy: *dataStreamSettings
+    so-metrics-nginx_x_stubstatus: *dataStreamSettings
+    so-metrics-vsphere_x_datastore: *dataStreamSettings
+    so-metrics-vsphere_x_host: *dataStreamSettings
+    so-metrics-vsphere_x_virtualmachine: *dataStreamSettings
+    so-common: *dataStreamSettings
+    so-endgame: *dataStreamSettings
+    so-idh: *dataStreamSettings
+    so-suricata: *dataStreamSettings
+    so-suricata_x_alerts: *dataStreamSettings
+    so-import: *dataStreamSettings
+    so-kratos: *dataStreamSettings
+    so-hydra: *dataStreamSettings
+    so-kismet: *dataStreamSettings
+    so-logstash: *dataStreamSettings
+    so-redis: *dataStreamSettings
+    so-strelka: *dataStreamSettings
+    so-syslog: *dataStreamSettings
+    so-zeek: *dataStreamSettings
+    # Managed SOC integration annotations are inserted below this line. Referencing '*dataStreamSettings'
+    so-case: &indexSettings
+      index_sorting:
+        description: Sorts the index by event time, at the cost of additional processing resource consumption.
+        forcedType: bool
+        global: True
+        advanced: True
+        helpLink: elasticsearch
+      index_template:
+        index_patterns:
+          description: Patterns for matching multiple indices or tables.
+          forcedType: "[]string"
+          multiline: True
+          global: True
+          advanced: True
+          helpLink: elasticsearch
+        template:
+          settings:
+            index:
+              number_of_replicas:
+                description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs.
+                forcedType: int
+                global: True
+                advanced: True
+                helpLink: elasticsearch
+              auto_expand_replicas:
+                description: Automatically expand the number of replicas based on the number of data nodes in the cluster. This can help ensure high availability as the cluster scales up or down.
+                forcedType: string
+                regex: "^(0-[1-9]|1-[2-9]|2-[3-9]|3-[4-9]|4-[5-9]|5-[6-9]|6-[7-9]|7-[89]|8-9|[0-9]-all|false)$"
+                regexFailureMessage: Must be in the format of "x-y" where x is minimum number of replicas and y is maximum number of replicas, or "0-all" to specify a minimum of 0 and no maximum, or "false" to disable automatic replica expansion.
+                global: True
+                advanced: True
+                helpLink: elasticsearch
+              mapping:
+                total_fields:
+                  limit:
+                    description: Max number of fields that can exist on a single index. Larger values will consume more resources.
+                    global: True
+                    advanced: True
+                    helpLink: elasticsearch
+              refresh_interval:
+                description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation.
+                global: True
+                advanced: True
+                helpLink: elasticsearch
+              number_of_shards:
+                description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs.
+                global: True
+                advanced: True
+                helpLink: elasticsearch
+              sort:
+                field:
+                  description: The field to sort by. Must set index_sorting to True.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+                order:
+                  description: The order to sort by. Must set index_sorting to True.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+          mappings:
+            _meta:
+              package:
+                name:
+                  description: Meta settings for the mapping.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+              managed_by:
+                  description: Meta settings for the mapping.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+              managed:
+                  description: Meta settings for the mapping.
+                  forcedType: bool
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+        composed_of:
+          description: The index template is composed of these component templates.
+          forcedType: "[]string"
+          global: True
+          advanced: True
+          helpLink: elasticsearch
+        priority:
+          description: The priority of the index template.
+          forcedType: int
+          global: True
+          advanced: True
+          helpLink: elasticsearch
+      policy:
+        phases:
+          hot:
+            min_age:
+              description: Minimum age of index. This determines when the index should be moved to the hot tier.
+              global: True
+              advanced: True
+              helpLink: elasticsearch
+            actions:
+              set_priority:
+                priority:
+                  description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
+                  forcedType: int
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+              rollover:
+                max_age:
+                  description: Maximum age of index.  Once an index reaches this limit, it will be rolled over into a new index.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+                max_primary_shard_size:
+                  description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+              shrink:
+                method:
+                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
+                  options:
+                  - COUNT
+                  - SIZE
+                  global: True
+                  advanced: True
+                  forcedType: string
+                number_of_shards:
+                  title: shard count
+                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                max_primary_shard_size:
+                  title: max shard size
+                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
+                  regex: ^[0-9]+(?:gb|tb|pb)$
+                  global: True
+                  forcedType: string
+                  advanced: True
+                allow_write_after_shrink:
+                  description: Allow writes after shrink.
+                  global: True
+                  forcedType: bool
+                  default: False
+                  advanced: True
+              forcemerge:
+                max_num_segments:
+                  description: Reduce the number of segments in each index shard and clean up deleted documents.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                index_codec:
+                  title: compression
+                  description: Use higher compression for stored fields at the cost of slower performance.
+                  forcedType: bool
+                  global: True
+                  default: False
+                  advanced: True
+          warm:
+            min_age:
+              description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
+              regex: ^[0-9]{1,5}d$
+              forcedType: string
+              global: True
+              advanced: True
+              helpLink: elasticsearch
+            actions:
+              set_priority:
+                priority:
+                  description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
+                  forcedType: int
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+              rollover:
+                max_age:
+                  description: Maximum age of index.  Once an index reaches this limit, it will be rolled over into a new index.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+                max_primary_shard_size:
+                  description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+              shrink:
+                method:
+                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
+                  options:
+                  - COUNT
+                  - SIZE
+                  global: True
+                  advanced: True
+                number_of_shards:
+                  title: shard count
+                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                max_primary_shard_size:
+                  title: max shard size
+                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
+                  regex: ^[0-9]+(?:gb|tb|pb)$
+                  global: True
+                  forcedType: string
+                  advanced: True
+                allow_write_after_shrink:
+                  description: Allow writes after shrink.
+                  global: True
+                  forcedType: bool
+                  default: False
+                  advanced: True
+              forcemerge:
+                max_num_segments:
+                  description: Reduce the number of segments in each index shard and clean up deleted documents.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                index_codec:
+                  title: compression
+                  description: Use higher compression for stored fields at the cost of slower performance.
+                  forcedType: bool
+                  global: True
+                  default: False
+                  advanced: True
+              allocate:
+                number_of_replicas:
+                  description: Set the number of replicas. Remains the same as the previous phase by default.
+                  forcedType: int
+                  global: True
+                  advanced: True
+          cold:
+            min_age:
+              description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
+              regex: ^[0-9]{1,5}d$
+              forcedType: string
+              global: True
+              advanced: True
+              helpLink: elasticsearch
+            actions:
+              set_priority:
+                priority:
+                  description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
+                  forcedType: int
+                  global: True
+                  advanced: True
+                  helpLink: elasticsearch
+              allocate:
+                number_of_replicas:
+                  description: Set the number of replicas. Remains the same as the previous phase by default.
+                  forcedType: int
+                  global: True
+                  advanced: True
+          delete:
+            min_age:
+              description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.
+              regex: ^[0-9]{1,5}d$
+              forcedType: string
+              global: True
+              advanced: True
+              helpLink: elasticsearch
+        _meta:
+          package:
+            name:
+              description: Meta settings for the mapping.
+              global: True
+              advanced: True
+              helpLink: elasticsearch
+          managed_by:
+            description: Meta settings for the mapping.
+            global: True
+            advanced: True
+            helpLink: elasticsearch
+          managed:
+            description: Meta settings for the mapping.
+            forcedType: bool
+            global: True
+            advanced: True
+            helpLink: elasticsearch
+    sos-backup: *indexSettings
+    so-detection: *indexSettings
+    so-assistant-chat: *indexSettings
+    so-assistant-session: *indexSettings
     so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings
       index_sorting:
         description: Sorts the index by event time, at the cost of additional processing resource consumption.
diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja
index ed1b49abe..7acb3eb87 100644
--- a/salt/elasticsearch/template.map.jinja
+++ b/salt/elasticsearch/template.map.jinja
@@ -5,6 +5,7 @@
 
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
 {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %}
+{% set DATA_RETENTION_METHOD = salt['pillar.get']('elasticsearch:data_retention_method', ELASTICSEARCHDEFAULTS.elasticsearch.get('data_retention_method', 'ILM')) %}
 
 {% set PILLAR_GLOBAL_OVERRIDES = {} %}
 {% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings', {}) %}
@@ -105,6 +106,17 @@
 {%     if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %}
 {%       do settings.index_template.template.settings.index.pop('sort') %}
 {%     endif %}
+{%     if DATA_RETENTION_METHOD == 'DLM' and settings.index_template.data_stream is defined and settings.data_stream_lifecycle is defined %}
+{%       if settings.data_stream_lifecycle.data_retention is defined and settings.data_stream_lifecycle.data_retention %}
+{%         do settings.index_template.template.update({'lifecycle': {'data_retention': settings.data_stream_lifecycle.data_retention}}) %}
+{%       else %}
+{%         do settings.index_template.template.update({'lifecycle': {}}) %}
+{%       endif %}
+{%       if settings.index_template.template.settings.index.lifecycle is not defined %}
+{%         do settings.index_template.template.settings.index.update({'lifecycle': {}}) %}
+{%       endif %}
+{%       do settings.index_template.template.settings.index.lifecycle.update({'prefer_ilm': false}) %}
+{%     endif %}
 {%   endif %}
 
 {# advanced ilm actions #}
diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load
index a0ebd66e8..4e87fd602 100755
--- a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load
+++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load
@@ -125,14 +125,6 @@ load_component_templates() {
     done
 }
 
-check_elasticsearch_responsive() {
-    # Cannot load templates if Elasticsearch is not responding.
-    #  NOTE: Slightly faster exit w/ failure than previous "retry 240 1" if there is a problem with Elasticsearch the
-    #    script should exit sooner rather than hang at the 'so-elasticsearch-templates' salt state.
-    retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" ||
-        fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot."
-}
-
 index_templates_exist() {
     local templates_dir="$1"
 
diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply
new file mode 100644
index 000000000..4943bdbff
--- /dev/null
+++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply
@@ -0,0 +1,135 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+{%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
+
+{%- set DATA_RETENTION_METHOD = salt['pillar.get']('elasticsearch:data_retention_method', ELASTICSEARCHDEFAULTS.elasticsearch.get('data_retention_method', 'ILM')) %}
+{%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
+{%- if GLOBALS.role != "so-heavynode" %}
+{%-   from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
+{%- endif %}
+{%- set DLM_STREAMS = [] %}
+{%- for index, settings in ES_INDEX_SETTINGS.items() %}
+{%-   if settings.index_template is defined and settings.index_template.data_stream is defined and settings.data_stream_lifecycle is defined %}
+{%-     do DLM_STREAMS.append({'template': index, 'data_retention': settings.data_stream_lifecycle.get('data_retention', '')}) %}
+{%-   endif %}
+{%- endfor %}
+{%- if GLOBALS.role != "so-heavynode" %}
+{%-   for index, settings in ALL_ADDON_SETTINGS.items() %}
+{%-     if settings.index_template is defined and settings.index_template.data_stream is defined and settings.data_stream_lifecycle is defined %}
+{%-       do DLM_STREAMS.append({'template': index, 'data_retention': settings.data_stream_lifecycle.get('data_retention', '')}) %}
+{%-     endif %}
+{%-   endfor %}
+{%- endif %}
+
+STREAM_CONFIG='{{ DLM_STREAMS | tojson }}'
+DATA_RETENTION_METHOD="{{ DATA_RETENTION_METHOD }}"
+DLM_FAILURES=0
+DLM_FAILURE_NAMES=()
+
+if [[ "$DATA_RETENTION_METHOD" != "DLM" && "$DATA_RETENTION_METHOD" != "ILM" ]]; then
+    echo "Unsupported data retention method $DATA_RETENTION_METHOD. Expected DLM or ILM."
+    exit 1
+fi
+
+set_data_stream_lifecycle() {
+    local data_stream="$1"
+    local data_retention="$2"
+    local body
+    local output
+
+    if [[ -n "$data_retention" ]]; then
+        if jq -e --arg data_stream "$data_stream" --arg data_retention "$data_retention" '.data_streams[]? | select(.name == $data_stream and .lifecycle.enabled == true and .lifecycle.data_retention == $data_retention)' >/dev/null 2>&1 <<< "$data_streams"; then
+            echo "DLM lifecycle already set for $data_stream with data_retention $data_retention, skipping."
+            return 0
+        fi
+    elif jq -e --arg data_stream "$data_stream" '.data_streams[]? | select(.name == $data_stream and .lifecycle.enabled == true and (.lifecycle.data_retention == null))' >/dev/null 2>&1 <<< "$data_streams"; then
+        echo "DLM lifecycle already set for $data_stream with indefinite retention, skipping."
+        return 0
+    fi
+
+    if [[ -n "$data_retention" ]]; then
+        body=$(jq -cn --arg data_retention "$data_retention" '{data_retention: $data_retention}')
+    else
+        # Setting indefinite retention
+        body='{}'
+    fi
+
+    if ! output=$(so-elasticsearch-query "_data_stream/${data_stream}/_lifecycle" -XPUT -d "$body" --retry 3 --retry-delay 5 --fail); then
+        echo "Failed to set data stream lifecycle for $data_stream."
+        echo "$output"
+        return 1
+    fi
+
+    if [[ -n "$data_retention" ]]; then
+        echo "Set DLM lifecycle for $data_stream with data_retention $data_retention."
+    else
+        echo "Set DLM lifecycle for $data_stream with indefinite retention."
+    fi
+}
+
+disable_data_stream_lifecycle() {
+    local data_stream="$1"
+    local body='{"enabled":false}'
+    local output
+
+    if ! jq -e --arg data_stream "$data_stream" '.data_streams[]? | select(.name == $data_stream and .lifecycle != null and .lifecycle.enabled != false)' >/dev/null 2>&1 <<< "$data_streams"; then
+        # No action needed
+        return 0
+    fi
+
+    if ! output=$(so-elasticsearch-query "_data_stream/${data_stream}/_lifecycle" -XPUT -d "$body" --retry 3 --retry-delay 5 --fail); then
+        echo "Failed to disable data stream lifecycle for $data_stream."
+        echo "$output"
+        return 1
+    fi
+
+    echo "Disabled DLM lifecycle for $data_stream."
+}
+
+process_data_stream() {
+    local data_stream="$1"
+    local data_retention="$2"
+
+    if [[ "$DATA_RETENTION_METHOD" == "DLM" ]]; then
+        set_data_stream_lifecycle "$data_stream" "$data_retention"
+    else
+        disable_data_stream_lifecycle "$data_stream"
+    fi
+}
+
+check_elasticsearch_responsive
+
+if ! data_streams=$(so-elasticsearch-query "_data_stream?format=json" --retry 3 --retry-delay 5 --fail); then
+    echo "Failed to retrieve data streams."
+    exit 1
+fi
+
+while read -r config; do
+    template=$(jq -r '.template' <<< "$config")
+    data_retention=$(jq -r '.data_retention // ""' <<< "$config")
+
+    while read -r data_stream; do
+        [[ -z "$data_stream" ]] && continue
+
+        if ! process_data_stream "$data_stream" "$data_retention"; then
+            DLM_FAILURES=$((DLM_FAILURES + 1))
+            DLM_FAILURE_NAMES+=("$data_stream")
+        fi
+    done <<< "$(jq -r --arg template "$template" '.data_streams[]? | select(.template == $template) | .name' <<< "$data_streams")"
+done <<< "$(jq -c '.[]' <<< "$STREAM_CONFIG")"
+
+if [[ $DLM_FAILURES -eq 0 ]]; then
+    echo "Data stream lifecycle updates completed successfully."
+else
+    echo "Encountered $DLM_FAILURES failure(s) updating data stream lifecycle:"
+    for failed_data_stream in "${DLM_FAILURE_NAMES[@]}"; do
+        echo "  - $failed_data_stream"
+    done
+    exit 1
+fi
diff --git a/salt/manager/managed_soc_annotations.sls b/salt/manager/managed_soc_annotations.sls
index b2fbb7334..46d727bf7 100644
--- a/salt/manager/managed_soc_annotations.sls
+++ b/salt/manager/managed_soc_annotations.sls
@@ -16,40 +16,35 @@
 {%       endif %}
 {%     endfor %}
 {%   endfor %}
+{%   set soc_annotation_lines = [] %}
+{%   set defaults_lines = [] %}
+{%   for k in matched_integration_names %}
+{%     do soc_annotation_lines.append('    ' ~ k ~ ': *dataStreamSettings') %}
+{%     do defaults_lines.append('    ' ~ k ~ ':') %}
+{%     set defaults_yaml = salt['slsutil.serialize']('yaml', ADDON_INTEGRATION_DEFAULTS[k], default_flow_style=False).strip() %}
+{%     for line in defaults_yaml.splitlines() %}
+{%       do defaults_lines.append('      ' ~ line) %}
+{%     endfor %}
+{%   endfor %}
 {%   set es_soc_annotations = '/opt/so/saltstack/default/salt/elasticsearch/soc_elasticsearch.yaml' %}
-{{   es_soc_annotations }}:
-     file.serialize:
-       - dataset:
-           {% set data = salt['file.read'](es_soc_annotations) | load_yaml %}
-           {% set es = data.get('elasticsearch', {}) %}
-           {% set index_settings = es.get('index_settings', {}) %}
-           {% set input = index_settings.get('so-logs', {}) %}
-           {% for k in matched_integration_names %}
-           {%   do index_settings.update({k: input}) %}
-           {% endfor %}
-           {% for k in addon_integration_keys %}
-           {%   if k not in matched_integration_names and k in index_settings %}
-           {%     do index_settings.pop(k) %}
-           {%   endif %}
-           {% endfor %}
-           {{ data }}
+manage_soc_annotations:
+  file.blockreplace:
+    - name: {{ es_soc_annotations }}
+    - marker_start: '    # START managed SOC integration annotations'
+    - marker_end: '    # END managed SOC integration annotations'
+    - content: {{ soc_annotation_lines | join('\n') | tojson }}
+    - insert_after_match: '^    # Managed SOC integration annotations are inserted below this line\.'
+    - append_if_not_found: False
+    - show_changes: True
 
 {#   Managed elasticsearch/defaults.yaml file for enabling 'Revert to default' via SOC UI for newly added config items #}
 {%   set es_defaults = '/opt/so/saltstack/default/salt/elasticsearch/defaults.yaml' %}
 {{   es_defaults }}:
-     file.serialize:
-       - dataset:
-           {% set data = salt['file.read'](es_defaults) | load_yaml %}
-           {% set es = data.get('elasticsearch', {}) %}
-           {% set index_settings = es.get('index_settings', {}) %}
-           {% for k in matched_integration_names %}
-           {%   set input = ADDON_INTEGRATION_DEFAULTS[k] %}
-           {%     do index_settings.update({k: input})%}
-           {% endfor %}
-           {% for k in addon_integration_keys %}
-           {%   if k not in matched_integration_names and k in index_settings %}
-           {%     do index_settings.pop(k) %}
-           {%   endif %}
-           {% endfor %}
-           {{ data }}
-{% endif %}
\ No newline at end of file
+  file.blockreplace:
+    - marker_start: '    # START managed SOC integration defaults'
+    - marker_end: '    # END managed SOC integration defaults'
+    - content: {{ defaults_lines | join('\n') | tojson }}
+    - insert_after_match: '^  index_settings:$'
+    - append_if_not_found: False
+    - show_changes: True
+{% endif %}
diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index d50187c9c..d955580fd 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -761,9 +761,57 @@ bootstrap_so_soc_database() {
   echo "so_soc bootstrap complete."
 }
 
+# Existing grids should keep ILM unless an admin explicitly opts in to DLM.
+pin_elasticsearch_data_retention_method() {
+  local elasticsearch_file=/opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls
+  mkdir -p "$(dirname "$elasticsearch_file")"
+  [[ -f "$elasticsearch_file" ]] || touch "$elasticsearch_file"
+
+  if so-yaml.py get -r "$elasticsearch_file" elasticsearch.data_retention_method >/dev/null 2>&1; then
+    echo "elasticsearch.data_retention_method already set; leaving as-is."
+    return 0
+  fi
+
+  echo "Pinning existing grid to ILM data retention."
+  so-yaml.py add "$elasticsearch_file" elasticsearch.data_retention_method ILM
+  chown socore:socore "$elasticsearch_file"
+}
+
+# Addes auto_expand_replicas setting to .kibana_streams index template
+#
+# In Kibana 9.3.3 the auto_expand_replicas setting was not added to the .kibana_streams index template. Causing single node deployments to be stuck in yellow state (unable to assign replica). Here we update the template in place using the so_kibana system user (system managed index template) to include the auto_expand_replicas setting
+#
+# Reference: https://github.com/elastic/kibana/issues/263048
+kibana_backport_streams_index_template() {
+    local current_template updated_template
+    current_template=$(so-elasticsearch-query "_index_template/.kibana_streams" --retry 3 --retry-delay 5 --fail)
+
+    if [[ -z "$current_template" ]]; then
+        echo "Unable to retrieve current .kibana_streams index template, skipping backport."
+        return 0
+    fi
+
+    updated_template=$(jq '.index_templates[0].index_template | .template.settings += {"index.auto_expand_replicas": "0-1"} | del(.created_date_millis, .modified_date_millis)' <<< "$current_template")
+
+    if ! kibana_user_pass=$(/usr/sbin/so-yaml.py get -r /opt/so/saltstack/local/pillar/elasticsearch/auth.sls elasticsearch.auth.users.so_kibana_user.pass); then
+        echo "Unable to retrieve so_kibana_user password, skipping .kibana_streams index template backport."
+        return 0
+    fi
+
+    if ! so-elasticsearch-query "_index_template/.kibana_streams" -XPUT -d "$updated_template" -u "so_kibana:$kibana_user_pass" --retry 3 --retry-delay 5 --fail; then
+        echo "Unable to automatically update .kibana_streams index template"
+        return 0
+    fi
+
+    ## NOTE: Should really add a check here for existing .kibana_streams index and then update its config in place
+
+}
+
 up_to_3.2.0() {
   fix_logstash_0013_lumberjack_pipeline_name
 
+  pin_elasticsearch_data_retention_method
+
   INSTALLEDVERSION=3.2.0
 }
 
@@ -774,6 +822,8 @@ post_to_3.2.0() {
   echo "Regenerating Elastic Agent Installers"
   /sbin/so-elastic-agent-gen-installers
 
+  kibana_backport_streams_index_template
+
   POSTVERSION=3.2.0
 }
 

From cf456dc58ca2d3da677e1c07b33f05f6562f8eeb Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 9 Jun 2026 23:21:43 -0500
Subject: [PATCH 03/28] reuse existing index templates

---
 .../sbin_jinja/so-elasticsearch-dlm-apply     | 103 +++++++++++++-----
 1 file changed, 73 insertions(+), 30 deletions(-)

diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply
index 4943bdbff..843fad625 100644
--- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply
+++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply
@@ -9,26 +9,16 @@
 {%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
 
 {%- set DATA_RETENTION_METHOD = salt['pillar.get']('elasticsearch:data_retention_method', ELASTICSEARCHDEFAULTS.elasticsearch.get('data_retention_method', 'ILM')) %}
-{%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
-{%- if GLOBALS.role != "so-heavynode" %}
-{%-   from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
-{%- endif %}
-{%- set DLM_STREAMS = [] %}
-{%- for index, settings in ES_INDEX_SETTINGS.items() %}
-{%-   if settings.index_template is defined and settings.index_template.data_stream is defined and settings.data_stream_lifecycle is defined %}
-{%-     do DLM_STREAMS.append({'template': index, 'data_retention': settings.data_stream_lifecycle.get('data_retention', '')}) %}
-{%-   endif %}
-{%- endfor %}
-{%- if GLOBALS.role != "so-heavynode" %}
-{%-   for index, settings in ALL_ADDON_SETTINGS.items() %}
-{%-     if settings.index_template is defined and settings.index_template.data_stream is defined and settings.data_stream_lifecycle is defined %}
-{%-       do DLM_STREAMS.append({'template': index, 'data_retention': settings.data_stream_lifecycle.get('data_retention', '')}) %}
-{%-     endif %}
-{%-   endfor %}
-{%- endif %}
 
-STREAM_CONFIG='{{ DLM_STREAMS | tojson }}'
-DATA_RETENTION_METHOD="{{ DATA_RETENTION_METHOD }}"
+ELASTICSEARCH_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR:-/opt/so/conf/elasticsearch/templates}"
+TEMPLATE_DIRS=(
+    "${ELASTICSEARCH_TEMPLATES_DIR}/index"
+    "${ELASTICSEARCH_TEMPLATES_DIR}/addon-index"
+)
+DATA_RETENTION_METHOD=$(cat <<'EOF'
+{{ DATA_RETENTION_METHOD }}
+EOF
+)
 DLM_FAILURES=0
 DLM_FAILURE_NAMES=()
 
@@ -37,6 +27,44 @@ if [[ "$DATA_RETENTION_METHOD" != "DLM" && "$DATA_RETENTION_METHOD" != "ILM" ]];
     exit 1
 fi
 
+validate_template_file() {
+    local template_file="$1"
+
+    if ! jq -e 'type == "object" and (.data_stream == null or (.data_stream | type == "object")) and (.template.lifecycle == null or (.template.lifecycle | type == "object")) and (.template.lifecycle.data_retention == null or (.template.lifecycle.data_retention | type == "string"))' >/dev/null 2>&1 "$template_file"; then
+        echo "Invalid index template JSON: $template_file"
+        return 1
+    fi
+}
+
+is_data_stream_template() {
+    jq -e '.data_stream | type == "object"' >/dev/null 2>&1 "$1"
+}
+
+has_data_stream_lifecycle() {
+    jq -e '.template.lifecycle | type == "object"' >/dev/null 2>&1 "$1"
+}
+
+get_data_retention() {
+    jq -r '.template.lifecycle.data_retention // ""' "$1"
+}
+
+find_template_file() {
+    local template="$1"
+    local template_dir
+    local template_file
+
+    for template_dir in "${TEMPLATE_DIRS[@]}"; do
+        template_file="${template_dir}/${template}-template.json"
+
+        if [[ -f "$template_file" ]]; then
+            echo "$template_file"
+            return 0
+        fi
+    done
+
+    return 1
+}
+
 set_data_stream_lifecycle() {
     local data_stream="$1"
     local data_retention="$2"
@@ -110,19 +138,34 @@ if ! data_streams=$(so-elasticsearch-query "_data_stream?format=json" --retry 3
     exit 1
 fi
 
-while read -r config; do
-    template=$(jq -r '.template' <<< "$config")
-    data_retention=$(jq -r '.data_retention // ""' <<< "$config")
+while read -r data_stream_config; do
+    data_stream=$(jq -r '.name' <<< "$data_stream_config")
+    template=$(jq -r '.template' <<< "$data_stream_config")
 
-    while read -r data_stream; do
-        [[ -z "$data_stream" ]] && continue
+    if ! template_file=$(find_template_file "$template"); then
+        echo "Skipping $data_stream: index template file not found for $template."
+        continue
+    fi
 
-        if ! process_data_stream "$data_stream" "$data_retention"; then
-            DLM_FAILURES=$((DLM_FAILURES + 1))
-            DLM_FAILURE_NAMES+=("$data_stream")
-        fi
-    done <<< "$(jq -r --arg template "$template" '.data_streams[]? | select(.template == $template) | .name' <<< "$data_streams")"
-done <<< "$(jq -c '.[]' <<< "$STREAM_CONFIG")"
+    validate_template_file "$template_file" || exit 1
+
+    if ! is_data_stream_template "$template_file"; then
+        echo "Skipping $data_stream: $template_file is not a data stream template."
+        continue
+    fi
+
+    if [[ "$DATA_RETENTION_METHOD" == "DLM" ]] && ! has_data_stream_lifecycle "$template_file"; then
+        echo "Skipping $data_stream: $template_file does not define data stream lifecycle."
+        continue
+    fi
+
+    data_retention=$(get_data_retention "$template_file")
+
+    if ! process_data_stream "$data_stream" "$data_retention"; then
+        DLM_FAILURES=$((DLM_FAILURES + 1))
+        DLM_FAILURE_NAMES+=("$data_stream")
+    fi
+done < <(jq -c '.data_streams[]' <<< "$data_streams")
 
 if [[ $DLM_FAILURES -eq 0 ]]; then
     echo "Data stream lifecycle updates completed successfully."

From 944e7737599326e7028bf4a3b075f876cfe29b32 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 10 Jun 2026 14:58:49 -0500
Subject: [PATCH 04/28] save exit until all packages have been attempted

---
 .../so-elastic-fleet-package-upgrade          | 23 ++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
index 18211a7c6..a89a0409b 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
@@ -8,18 +8,35 @@
 
 . /usr/sbin/so-elastic-fleet-common
 
+PKG_LOAD_FAILURES=0
+PKG_LOAD_FAILURES_NAMES=()
+
 {%- for PACKAGE in SUPPORTED_PACKAGES %}
 echo "Upgrading {{ PACKAGE }} package..."
 if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
     if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
         # exit 1 on failure to upgrade a default package, allow salt to handle retries
-        echo -e "\nERROR: Failed to upgrade $PACKAGE to version: $VERSION"
-        exit 1
+        echo -e "\nERROR: Failed to upgrade {{ PACKAGE }} to version: $VERSION"
+        PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
+        PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
     fi
 else
-    echo -e "\nERROR: Failed to get version information for integration $PACKAGE"
+    echo -e "\nERROR: Failed to get version information for integration {{ PACKAGE }}"
+    PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
+    PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
 fi
 echo
 {%- endfor %}
+
+if [ $PKG_LOAD_FAILURES -gt 0 ]; then
+    echo "ERROR: Failed to upgrade $PKG_LOAD_FAILURES package(s):"
+    for PKG in "${PKG_LOAD_FAILURES_NAMES[@]}"; do
+        echo " - $PKG"
+    done
+    exit 1
+else
+    echo "Successfully upgraded all packages."
+fi
+
 echo
 /usr/sbin/so-elasticsearch-templates-load

From bd5e77afc58fb6f14136d067b7ae3fa0045590ab Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 10 Jun 2026 14:59:29 -0500
Subject: [PATCH 05/28] increase delay in so-elastic-fleet-package-upgrade
 attempts

---
 salt/elasticfleet/manager.sls | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/elasticfleet/manager.sls b/salt/elasticfleet/manager.sls
index 6cb672bef..04430d496 100644
--- a/salt/elasticfleet/manager.sls
+++ b/salt/elasticfleet/manager.sls
@@ -55,7 +55,7 @@ so-elastic-fleet-package-upgrade:
     - name: /usr/sbin/so-elastic-fleet-package-upgrade
     - retry:
         attempts: 3
-        interval: 10
+        interval: 30
     - onchanges:
       - file: /opt/so/state/elastic_fleet_packages.txt
 

From f905afbc6fda4a94b2090452010d7f8cabf2143b Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 10 Jun 2026 15:01:22 -0500
Subject: [PATCH 06/28] logging

---
 .../tools/sbin_jinja/so-elastic-fleet-package-upgrade         | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
index a89a0409b..8ba250c00 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
@@ -15,13 +15,10 @@ PKG_LOAD_FAILURES_NAMES=()
 echo "Upgrading {{ PACKAGE }} package..."
 if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
     if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
-        # exit 1 on failure to upgrade a default package, allow salt to handle retries
-        echo -e "\nERROR: Failed to upgrade {{ PACKAGE }} to version: $VERSION"
         PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
         PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
     fi
 else
-    echo -e "\nERROR: Failed to get version information for integration {{ PACKAGE }}"
     PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
     PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
 fi
@@ -33,6 +30,7 @@ if [ $PKG_LOAD_FAILURES -gt 0 ]; then
     for PKG in "${PKG_LOAD_FAILURES_NAMES[@]}"; do
         echo " - $PKG"
     done
+    # exit 1 on failure to upgrade a default package, allow salt to handle retries
     exit 1
 else
     echo "Successfully upgraded all packages."

From 289ddda5e8054ef324f2d06a41922e86d5a38542 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 10 Jun 2026 17:06:22 -0500
Subject: [PATCH 07/28] kibana health check for fleet scripts

---
 salt/elasticfleet/enabled.sls                 | 11 ++++++++
 salt/elasticfleet/manager.sls                 | 21 +++++++++++++++
 .../so-elastic-fleet-integration-policy-load  |  7 +++--
 salt/kibana/enabled.sls                       |  3 +++
 salt/kibana/healthcheck.sls                   | 27 +++++++++++++++++++
 5 files changed, 67 insertions(+), 2 deletions(-)
 create mode 100644 salt/kibana/healthcheck.sls

diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls
index 166cb9719..60cf27deb 100644
--- a/salt/elasticfleet/enabled.sls
+++ b/salt/elasticfleet/enabled.sls
@@ -101,6 +101,17 @@ so-elastic-fleet:
       - file: trusttheca
       - x509: etc_elasticfleet_key
       - x509: etc_elasticfleet_crt
+
+wait_for_so-elastic-fleet:
+  http.wait_for_successful_query:
+    - name: "https://localhost:8220/api/status"
+    - ssl: True
+    - verify_ssl: False
+    - status: 200
+    - wait_for: 300
+    - request_interval: 15
+    - require:
+      - docker_container: so-elastic-fleet
 {%   endif %}
 
 delete_so-elastic-fleet_so-status.disabled:
diff --git a/salt/elasticfleet/manager.sls b/salt/elasticfleet/manager.sls
index 04430d496..a0aa83460 100644
--- a/salt/elasticfleet/manager.sls
+++ b/salt/elasticfleet/manager.sls
@@ -9,6 +9,7 @@
 
 include:
   - elasticfleet.config
+  - kibana.healthcheck
 
 # If enabled, automatically update Fleet Logstash Outputs
 {% if ELASTICFLEETMERGED.config.server.enable_auto_configuration %}
@@ -19,6 +20,8 @@ so-elastic-fleet-auto-configure-logstash-outputs:
     - retry:
         attempts: 4
         interval: 30
+    - require:
+      - http: wait_for_so-kibana
 {%   endif %}
 
 # If enabled, automatically update Fleet Server URLs & ES Connection
@@ -28,6 +31,8 @@ so-elastic-fleet-auto-configure-server-urls:
     - retry:
         attempts: 4
         interval: 30
+    - require:
+      - http: wait_for_so-kibana
 {% endif %}
 
 # Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
@@ -37,6 +42,8 @@ so-elastic-fleet-auto-configure-elasticsearch-urls:
     - retry:
         attempts: 4
         interval: 30
+    - require:
+      - http: wait_for_so-kibana
 
 so-elastic-fleet-auto-configure-artifact-urls:
   cmd.run:
@@ -44,6 +51,8 @@ so-elastic-fleet-auto-configure-artifact-urls:
     - retry:
         attempts: 4
         interval: 30
+    - require:
+      - http: wait_for_so-kibana
 
 so-elastic-fleet-package-statefile:
   file.managed:
@@ -56,6 +65,8 @@ so-elastic-fleet-package-upgrade:
     - retry:
         attempts: 3
         interval: 30
+    - require:
+      - http: wait_for_so-kibana
     - onchanges:
       - file: /opt/so/state/elastic_fleet_packages.txt
 
@@ -65,6 +76,8 @@ so-elastic-fleet-integrations:
     - retry:
         attempts: 3
         interval: 10
+    - require:
+      - http: wait_for_so-kibana
 
 so-elastic-agent-grid-upgrade:
   cmd.run:
@@ -72,6 +85,8 @@ so-elastic-agent-grid-upgrade:
     - retry:
         attempts: 12
         interval: 5
+    - require:
+      - http: wait_for_so-kibana
 
 so-elastic-fleet-integration-upgrade:
   cmd.run:
@@ -79,16 +94,22 @@ so-elastic-fleet-integration-upgrade:
     - retry:
         attempts: 3
         interval: 10
+    - require:
+      - http: wait_for_so-kibana
 
 {# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
 so-elastic-fleet-addon-integrations:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
+    - require:
+      - http: wait_for_so-kibana
 
 {% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
 so-elastic-defend-manage-filters-file-watch:
   cmd.run:
     - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log
+    - require:
+      - http: wait_for_so-kibana
     - onchanges:
       - file: elasticdefendcustom
       - file: elasticdefenddisabled
diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load
index e548c7f86..81a3c74be 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load
@@ -108,9 +108,12 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
   done
 
   # Only create the state file if all policies were created/updated successfully
-  if [[ "$RETURN_CODE" != "1" ]]; then
+  if [[ $RETURN_CODE -eq 0 ]]; then
     touch /opt/so/state/eaintegrations.txt
+  else
+    exit 1
   fi
 else
-  exit $RETURN_CODE
+  echo "Fleet integration policies already loaded."
+  exit 0
 fi
diff --git a/salt/kibana/enabled.sls b/salt/kibana/enabled.sls
index 04f44e508..4bb5fef9c 100644
--- a/salt/kibana/enabled.sls
+++ b/salt/kibana/enabled.sls
@@ -10,6 +10,7 @@
 
 include:
   - kibana.config
+  - kibana.healthcheck
   - kibana.sostatus
 
 # Start the kibana docker
@@ -59,6 +60,8 @@ so-kibana:
     {% endif %}
     - watch:
       - file: kibanaconfig
+    - require_in:
+      - http: wait_for_so-kibana
 
 delete_so-kibana_so-status.disabled:
   file.uncomment:
diff --git a/salt/kibana/healthcheck.sls b/salt/kibana/healthcheck.sls
new file mode 100644
index 000000000..f209c3964
--- /dev/null
+++ b/salt/kibana/healthcheck.sls
@@ -0,0 +1,27 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
+
+wait_for_so-kibana:
+  http.wait_for_successful_query:
+    - name: "https://localhost:5601/api/status"
+    - username: 'so_elastic'
+    - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}'
+    - ssl: True
+    - verify_ssl: False
+    - status: 200
+    - wait_for: 300
+    - request_interval: 15
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

From 46655860e926ad0b6692b5ba11ca6218b2d5b9e8 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 10 Jun 2026 17:27:23 -0500
Subject: [PATCH 08/28] http

---
 salt/kibana/healthcheck.sls | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/kibana/healthcheck.sls b/salt/kibana/healthcheck.sls
index f209c3964..bfeb633c5 100644
--- a/salt/kibana/healthcheck.sls
+++ b/salt/kibana/healthcheck.sls
@@ -9,7 +9,7 @@
 
 wait_for_so-kibana:
   http.wait_for_successful_query:
-    - name: "https://localhost:5601/api/status"
+    - name: "http://localhost:5601/api/status"
     - username: 'so_elastic'
     - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}'
     - ssl: True

From 4741cc92bd8d6d84d8551ac10331f091caef2b82 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 10 Jun 2026 17:52:08 -0500
Subject: [PATCH 09/28] fleet manager start kibana if it isn't already running
 and wait for healthly status

---
 salt/elasticfleet/manager.sls |  2 +-
 salt/kibana/enabled.sls       | 17 ++++++++++++++---
 salt/kibana/healthcheck.sls   | 27 ---------------------------
 3 files changed, 15 insertions(+), 31 deletions(-)
 delete mode 100644 salt/kibana/healthcheck.sls

diff --git a/salt/elasticfleet/manager.sls b/salt/elasticfleet/manager.sls
index a0aa83460..c9fe91d4d 100644
--- a/salt/elasticfleet/manager.sls
+++ b/salt/elasticfleet/manager.sls
@@ -9,7 +9,7 @@
 
 include:
   - elasticfleet.config
-  - kibana.healthcheck
+  - kibana.enabled
 
 # If enabled, automatically update Fleet Logstash Outputs
 {% if ELASTICFLEETMERGED.config.server.enable_auto_configuration %}
diff --git a/salt/kibana/enabled.sls b/salt/kibana/enabled.sls
index 4bb5fef9c..a2fb6cde9 100644
--- a/salt/kibana/enabled.sls
+++ b/salt/kibana/enabled.sls
@@ -6,11 +6,11 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 
 include:
   - kibana.config
-  - kibana.healthcheck
   - kibana.sostatus
 
 # Start the kibana docker
@@ -60,8 +60,19 @@ so-kibana:
     {% endif %}
     - watch:
       - file: kibanaconfig
-    - require_in:
-      - http: wait_for_so-kibana
+
+wait_for_so-kibana:
+  http.wait_for_successful_query:
+    - name: "http://localhost:5601/api/status"
+    - username: 'so_elastic'
+    - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}'
+    - ssl: True
+    - verify_ssl: False
+    - status: 200
+    - wait_for: 300
+    - request_interval: 15
+    - require:
+      - docker_container: so-kibana
 
 delete_so-kibana_so-status.disabled:
   file.uncomment:
diff --git a/salt/kibana/healthcheck.sls b/salt/kibana/healthcheck.sls
deleted file mode 100644
index bfeb633c5..000000000
--- a/salt/kibana/healthcheck.sls
+++ /dev/null
@@ -1,27 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
-
-wait_for_so-kibana:
-  http.wait_for_successful_query:
-    - name: "http://localhost:5601/api/status"
-    - username: 'so_elastic'
-    - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}'
-    - ssl: True
-    - verify_ssl: False
-    - status: 200
-    - wait_for: 300
-    - request_interval: 15
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

From f083db67e4b735aaad2e4a464115e003d2d767c8 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Thu, 11 Jun 2026 08:17:39 -0400
Subject: [PATCH 10/28] disable telemetry for automated tests

---
 setup/so-setup | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/setup/so-setup b/setup/so-setup
index 6c77e781c..f9c8c4460 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -223,6 +223,8 @@ if [ -n "$test_profile" ]; then
 	WEBPASSWD1=0n10nus3r
 	WEBPASSWD2=0n10nus3r
 	NODE_DESCRIPTION="${HOSTNAME} - ${install_type} - ${MSRVIP_OFFSET}"
+	# opt out of telemetry for automated testing
+	telemetry=1
 
 	update_sudoers_for_testing
 fi

From b8bf684077c8e06f0420807f220a530d1f433ed3 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Thu, 11 Jun 2026 08:18:38 -0400
Subject: [PATCH 11/28] ver

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 03e153fda..944880fa1 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-3.0.0-kilo
+3.2.0

From d9f6cde4e19d7d303a2d21418f4e1e5aa06d74ae Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 11 Jun 2026 15:11:29 -0500
Subject: [PATCH 12/28] remove global setting from data_retention annotation

---
 salt/elasticsearch/soc_elasticsearch.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml
index 251a0c1d8..c95a9467e 100644
--- a/salt/elasticsearch/soc_elasticsearch.yaml
+++ b/salt/elasticsearch/soc_elasticsearch.yaml
@@ -158,7 +158,6 @@ elasticsearch:
               - If retention is less than or equal to 14 days, max_age will be 1 day
               - If retention is less than or equal to 90 days, max_age will be 7 days
               - If retention is greater than 90 days, max_age will be 30 days
-          global: True
           forcedType: string
           regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$
           regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d.
@@ -353,7 +352,6 @@ elasticsearch:
               - If retention is less than or equal to 14 days, max_age will be 1 day
               - If retention is less than or equal to 90 days, max_age will be 7 days
               - If retention is greater than 90 days, max_age will be 30 days
-          global: True
           forcedType: string
           regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$
           regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d.

From c5051604807b1125367ebb223f8b3c7b8fe1778d Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 11 Jun 2026 15:13:28 -0500
Subject: [PATCH 13/28] set default DLM retention 90d

---
 salt/elasticsearch/defaults.yaml | 122 +++++++++++++++----------------
 1 file changed, 61 insertions(+), 61 deletions(-)

diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml
index 7b49074e8..ffb53ecbc 100644
--- a/salt/elasticsearch/defaults.yaml
+++ b/salt/elasticsearch/defaults.yaml
@@ -75,7 +75,7 @@ elasticsearch:
     global_overrides:
     # Tie this into cluster setting for data_streams.lifecycle.retention.default
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         template:
           settings:
@@ -157,7 +157,7 @@ elasticsearch:
     so-common:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -517,7 +517,7 @@ elasticsearch:
     so-idh:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -627,7 +627,7 @@ elasticsearch:
     so-import:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -811,7 +811,7 @@ elasticsearch:
     so-kismet:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - kismet-mappings
@@ -862,7 +862,7 @@ elasticsearch:
     so-kratos:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -932,7 +932,7 @@ elasticsearch:
     so-hydra:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -1079,7 +1079,7 @@ elasticsearch:
     so-logs:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - so-data-streams-mappings
@@ -1161,7 +1161,7 @@ elasticsearch:
     so-logs-detections_x_alerts:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - so-data-streams-mappings
@@ -1226,7 +1226,7 @@ elasticsearch:
     so-logs-elastic_agent:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1343,7 +1343,7 @@ elasticsearch:
     so-elastic-agent-monitor:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1407,7 +1407,7 @@ elasticsearch:
     so-logs-elastic_agent_x_apm_server:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.apm_server@package
@@ -1473,7 +1473,7 @@ elasticsearch:
     so-logs-elastic_agent_x_auditbeat:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.auditbeat@package
@@ -1539,7 +1539,7 @@ elasticsearch:
     so-logs-elastic_agent_x_cloudbeat:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.cloudbeat@package
@@ -1605,7 +1605,7 @@ elasticsearch:
     so-logs-elastic_agent_x_endpoint_security:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1666,7 +1666,7 @@ elasticsearch:
     so-logs-elastic_agent_x_filebeat:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1727,7 +1727,7 @@ elasticsearch:
     so-logs-elastic_agent_x_fleet_server:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1785,7 +1785,7 @@ elasticsearch:
     so-logs-elastic_agent_x_heartbeat:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.heartbeat@package
@@ -1851,7 +1851,7 @@ elasticsearch:
     so-logs-elastic_agent_x_metricbeat:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1912,7 +1912,7 @@ elasticsearch:
     so-logs-elastic_agent_x_osquerybeat:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1973,7 +1973,7 @@ elasticsearch:
     so-logs-elastic_agent_x_packetbeat:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.packetbeat@package
@@ -2039,7 +2039,7 @@ elasticsearch:
     so-logs-elasticsearch_x_server:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-elasticsearch.server@package
@@ -2105,7 +2105,7 @@ elasticsearch:
     so-logs-endpoint_x_actions:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - .logs-endpoint.actions@package
@@ -2166,7 +2166,7 @@ elasticsearch:
     so-logs-endpoint_x_action_x_responses:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - .logs-endpoint.action.responses@package
@@ -2227,7 +2227,7 @@ elasticsearch:
     so-logs-endpoint_x_alerts:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.alerts@package
@@ -2288,7 +2288,7 @@ elasticsearch:
     so-logs-endpoint_x_diagnostic_x_collection:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - .logs-endpoint.diagnostic.collection@package
@@ -2365,7 +2365,7 @@ elasticsearch:
     so-logs-endpoint_x_events_x_api:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.api@package
@@ -2426,7 +2426,7 @@ elasticsearch:
     so-logs-endpoint_x_events_x_file:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.file@package
@@ -2487,7 +2487,7 @@ elasticsearch:
     so-logs-endpoint_x_events_x_library:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.library@package
@@ -2548,7 +2548,7 @@ elasticsearch:
     so-logs-endpoint_x_events_x_network:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.network@package
@@ -2609,7 +2609,7 @@ elasticsearch:
     so-logs-endpoint_x_events_x_process:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.process@package
@@ -2670,7 +2670,7 @@ elasticsearch:
     so-logs-endpoint_x_events_x_registry:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.registry@package
@@ -2731,7 +2731,7 @@ elasticsearch:
     so-logs-endpoint_x_events_x_security:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.security@package
@@ -2792,7 +2792,7 @@ elasticsearch:
     so-logs-endpoint_x_heartbeat:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - .logs-endpoint.heartbeat@package
@@ -2853,7 +2853,7 @@ elasticsearch:
     so-logs-http_endpoint_x_generic:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-http_endpoint.generic@package
@@ -2903,7 +2903,7 @@ elasticsearch:
     so-logs-httpjson_x_generic:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-httpjson.generic@package
@@ -2970,7 +2970,7 @@ elasticsearch:
     so-logs-osquery-manager_x_action_x_responses:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         _meta:
           managed: true
@@ -3043,7 +3043,7 @@ elasticsearch:
     so-logs-osquery-manager_x_result:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         _meta:
           managed: true
@@ -3097,7 +3097,7 @@ elasticsearch:
     so-logs-soc:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -3207,7 +3207,7 @@ elasticsearch:
     so-logs-system_x_application:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3258,7 +3258,7 @@ elasticsearch:
     so-logs-system_x_auth:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3309,7 +3309,7 @@ elasticsearch:
     so-logs-system_x_security:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3360,7 +3360,7 @@ elasticsearch:
     so-logs-system_x_syslog:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3411,7 +3411,7 @@ elasticsearch:
     so-logs-system_x_system:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3462,7 +3462,7 @@ elasticsearch:
     so-logs-windows_x_forwarded:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-windows.forwarded@package
@@ -3511,7 +3511,7 @@ elasticsearch:
     so-logs-windows_x_powershell:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-windows.powershell@package
@@ -3560,7 +3560,7 @@ elasticsearch:
     so-logs-windows_x_powershell_operational:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-windows.powershell_operational@package
@@ -3609,7 +3609,7 @@ elasticsearch:
     so-logs-windows_x_sysmon_operational:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-windows.sysmon_operational@package
@@ -3658,7 +3658,7 @@ elasticsearch:
     so-logs-winlog_x_winlog:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - logs-winlog.winlog@package
@@ -3708,7 +3708,7 @@ elasticsearch:
     so-logstash:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -3825,7 +3825,7 @@ elasticsearch:
     so-metrics-endpoint_x_metadata:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - metrics-endpoint.metadata@package
@@ -3874,7 +3874,7 @@ elasticsearch:
     so-metrics-endpoint_x_metrics:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - metrics-endpoint.metrics@package
@@ -3923,7 +3923,7 @@ elasticsearch:
     so-metrics-endpoint_x_policy:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - metrics-endpoint.policy@package
@@ -3972,7 +3972,7 @@ elasticsearch:
     so-metrics-fleet_server_x_agent_status:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - metrics@tsdb-settings
@@ -3998,7 +3998,7 @@ elasticsearch:
     so-metrics-fleet_server_x_agent_versions:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - metrics@tsdb-settings
@@ -4024,7 +4024,7 @@ elasticsearch:
     so-redis:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4141,7 +4141,7 @@ elasticsearch:
     so-strelka:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4260,7 +4260,7 @@ elasticsearch:
     so-suricata:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4378,7 +4378,7 @@ elasticsearch:
     so-suricata_x_alerts:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4496,7 +4496,7 @@ elasticsearch:
     so-syslog:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4614,7 +4614,7 @@ elasticsearch:
     so-zeek:
       index_sorting: false
       data_stream_lifecycle:
-        data_retention: 7d
+        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings

From 80c39d612cb65d8c10fabdcb76e0a4f886b37bd8 Mon Sep 17 00:00:00 2001
From: Mike Reeves <reevesmk@gmail.com>
Date: Thu, 11 Jun 2026 18:40:43 -0400
Subject: [PATCH 14/28] Pin NIC names by MAC via udev (run-once) from the
 common state

Add so-nic-pin, which writes by-MAC persistent-net udev rules pinning each
physical NIC to its current name so a kernel upgrade can't renumber the
interfaces Security Onion binds by name (host:mainint, sensor:mainint, bond0).

Gated by the drop file /opt/so/state/nic_names_pinned: run-once on highstate,
and an admin can pre-create the marker to opt out. Wired into common/init.sls
as pin_nic_names, guarded by a matching unless.
---
 salt/common/init.sls              | 11 +++++
 salt/common/tools/sbin/so-nic-pin | 76 +++++++++++++++++++++++++++++++
 2 files changed, 87 insertions(+)
 create mode 100644 salt/common/tools/sbin/so-nic-pin

diff --git a/salt/common/init.sls b/salt/common/init.sls
index 120e73e3e..9618d2c67 100644
--- a/salt/common/init.sls
+++ b/salt/common/init.sls
@@ -130,6 +130,17 @@ common_sbin:
       - so-pcap-import
 {% endif %}
 
+# Pin physical NIC names by MAC (run-once) so a kernel upgrade can't renumber the
+# interfaces SO binds by name. The marker keeps it a one-time setup; an admin can
+# pre-create the marker to opt out.
+pin_nic_names:
+  cmd.run:
+    - name: /usr/sbin/so-nic-pin
+    - unless: 'test -e /opt/so/state/nic_names_pinned'
+    - require:
+      - file: common_sbin
+      - file: statedir
+
 common_sbin_jinja:
   file.recurse:
     - name: /usr/sbin
diff --git a/salt/common/tools/sbin/so-nic-pin b/salt/common/tools/sbin/so-nic-pin
new file mode 100644
index 000000000..9e2b4e13a
--- /dev/null
+++ b/salt/common/tools/sbin/so-nic-pin
@@ -0,0 +1,76 @@
+#!/bin/bash
+#
+# so-nic-pin — pin physical NIC names by permanent MAC via classic by-MAC udev
+#              rules, so a kernel upgrade can't renumber them.
+#
+# Security Onion binds its management and monitor interfaces BY NAME in pillar
+# (host:mainint, sensor:mainint, and bond0 is built on a specific physical NIC).
+# A kernel upgrade can change the kernel/systemd-udevd predictable-naming output
+# and renumber those NICs (e.g. enp1s0 -> enp2s0), which breaks the grid: the
+# pillar references a name that no longer exists and bond/bridge bring-up fails.
+#
+# This writes /etc/udev/rules.d/70-persistent-net.rules pinning each PHYSICAL NIC
+# to its CURRENT name by its PERMANENT MAC, freezing the names across future kernel
+# changes. It only writes the rules file; it does NOT live-trigger a rename (the
+# rules apply on the next boot/kernel, and a live rename would be disruptive).
+#
+# Run-once: gated by the drop file /opt/so/state/nic_names_pinned. If the marker is
+# present the script does nothing, so an admin can pre-create it to opt out. Invoked
+# from the common state on every highstate; the marker keeps it a one-time setup.
+
+NET_RULES_FILE="/etc/udev/rules.d/70-persistent-net.rules"
+MARKER="/opt/so/state/nic_names_pinned"
+
+log() { echo -e "[so-nic-pin] $*"; }
+
+# Echo "<name> <permanent-mac>" for every PHYSICAL NIC. A physical NIC is backed by a
+# real device (has device/driver), which excludes bond0/sobridge/docker0/veth*/lo whose
+# MACs are dynamic and must never be pinned. The PERMANENT MAC is used (ethtool -P, with
+# fallbacks), not the current one: an enslaved bond member's current MAC is rewritten to
+# the bond's, so matching on it would be wrong/ambiguous.
+physical_nics() {
+    local path n mac
+    for path in /sys/class/net/*; do
+        n="${path##*/}"
+        [ "$n" = "lo" ] && continue
+        [ -e "${path}/device/driver" ] || continue          # real device only
+        mac="$(ethtool -P "$n" 2>/dev/null | awk '/Permanent address/{print $NF}')"
+        case "$mac" in ""|00:00:00:00:00:00) mac="$(cat "${path}/bonding_slave/perm_hwaddr" 2>/dev/null)" ;; esac
+        case "$mac" in ""|00:00:00:00:00:00) mac="$(cat "${path}/address" 2>/dev/null)" ;; esac
+        case "$mac" in ""|00:00:00:00:00:00) continue ;; esac
+        echo "$n $mac"
+    done
+}
+
+# Turn "<name> <mac>" lines on stdin into classic by-MAC persistent-net udev rules.
+render_net_rules() {
+    echo "# Generated by so-nic-pin: pin NIC names by MAC so kernel upgrades can't renumber them."
+    echo "# Security Onion binds its management/monitor interfaces by name; do not hand-edit."
+    local n mac
+    while read -r n mac; do
+        [ -n "$n" ] || continue
+        printf 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="%s", NAME="%s"\n' \
+            "$mac" "$n"
+    done
+}
+
+[ "$(id -u)" -eq 0 ] || exit 0                   # salt runs us as root; bail quietly otherwise
+[ -e "${MARKER}" ] && exit 0                      # run-once guard (mirrors the state's unless)
+
+nics="$(physical_nics)"
+if [ -z "${nics}" ]; then
+    log "no physical NICs detected — nothing to pin (will retry on next highstate)"
+    exit 0                                         # do NOT drop the marker; let it retry later
+fi
+
+log "pinning physical NICs by permanent MAC:"
+echo "${nics}" | sed 's/^/    /'
+
+[ -f "${NET_RULES_FILE}" ] && cp -f "${NET_RULES_FILE}" "${NET_RULES_FILE}.bak"
+echo "${nics}" | render_net_rules > "${NET_RULES_FILE}" || {
+    log "ERROR: failed to write ${NET_RULES_FILE}"
+    exit 1
+}
+
+mkdir -p "$(dirname "${MARKER}")" && touch "${MARKER}"
+log "wrote ${NET_RULES_FILE} ($(grep -c '^SUBSYSTEM' "${NET_RULES_FILE}") NIC(s) pinned); dropped ${MARKER}"

From 9031c1fd220acd1a074412e1241fa81ddc15475f Mon Sep 17 00:00:00 2001
From: Josh Brower <josh@defensivedepth.com>
Date: Fri, 12 Jun 2026 11:18:59 -0400
Subject: [PATCH 15/28] userid vs names

---
 salt/kafka/enabled.sls    | 2 +-
 salt/kibana/enabled.sls   | 2 +-
 salt/logstash/enabled.sls | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/salt/kafka/enabled.sls b/salt/kafka/enabled.sls
index 06fa701c6..504efea05 100644
--- a/salt/kafka/enabled.sls
+++ b/salt/kafka/enabled.sls
@@ -32,7 +32,7 @@ so-kafka:
     - networks:
       - sobridge:
         - ipv4_address: {{ DOCKERMERGED.containers['so-kafka'].ip }}
-    - user: kafka
+    - user: "960"
     - environment:
         KAFKA_HEAP_OPTS: -Xmx2G -Xms1G
         KAFKA_OPTS: "-javaagent:/opt/jolokia/agents/jolokia-agent-jvm-javaagent.jar=port=8778,host={{ DOCKERMERGED.containers['so-kafka'].ip }},policyLocation=file:/opt/jolokia/jolokia.xml {%- if KAFKA_EXTERNAL_ACCESS %} -Djava.security.auth.login.config=/opt/kafka/config/kafka_server_jaas.conf {% endif -%}"
diff --git a/salt/kibana/enabled.sls b/salt/kibana/enabled.sls
index a2fb6cde9..1257f66c6 100644
--- a/salt/kibana/enabled.sls
+++ b/salt/kibana/enabled.sls
@@ -18,7 +18,7 @@ so-kibana:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kibana:{{ GLOBALS.so_version }}
     - hostname: kibana
-    - user: kibana
+    - user: "932:0"
     - networks:
       - sobridge:
         - ipv4_address: {{ DOCKERMERGED.containers['so-kibana'].ip }}
diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls
index d89304144..60fed4ced 100644
--- a/salt/logstash/enabled.sls
+++ b/salt/logstash/enabled.sls
@@ -33,7 +33,7 @@ so-logstash:
     - networks:
       - sobridge:
         - ipv4_address: {{ DOCKERMERGED.containers['so-logstash'].ip }}
-    - user: logstash
+    - user: "931:0"
     - extra_hosts:
     {% for node in LOGSTASH_NODES %}
     {%   for hostname, ip in node.items() %}

From ae1ddf38173d2e46eea36233b7797bcadea3ddf0 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Mon, 15 Jun 2026 12:33:08 -0400
Subject: [PATCH 16/28] es|ql defaults

---
 salt/soc/defaults.yaml | 1 +
 salt/soc/soc_soc.yaml  | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml
index c9399eab4..7e8e76094 100644
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -1464,6 +1464,7 @@ soc:
           sigmaRulePackages:
             - core
             - emerging_threats_addon
+          useEsql: false
         elastic:
           hostUrl:
           remoteHostUrls: []
diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml
index b2ac6d175..19853196a 100644
--- a/salt/soc/soc_soc.yaml
+++ b/salt/soc/soc_soc.yaml
@@ -383,6 +383,11 @@ soc:
             global: True
             advanced: False
             helpLink: sigma
+          useEsql:
+            description: "(Pre-release) Use Elasticsearch Piped Query Language (ES|QL) instead of EQL (Elastic Query Language) for Elasticsearch queries. The Sigma converter will output ES|QL instead of EQL, allowing support for correlations."
+            global: True
+            advanced: True
+            forcedType: bool
         elastic:
           index:
             description: Comma-separated list of indices or index patterns (wildcard "*" supported) that SOC will search for records.

From d10f21399c72bff0b16d80dbae9a7f3373383d8d Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Mon, 15 Jun 2026 13:31:23 -0500
Subject: [PATCH 17/28] remove comments

---
 salt/elasticsearch/defaults.yaml              | 20 +++++++------------
 .../sbin_jinja/so-elasticsearch-dlm-apply     |  2 --
 salt/manager/tools/sbin/soup                  |  2 --
 3 files changed, 7 insertions(+), 17 deletions(-)

diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml
index ffb53ecbc..f0b01b3ca 100644
--- a/salt/elasticsearch/defaults.yaml
+++ b/salt/elasticsearch/defaults.yaml
@@ -19,18 +19,9 @@ elasticsearch:
               flood_stage: 90%
               high: 85%
               low: 80%
-    # don't want to set retention here since it will make ES restart with every update +
-    #   potentially case where we could unintentially fall back to retention 7d and cause data loss
-    # data_streams:
-    #   lifecycle:
-    #     retention:
-    #       default: 7d
     indices:
       id_field_data:
         enabled: false
-    # index:
-      # lifecycle:
-        # prefer_ilm: true
     logger:
       org:
         elasticsearch:
@@ -73,7 +64,6 @@ elasticsearch:
             verification_mode: none
   index_settings:
     global_overrides:
-    # Tie this into cluster setting for data_streams.lifecycle.retention.default
       data_stream_lifecycle:
         data_retention: 90d
       index_template:
@@ -2110,6 +2100,7 @@ elasticsearch:
         composed_of:
         - .logs-endpoint.actions@package
         - .logs-endpoint.actions@custom
+        - endpoint@custom
         - event-mappings
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -2119,8 +2110,9 @@ elasticsearch:
           hidden: false
         ignore_missing_component_templates:
         - .logs-endpoint.actions@custom
+        - endpoint@custom
         index_patterns:
-        - logs-endpoint.actions-*
+        - .logs-endpoint.actions-*
         priority: 501
         template:
           settings:
@@ -2171,6 +2163,7 @@ elasticsearch:
         composed_of:
         - .logs-endpoint.action.responses@package
         - .logs-endpoint.action.responses@custom
+        - endpoint@custom
         - event-mappings
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -2180,14 +2173,15 @@ elasticsearch:
           hidden: false
         ignore_missing_component_templates:
         - .logs-endpoint.action.responses@custom
+        - endpoint@custom
         index_patterns:
-        - logs-endpoint.action.responses-*
+        - .logs-endpoint.action.responses-*
         priority: 501
         template:
           settings:
             index:
               lifecycle:
-                name: so-logs-endpoint.actions-logs
+                name: so-logs-endpoint.action.responses-logs
               mapping:
                 total_fields:
                   limit: 5000
diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply
index 843fad625..af761973c 100644
--- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply
+++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply
@@ -90,7 +90,6 @@ set_data_stream_lifecycle() {
 
     if ! output=$(so-elasticsearch-query "_data_stream/${data_stream}/_lifecycle" -XPUT -d "$body" --retry 3 --retry-delay 5 --fail); then
         echo "Failed to set data stream lifecycle for $data_stream."
-        echo "$output"
         return 1
     fi
 
@@ -113,7 +112,6 @@ disable_data_stream_lifecycle() {
 
     if ! output=$(so-elasticsearch-query "_data_stream/${data_stream}/_lifecycle" -XPUT -d "$body" --retry 3 --retry-delay 5 --fail); then
         echo "Failed to disable data stream lifecycle for $data_stream."
-        echo "$output"
         return 1
     fi
 
diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index d955580fd..b0f2610b7 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -803,8 +803,6 @@ kibana_backport_streams_index_template() {
         return 0
     fi
 
-    ## NOTE: Should really add a check here for existing .kibana_streams index and then update its config in place
-
 }
 
 up_to_3.2.0() {

From 596471e1404b8287b453e5892ba0e82fc72fea6f Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Mon, 15 Jun 2026 13:31:53 -0500
Subject: [PATCH 18/28] using new annotation config

---
 salt/elasticsearch/soc_elasticsearch.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml
index c95a9467e..46e836c9c 100644
--- a/salt/elasticsearch/soc_elasticsearch.yaml
+++ b/salt/elasticsearch/soc_elasticsearch.yaml
@@ -159,6 +159,8 @@ elasticsearch:
               - If retention is less than or equal to 90 days, max_age will be 7 days
               - If retention is greater than 90 days, max_age will be 30 days
           forcedType: string
+          allowedNodeTypes:
+            - heavynode
           regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$
           regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d.
       index_template:
@@ -353,6 +355,8 @@ elasticsearch:
               - If retention is less than or equal to 90 days, max_age will be 7 days
               - If retention is greater than 90 days, max_age will be 30 days
           forcedType: string
+          allowedNodeTypes:
+            - heavynode
           regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$
           regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d.
       index_template:

From 95cae4c734ec04ed06ff7ac96c7d6572e062f639 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Mon, 15 Jun 2026 13:32:45 -0500
Subject: [PATCH 19/28] remove so-elasticsearch-indices-delete cron when using
 DLM

---
 salt/elasticsearch/cluster.sls | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/salt/elasticsearch/cluster.sls b/salt/elasticsearch/cluster.sls
index efa50285e..a8ccd3780 100644
--- a/salt/elasticsearch/cluster.sls
+++ b/salt/elasticsearch/cluster.sls
@@ -165,7 +165,8 @@ so-elasticsearch-roles-load:
 {%    set ap = "absent" %}
 {%  endif %}
 {%  if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %}
-{%    if ELASTICSEARCHMERGED.index_clean %}
+{#    Remove so-elasticsearch-indices-delete script when using DLM #}
+{%    if ELASTICSEARCHMERGED.index_clean and ELASTICSEARCHMERGED.data_retention_method == "ILM" %}
 {%      set ap = "present" %}
 {%    else %}
 {%      set ap = "absent" %}

From 1a423a243401944cc6493f84e9510243d6bc5be6 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Mon, 15 Jun 2026 14:17:34 -0500
Subject: [PATCH 20/28] update message

---
 salt/manager/tools/sbin/soup | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index b0f2610b7..0016cc00e 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -787,7 +787,7 @@ kibana_backport_streams_index_template() {
     current_template=$(so-elasticsearch-query "_index_template/.kibana_streams" --retry 3 --retry-delay 5 --fail)
 
     if [[ -z "$current_template" ]]; then
-        echo "Unable to retrieve current .kibana_streams index template, skipping backport."
+        echo "Index template .kibana_streams does not exist, skipping backport."
         return 0
     fi
 

From f68e3e47a1f7d0d29096ec22fc39bec3f2d2b699 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 16 Jun 2026 09:19:10 -0500
Subject: [PATCH 21/28] remove pillar merge

---
 salt/elasticsearch/template.map.jinja                        | 2 +-
 .../tools/sbin_jinja/so-elasticsearch-dlm-apply              | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja
index 7acb3eb87..15481ee4c 100644
--- a/salt/elasticsearch/template.map.jinja
+++ b/salt/elasticsearch/template.map.jinja
@@ -5,7 +5,7 @@
 
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
 {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %}
-{% set DATA_RETENTION_METHOD = salt['pillar.get']('elasticsearch:data_retention_method', ELASTICSEARCHDEFAULTS.elasticsearch.get('data_retention_method', 'ILM')) %}
+{% set DATA_RETENTION_METHOD = salt['pillar.get']('elasticsearch:data_retention_method', ELASTICSEARCHDEFAULTS.elasticsearch.data_retention_method) %}
 
 {% set PILLAR_GLOBAL_OVERRIDES = {} %}
 {% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings', {}) %}
diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply
index af761973c..e25e07223 100644
--- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply
+++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply
@@ -6,9 +6,8 @@
 
 . /usr/sbin/so-common
 
-{%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
-
-{%- set DATA_RETENTION_METHOD = salt['pillar.get']('elasticsearch:data_retention_method', ELASTICSEARCHDEFAULTS.elasticsearch.get('data_retention_method', 'ILM')) %}
+{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
+{%- set DATA_RETENTION_METHOD = ELASTICSEARCHMERGED.data_retention_method %}
 
 ELASTICSEARCH_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR:-/opt/so/conf/elasticsearch/templates}"
 TEMPLATE_DIRS=(

From a769d4c68094c8b19fcb455f0ece69953ba9d7fc Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 16 Jun 2026 09:32:37 -0500
Subject: [PATCH 22/28] another unneeded default

---
 salt/elasticsearch/template.map.jinja | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja
index 15481ee4c..72be1ec58 100644
--- a/salt/elasticsearch/template.map.jinja
+++ b/salt/elasticsearch/template.map.jinja
@@ -4,8 +4,11 @@
    Elastic License 2.0. #}
 
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
+{# ELASTICSEARCHMERGED only used here to collect data_retention_method. This file intentionally works with ELASTICSEARCHDEFAULTS #}
+{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
+
 {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %}
-{% set DATA_RETENTION_METHOD = salt['pillar.get']('elasticsearch:data_retention_method', ELASTICSEARCHDEFAULTS.elasticsearch.data_retention_method) %}
+{% set DATA_RETENTION_METHOD = ELASTICSEARCHMERGED.data_retention_method %}
 
 {% set PILLAR_GLOBAL_OVERRIDES = {} %}
 {% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings', {}) %}

From 4a6c6752236a5b3cf98f7717f8a9e4b3d7a59169 Mon Sep 17 00:00:00 2001
From: Jorge Reyes <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 16 Jun 2026 10:33:11 -0500
Subject: [PATCH 23/28] skip kibana backport if the template doesn't exist

---
 salt/manager/tools/sbin/soup | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 0016cc00e..668d7d1de 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -784,9 +784,8 @@ pin_elasticsearch_data_retention_method() {
 # Reference: https://github.com/elastic/kibana/issues/263048
 kibana_backport_streams_index_template() {
     local current_template updated_template
-    current_template=$(so-elasticsearch-query "_index_template/.kibana_streams" --retry 3 --retry-delay 5 --fail)
 
-    if [[ -z "$current_template" ]]; then
+    if ! current_template=$(so-elasticsearch-query "_index_template/.kibana_streams" --retry 3 --retry-delay 5 --fail); then
         echo "Index template .kibana_streams does not exist, skipping backport."
         return 0
     fi

From 4456bde1c820c2c5cbda1054e1398c010fed375b Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 16 Jun 2026 10:45:53 -0500
Subject: [PATCH 24/28] check if template exists without --fail flag

---
 salt/manager/tools/sbin/soup | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 668d7d1de..12be85eaf 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -783,10 +783,16 @@ pin_elasticsearch_data_retention_method() {
 #
 # Reference: https://github.com/elastic/kibana/issues/263048
 kibana_backport_streams_index_template() {
-    local current_template updated_template
+    local current_template updated_template current_template_exists
+
+    current_template_exists=$(so-elasticsearch-query "_index_template/.kibana_streams" -o /dev/null -w "%{http_code}")
+    if [[ "$current_template_exists" == "404" ]]; then
+        echo "Index template .kibana_streams does not exist, skipping backport."
+        return 0
+    fi
 
     if ! current_template=$(so-elasticsearch-query "_index_template/.kibana_streams" --retry 3 --retry-delay 5 --fail); then
-        echo "Index template .kibana_streams does not exist, skipping backport."
+        echo "Unable to retrieve .kibana_streams index template, skipping backport."
         return 0
     fi
 

From 3daed551df0a3e435be6f98b341630f808e2d88d Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 16 Jun 2026 11:17:04 -0500
Subject: [PATCH 25/28] use --fail flag without set -x, since elasticsearch can
 return a 404 on the template lookup

---
 salt/manager/tools/sbin/soup | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 12be85eaf..96313aea4 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -783,18 +783,14 @@ pin_elasticsearch_data_retention_method() {
 #
 # Reference: https://github.com/elastic/kibana/issues/263048
 kibana_backport_streams_index_template() {
-    local current_template updated_template current_template_exists
+    local current_template updated_template
 
-    current_template_exists=$(so-elasticsearch-query "_index_template/.kibana_streams" -o /dev/null -w "%{http_code}")
-    if [[ "$current_template_exists" == "404" ]]; then
+    set +e
+    if ! current_template=$(so-elasticsearch-query "_index_template/.kibana_streams" --retry 3 --retry-delay 5 --fail); then
         echo "Index template .kibana_streams does not exist, skipping backport."
         return 0
     fi
-
-    if ! current_template=$(so-elasticsearch-query "_index_template/.kibana_streams" --retry 3 --retry-delay 5 --fail); then
-        echo "Unable to retrieve .kibana_streams index template, skipping backport."
-        return 0
-    fi
+    set -e
 
     updated_template=$(jq '.index_templates[0].index_template | .template.settings += {"index.auto_expand_replicas": "0-1"} | del(.created_date_millis, .modified_date_millis)' <<< "$current_template")
 

From 6a18f35020173a2f68fb9a1c087382e219c27042 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 16 Jun 2026 17:47:46 -0500
Subject: [PATCH 26/28] add context to soup errors and optional soup debug log
 with xtrace output

---
 salt/manager/tools/sbin/soup | 58 +++++++++++++++++++++++++++++++++++-
 1 file changed, 57 insertions(+), 1 deletion(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 96313aea4..a64524bff 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -16,6 +16,7 @@ POSTVERSION=$INSTALLEDVERSION
 INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
 BATCHSIZE=5
 SOUP_LOG=/root/soup.log
+SOUP_DEBUG_LOG=/root/soup-debug.log
 WHATWOULDYOUSAYYAHDOHERE=soup
 whiptail_title='Security Onion UPdater'
 NOTIFYCUSTOMELASTICCONFIG=false
@@ -34,6 +35,7 @@ if [[ -f /etc/salt/cloud.profiles.d/socloud.conf ]]; then
 fi
 # used to display messages to the user at the end of soup
 declare -a FINAL_MESSAGE_QUEUE=()
+SOUP_ERR_CONTEXT=
 
 
 check_err() {
@@ -114,11 +116,49 @@ check_err() {
       echo "$err_msg"
     fi
 
+    if [[ -n $SOUP_ERR_CONTEXT ]]; then
+      echo ""
+      printf '%s\n' "$SOUP_ERR_CONTEXT"
+    fi
+
+    echo "SOUP XTRACE debug log (if enabled) at $SOUP_DEBUG_LOG. Re-run soup with SOUP_DEBUG=1 to create $SOUP_DEBUG_LOG"
+
     exit $exit_code
   fi
 
 }
 
+# Collect bash error context before passing off to check_err()
+on_err() {
+  local exit_code=$?
+  # Use first error context, multiple errors can happen with command substitutions or nested functions. We just need context from the initial error.
+  [[ -n $SOUP_ERR_CONTEXT ]] && return $exit_code
+  # turn off xtrace to prevent added noise in debug log
+  [[ $- == *x* ]] && set +x
+
+  local cmd=$BASH_COMMAND
+  local line=${BASH_LINENO[0]}
+  local function=${FUNCNAME[1]:-main}
+  local source=${BASH_SOURCE[1]##*/}
+  local -a err_lines=(
+    "ERROR on: ${cmd}"
+    "  source: ${source}:${line} in ${function}()"
+  )
+  local i caller_line caller_src caller_func
+
+  for ((i=2; i<${#FUNCNAME[@]}-1; i++)); do
+    caller_line=${BASH_LINENO[$((i-1))]}
+    [[ -n $caller_line && $caller_line -gt 0 ]] || continue
+    caller_src=${BASH_SOURCE[$i]##*/}
+    caller_func=${FUNCNAME[$i]:-main}
+    err_lines+=("  called by: ${caller_src}:${caller_line} in ${caller_func}()")
+  done
+
+  SOUP_ERR_CONTEXT=$(printf '%s\n' "${err_lines[@]}")
+
+  return $exit_code
+}
+
 airgap_mounted() {
   # Let's see if the ISO is already mounted.
   if [[ -f /tmp/soagupdate/SecurityOnion/VERSION ]]; then
@@ -1982,4 +2022,20 @@ EOF
   read -r input
 fi
 
-main "$@" | tee -a $SOUP_LOG
+set -o errtrace
+trap on_err ERR
+
+if [[ $SOUP_DEBUG == 1 ]]; then
+  if [ -f $SOUP_DEBUG_LOG ]; then
+    current_time=$(date +%Y%m%d.%H%M%S)
+    mv $SOUP_DEBUG_LOG $SOUP_DEBUG_LOG.$INSTALLEDVERSION.$current_time
+  fi
+  exec {SOUP_XTRACE_FD}>>"$SOUP_DEBUG_LOG"
+  export SOUP_XTRACE_FD
+  BASH_XTRACEFD=$SOUP_XTRACE_FD
+  PS4='[${BASH_SOURCE##*/}:${LINENO} ${FUNCNAME[0]:-main}] | '
+  set -x
+  export SOUP_DEBUG
+fi
+
+main "$@" 2>&1 | tee -a $SOUP_LOG

From 16149df71fd25c797e318c057c02dac1ead0ee6c Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 16 Jun 2026 18:06:27 -0500
Subject: [PATCH 27/28] formatting

---
 salt/manager/tools/sbin/soup | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index a64524bff..fcde61d9e 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -131,10 +131,11 @@ check_err() {
 # Collect bash error context before passing off to check_err()
 on_err() {
   local exit_code=$?
+  # turn off xtrace to prevent added noise in debug log
+  set +x 2>/dev/null || true
+
   # Use first error context, multiple errors can happen with command substitutions or nested functions. We just need context from the initial error.
   [[ -n $SOUP_ERR_CONTEXT ]] && return $exit_code
-  # turn off xtrace to prevent added noise in debug log
-  [[ $- == *x* ]] && set +x
 
   local cmd=$BASH_COMMAND
   local line=${BASH_LINENO[0]}
@@ -2033,7 +2034,7 @@ if [[ $SOUP_DEBUG == 1 ]]; then
   exec {SOUP_XTRACE_FD}>>"$SOUP_DEBUG_LOG"
   export SOUP_XTRACE_FD
   BASH_XTRACEFD=$SOUP_XTRACE_FD
-  PS4='[${BASH_SOURCE##*/}:${LINENO} ${FUNCNAME[0]:-main}] | '
+  PS4='+ [${BASH_SOURCE##*/}:${LINENO} ${FUNCNAME[0]:-main}()] | '
   set -x
   export SOUP_DEBUG
 fi

From 22d5c96bd50d7d4ad81e8297d820945e9e312ff0 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 18 Jun 2026 14:56:29 -0500
Subject: [PATCH 28/28] don't create stack trace when set -e is disabled

---
 salt/manager/tools/sbin/soup | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index fcde61d9e..73bb2eec2 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -131,6 +131,8 @@ check_err() {
 # Collect bash error context before passing off to check_err()
 on_err() {
   local exit_code=$?
+  # Ignore failures in blocks that explicitly disabled errexit with `set +e`.
+  [[ $- == *e* ]] || return $exit_code
   # turn off xtrace to prevent added noise in debug log
   set +x 2>/dev/null || true