maxSubSessionTokens and maxDelegationDepth config settings

salt/common/tools/sbin/so-common

@@ -142,11 +142,6 @@ check_elastic_license() {
 	fi  
 }
 
-check_elasticsearch_responsive() {
-    retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" ||
-        fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot."
-}
-
 check_salt_master_status() {
 	local count=0
     local attempts="${1:- 10}"

salt/elasticfleet/content-defaults.map.jinja

@@ -9,6 +9,7 @@
 
 {% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
 {% set ADDON_CONTENT_INTEGRATION_DEFAULTS = {} %}
+{% set DEBUG_STUFF = {} %}
 
 {% for pkg in ADDON_CONTENT_PACKAGE_COMPONENTS %}
 {%   if pkg.name in CORE_ESFLEET_PACKAGES %}

salt/elasticfleet/input-defaults.map.jinja

@@ -9,6 +9,7 @@
 
 {% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
 {% set ADDON_INPUT_INTEGRATION_DEFAULTS = {} %}
+{% set DEBUG_STUFF = {} %}
 
 {% for pkg in ADDON_INPUT_PACKAGE_COMPONENTS %}
 {%   if pkg.name in CORE_ESFLEET_PACKAGES %}
@@ -115,6 +116,7 @@
 
 
 {%         do ADDON_INPUT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
+{%         do DEBUG_STUFF.update({integration_key: "Generating defaults for "+ pkg.name })%}
 {%       endfor %}
 {%     endif %}
 {%   endif %}

salt/elasticfleet/tools/sbin/so-elastic-fleet-common

@@ -30,82 +30,6 @@ fleet_api() {
     curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/${QUERYPATH}" "$@" --retry 3 --retry-delay 10 --fail 2>/dev/null
 }
 
-# Max number of concurrent Fleet write jobs (create/update). Override via env if needed.
-MAX_FLEET_JOBS=${MAX_FLEET_JOBS:-10}
-
-# Block until fewer than MAX_FLEET_JOBS background jobs are running.
-elastic_fleet_throttle() {
-    while (( $(jobs -rp | wc -l) >= MAX_FLEET_JOBS )); do
-        wait -n
-    done
-}
-
-# Load every integration JSON in a directory into a single agent policy.
-# The agent policy is fetched ONCE (not per file), and the create/update writes
-# are dispatched as throttled background jobs.
-#   $1 AGENT_POLICY     - the agent policy id/name to load integrations into
-#   $2 DIR              - directory of integration *.json files
-#   $3 LABEL           - human-readable label for log output
-#   $4 SKIP_CREATE_NAME - (optional) integration name to skip when creating (still updated if present)
-# Returns 1 if any integration failed to create/update.
-elastic_fleet_load_integrations_dir() {
-    local AGENT_POLICY=$1
-    local DIR=$2
-    local LABEL=$3
-    local SKIP_CREATE_NAME=$4
-    local POLICY_JSON FAIL_FILE OUT_DIR INTEGRATION NAME ID i
-
-    FAIL_FILE=$(mktemp)
-    # Each job buffers its full output (header + API response) into its own file so the
-    # parent can print them grouped and in submission order after concurrent writes finish.
-    OUT_DIR=$(mktemp -d)
-    i=0
-
-    # Fetch the agent policy a single time; we look up integration ids locally below.
-    POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY")
-
-    for INTEGRATION in "$DIR"/*.json; do
-        [ -e "$INTEGRATION" ] || continue
-        NAME=$(jq -r .name "$INTEGRATION")
-        ID=$(jq -r --arg n "$NAME" '.item.package_policies[]? | select(.name==$n) | .id' <<<"$POLICY_JSON")
-
-        elastic_fleet_throttle
-        {
-            local RESP
-            if [ -n "$ID" ]; then
-                printf "\n\n%s - Updating integration %s\n" "$LABEL" "$NAME"
-                if ! RESP=$(elastic_fleet_integration_update "$ID" "@$INTEGRATION"); then
-                    flock 9; echo "update ${INTEGRATION##*/}" >&9
-                fi
-                printf '%s\n' "$RESP"
-            elif [ -n "$SKIP_CREATE_NAME" ] && [ "$NAME" == "$SKIP_CREATE_NAME" ]; then
-                printf "\n\n%s - Skipping creation of %s\n" "$LABEL" "$NAME"
-            else
-                printf "\n\n%s - Creating integration %s\n" "$LABEL" "$NAME"
-                if ! RESP=$(elastic_fleet_integration_create "@$INTEGRATION"); then
-                    flock 9; echo "create ${INTEGRATION##*/}" >&9
-                fi
-                printf '%s\n' "$RESP"
-            fi
-        } >"$OUT_DIR/$(printf '%03d' "$i")" 9>>"$FAIL_FILE" &
-        i=$((i+1))
-    done
-    wait
-
-    # Emit per-integration output grouped and in submission order (glob sorts numerically).
-    cat "$OUT_DIR"/* 2>/dev/null
-    rm -rf "$OUT_DIR"
-
-    local rc=0
-    if [ -s "$FAIL_FILE" ]; then
-        printf "\n%s: failed integrations:\n" "$LABEL"
-        cat "$FAIL_FILE"
-        rc=1
-    fi
-    rm -f "$FAIL_FILE"
-    return $rc
-}
-
 elastic_fleet_integration_check() {
 
     AGENT_POLICY=$1
@@ -122,9 +46,7 @@ elastic_fleet_integration_create() {
 
     JSON_STRING=$1
 
-    # --retry-all-errors so transient 409 conflicts (concurrent writes to the same agent
+    if ! fleet_api "package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPOST -d "$JSON_STRING"; then
-    # policy) are retried; curl --retry alone does not retry 409.
-    if ! fleet_api "package_policies" --retry-all-errors -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPOST -d "$JSON_STRING"; then
         return 1
     fi
 }
@@ -155,9 +77,7 @@ elastic_fleet_integration_update() {
 
     JSON_STRING=$2
 
-    # --retry-all-errors so transient 409 conflicts (concurrent writes to the same agent
+    if ! fleet_api "package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPUT -d "$JSON_STRING"; then
-    # policy) are retried; curl --retry alone does not retry 409.
-    if ! fleet_api "package_policies/$UPDATE_ID" --retry-all-errors -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPUT -d "$JSON_STRING"; then
         return 1
     fi
 }

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -18,26 +18,93 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
   # Third, configure Elastic Defend Integration seperately
   /usr/sbin/so-elastic-fleet-integration-policy-elastic-defend
 
-  # Each group fetches its agent policy once and dispatches create/update writes concurrently.
-
   # Initial Endpoints
-  elastic_fleet_load_integrations_dir "endpoints-initial" \
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/endpoints-initial/*.json; do
-    /opt/so/conf/elastic-fleet/integrations/endpoints-initial "Initial Endpoints Policy" || RETURN_CODE=1
+    printf "\n\nInitial Endpoints Policy - Loading $INTEGRATION\n"
+    elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
+    if [ -n "$INTEGRATION_ID" ]; then
+      printf "\n\nIntegration $NAME exists - Updating integration\n"
+      if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+        echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
+    else
+      printf "\n\nIntegration does not exist - Creating integration\n"
+      if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
+    fi
+  done
 
   # Grid Nodes - General
-  elastic_fleet_load_integrations_dir "so-grid-nodes_general" \
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_general/*.json; do
-    /opt/so/conf/elastic-fleet/integrations/grid-nodes_general "Grid Nodes Policy_General" || RETURN_CODE=1
+    printf "\n\nGrid Nodes Policy_General - Loading $INTEGRATION\n"
+    elastic_fleet_integration_check "so-grid-nodes_general" "$INTEGRATION"
+    if [ -n "$INTEGRATION_ID" ]; then
+      printf "\n\nIntegration $NAME exists - Updating integration\n"
+      if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+        echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
+    else
+      printf "\n\nIntegration does not exist - Creating integration\n"
+      if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
+    fi
+  done
 
   # Grid Nodes - Heavy
-  elastic_fleet_load_integrations_dir "so-grid-nodes_heavy" \
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy/*.json; do
-    /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy "Grid Nodes Policy_Heavy" || RETURN_CODE=1
+    printf "\n\nGrid Nodes Policy_Heavy - Loading $INTEGRATION\n"
+    elastic_fleet_integration_check "so-grid-nodes_heavy" "$INTEGRATION"
+    if [ -n "$INTEGRATION_ID" ]; then
+      printf "\n\nIntegration $NAME exists - Updating integration\n"
+      if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+        echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
+    else
+      printf "\n\nIntegration does not exist - Creating integration\n"
+      if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
+    fi
+  done
 
-  # Fleet Server - Optional integrations (one agent policy per FleetServer_* directory)
+  # Fleet Server - Optional integrations
-  for FLEET_DIR in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/; do
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json; do
-    [ -d "$FLEET_DIR" ] || continue
+    if ! [ "$INTEGRATION" == "/opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json" ]; then
-    FLEET_POLICY=$(basename "$FLEET_DIR")
+      FLEET_POLICY=`echo "$INTEGRATION"| cut -d'/' -f7`
-    elastic_fleet_load_integrations_dir "$FLEET_POLICY" \
+      printf "\n\nFleet Server Policy - Loading $INTEGRATION\n"
-      "${FLEET_DIR%/}" "Fleet Server Policy" "elasticsearch-logs" || RETURN_CODE=1
+      elastic_fleet_integration_check "$FLEET_POLICY" "$INTEGRATION"
+      if [ -n "$INTEGRATION_ID" ]; then
+        printf "\n\nIntegration $NAME exists - Updating integration\n"
+        if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+          echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+          RETURN_CODE=1
+          continue
+        fi
+      else
+        printf "\n\nIntegration does not exist - Creating integration\n"
+        if [ "$NAME" != "elasticsearch-logs" ]; then
+          if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+            echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+            RETURN_CODE=1
+            continue
+          fi
+        fi
+      fi
+    fi
   done
 
   # Only create the state file if all policies were created/updated successfully

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade

@@ -23,90 +23,73 @@ if [ $? -ne 0 ]; then
 fi
 
 default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %})
-# JSON array of the default packages, used by the jq filter below.
-default_packages_json=$(printf '%s\n' "${default_packages[@]}" | jq -R . | jq -s '.')
-
-# Output lock (serializes concurrent job output) and failure file (one marker line per
-# failed integration). Mirrors the pattern used by elastic_fleet_load_integrations_dir.
-OUTPUT_LOCK=$(mktemp)
-FAIL_FILE=$(mktemp)
-trap 'rm -f "$OUTPUT_LOCK" "$FAIL_FILE"' EXIT
-
-# Cache of package name -> latest available version, so the same package is only looked up
-# once instead of once per (policy, integration).
-declare -A LATEST_VERSION_CACHE
 
+ERROR=false
 for AGENT_POLICY in $agent_policies; do
-    # Fetch the agent policy a single time; package name/version and integration id are all
+    if ! integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY"); then
-    # extracted locally below instead of re-fetching the same policy per integration.
-    if ! POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY"); then
         # this script upgrades default integration packages, exit 1 and let salt handle retrying
         exit 1
     fi
-
+    for INTEGRATION in $integrations; do
-    # One jq pass emits name/package.name/package.version/id for every eligible integration.
+        if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then
-    # The endpoint/fleet_server skips and the default-package gate are applied here in jq.
+            # Get package name so we know what package to look for when checking the current and latest available version
-    # $defaults (not $def, a jq reserved keyword) holds the default package list.
+            if ! PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION"); then
-    while IFS=$'\t' read -r INTEGRATION PACKAGE_NAME PACKAGE_VERSION INTEGRATION_ID; do
-        [ -n "$INTEGRATION" ] || continue
-
-        # Look up the latest available version once per package, then memoize it.
-        if [[ -z "${LATEST_VERSION_CACHE[$PACKAGE_NAME]+set}" ]]; then
-            if ! AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME"); then
-                echo "Error: Failed getting latest version for $PACKAGE_NAME"
                 exit 1
             fi
-            LATEST_VERSION_CACHE[$PACKAGE_NAME]=$AVAILABLE_VERSION
+            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
-        fi
+            if [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then
-        AVAILABLE_VERSION=${LATEST_VERSION_CACHE[$PACKAGE_NAME]}
+            {%- endif %}
-
+                # Get currently installed version of package
-        if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then
+                attempt=0
-            # Dry run, then (if clean) the actual upgrade, dispatched as a throttled background
+                max_attempts=3
-            # job. Each job builds its full log into one block, then flushes it under a single
+                while [ $attempt -lt $max_attempts ]; do
-            # shared lock (OUTPUT_LOCK) so concurrent jobs never interleave on stdout; a failed
+                    if PACKAGE_VERSION=$(elastic_fleet_integration_policy_package_version "$AGENT_POLICY" "$INTEGRATION") && AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME"); then
-            # job also appends a marker line to FAIL_FILE while holding that same lock.
+                        break
-            elastic_fleet_throttle
-            {
-                block=$'\n'"Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..."$'\n'
-                block+="Upgrading $INTEGRATION..."$'\n'"Starting dry run..."$'\n'
-                fail=""
-                if ! DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID"); then
-                    block+="Error: Failed to complete dry run for '$INTEGRATION_ID'."$'\n'
-                    fail="dryrun $INTEGRATION"
-                elif [[ "$(jq .[].hasErrors <<<"$DRYRUN_OUTPUT")" == "false" ]]; then
-                    block+="No errors detected. Proceeding with upgrade..."$'\n'
-                    if ! elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"; then
-                        block+="Error: Upgrade failed for $PACKAGE_NAME with integration ID '$INTEGRATION_ID'."$'\n'
-                        fail="upgrade $INTEGRATION"
                     fi
-                else
+                    attempt=$((attempt + 1))
-                    block+="Errors detected during dry run for $PACKAGE_NAME policy upgrade..."$'\n'
+                done
-                    fail="dryrun-errors $INTEGRATION"
+                if [ $attempt -eq $max_attempts ]; then
+                    echo "Error: Failed getting $PACKAGE_VERSION or $AVAILABLE_VERSION"
+                    exit 1
                 fi
-                {
+
-                    flock 9
+                # Get integration ID
-                    printf '%s' "$block"
+                if ! INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION"); then
-                    [ -n "$fail" ] && printf '%s\n' "$fail" >>"$FAIL_FILE"
+                    exit 1
-                } 9>>"$OUTPUT_LOCK"
+                fi
-            } &
+
+                if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then
+                    # Dry run of the upgrade
+                    echo ""
+                    echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..."
+                    echo "Upgrading $INTEGRATION..."
+                    echo "Starting dry run..."
+                    if ! DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID"); then
+                        exit 1
+                    fi
+                    DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors)
+
+                    # If no errors with dry run, proceed with actual upgrade
+                    if [[ "$DRYRUN_ERRORS" == "false" ]]; then
+                        echo "No errors detected. Proceeding with upgrade..."
+                        if ! elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"; then
+                            echo "Error: Upgrade failed for $PACKAGE_NAME with integration ID '$INTEGRATION_ID'."
+                            ERROR=true
+                            continue
+                        fi
+                    else
+                        echo "Errors detected during dry run for $PACKAGE_NAME policy upgrade..."
+                        ERROR=true
+                        continue
+                    fi
+                fi
+            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+            fi
+            {%- endif %}
         fi
-    done < <(jq -r --argjson defaults "$default_packages_json" '
+    done
-        .item.package_policies[]
-        | select(.name != "elastic-defend-endpoints")
-        | select(.name | startswith("fleet_server-") | not)
-        {%- if not AUTO_UPGRADE_INTEGRATIONS %}
-        | select(.package.name | IN($defaults[]))
-        {%- endif %}
-        | [.name, .package.name, .package.version, .id] | @tsv
-    ' <<<"$POLICY_JSON")
 done
-
+if [[ "$ERROR" == "true" ]]; then
-# Barrier: wait for every dispatched dry-run/upgrade job to finish.
-wait
-
-if [ -s "$FAIL_FILE" ]; then
-    printf '\nFailed integration upgrades:\n'
-    cat "$FAIL_FILE"
     exit 1
 fi
 echo

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load

@@ -16,6 +16,7 @@
 STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt
 INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json
 BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
+BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
 BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json
 INTEGRATION_PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
 INPUT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_input_package_components.json
@@ -28,6 +29,29 @@ PENDING_UPDATE=false
 #   Requiring some level of manual Elastic Stack configuration before installation
 EXCLUDED_INTEGRATIONS=('apm')
 
+version_conversion(){
+    version=$1
+    echo "$version" | awk -F '.' '{ printf("%d%03d%03d\n", $1, $2, $3); }'
+}
+
+compare_versions() {
+    version1=$1
+    version2=$2
+
+    # Convert versions to numbers
+    num1=$(version_conversion "$version1")
+    num2=$(version_conversion "$version2")
+
+    # Compare using bc
+    if (( $(echo "$num1 < $num2" | bc -l) )); then
+        echo "less"
+    elif (( $(echo "$num1 > $num2" | bc -l) )); then
+        echo "greater"
+    else
+        echo "equal"
+    fi
+}
+
 IFS=$'\n'
 agent_policies=$(elastic_fleet_agent_policy_ids)
 if [ $? -ne 0 ]; then
@@ -39,23 +63,23 @@ default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.l
 
 in_use_integrations=()
 
-# Fetch each agent policy once; its package_policies[] already contain both the integration name
-#  and the .package.name, so extract all non-default package names locally in a single jq instead
-#  of re-fetching the same policy per integration.
-default_packages_json=$(printf '%s\n' "${default_packages[@]}" | jq -R . | jq -s '.')
 for AGENT_POLICY in $agent_policies; do
 
-    if ! policy_json=$(fleet_api "agent_policies/$AGENT_POLICY"); then
+    if ! integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY"); then
         # skip the agent policy if we can't get required info, let salt retry. Integrations loaded by this script are non-default integrations.
         echo "Skipping $AGENT_POLICY.. "
         continue
     fi
-    # non-default integrations that are in-use in any policy
+    for INTEGRATION in $integrations; do
-    while IFS= read -r PACKAGE_NAME; do
+        if ! PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION"); then
-        [ -n "$PACKAGE_NAME" ] && in_use_integrations+=("$PACKAGE_NAME")
+            echo  "Not adding $INTEGRATION, couldn't get package name"
-    done < <(jq -r --argjson defaults "$default_packages_json" \
+            continue
-        '.item.package_policies[].package.name | select(. as $n | ($defaults | index($n)) | not)' \
+        fi
-        <<<"$policy_json")
+        # non-default integrations that are in-use in any policy
+        if ! [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then
+            in_use_integrations+=("$PACKAGE_NAME")
+        fi
+    done
 done
 
 if [[ -f $STATE_FILE_SUCCESS  ]]; then
@@ -66,55 +90,72 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
         rm -f $INSTALLED_PACKAGE_LIST
         echo $latest_package_list | jq '{packages: [.items[] | {name: .name, latest_version: .version, installed_version: .installationInfo.version, subscription: .conditions.elastic.subscription }]}' >> $INSTALLED_PACKAGE_LIST
 
-        # Build the bulk install list and the per-package status messages with two jq passes
+        while read -r package; do
-        #  instead of a per-package bash loop. The old loop forked ~10 processes per package
+            # get package details
-        #  (5 jq + awk/bc for the version compare) and re-parsed/rewrote a growing JSON file on
+            package_name=$(echo "$package" | jq -r '.name')
-        #  every add (O(n^2)). Selection and messages below are identical to that logic.
+            latest_version=$(echo "$package" | jq -r '.latest_version')
-        SUB={% if SUB %}true{% else %}false{% endif %}
+            installed_version=$(echo "$package" | jq -r '.installed_version')
-        AUTOUP={% if AUTO_UPGRADE_INTEGRATIONS %}true{% else %}false{% endif %}
+            subscription=$(echo "$package" | jq -r '.subscription')
-        EXCLUDED_JSON=$(printf '%s\n' "${EXCLUDED_INTEGRATIONS[@]}" | jq -R 'select(length>0)' | jq -s '.')
+            bulk_package=$(echo "$package" | jq '{name: .name, version: .latest_version}' )
-        INUSE_JSON=$(printf '%s\n' "${in_use_integrations[@]}" | jq -R 'select(length>0)' | jq -s 'unique')
 
-        # vnum replicates the previous version_conversion (%d%03d%03d of the first three dotted
+            if [[ ! "${EXCLUDED_INTEGRATIONS[@]}" =~ "$package_name" ]]; then
-        #  fields); needs() replicates the excluded/subscription/installed/upgrade/in-use logic.
+            {% if not SUB %}
-        JQ_DECISION='
+                if [[ "$subscription" != "basic" && "$subscription" != "null" && -n "$subscription" ]]; then
-def vnum:
+                    # pass over integrations that require non-basic elastic license
-  [ (split(".")|.[0:3][] | gsub("[^0-9].*";"") | (if .=="" then "0" else . end) | tonumber) ]
+                    echo "$package_name integration requires an Elastic license of $subscription or greater... skipping"
-  | (.[0]//0)*1000000 + (.[1]//0)*1000 + (.[2]//0);
+                    continue
-def needs($sub;$autoup;$excluded;$inuse):
+                else
-  .name as $n
+                    if [[ "$installed_version" == "null" || -z "$installed_version" ]]; then
-  | ($n | IN($excluded[]) | not)
+                        echo "$package_name is not installed... Adding to next update."
-  and ( $sub or (.subscription==null or .subscription=="basic" or .subscription=="") )
+                        jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
-  and ( (.installed_version==null or .installed_version=="")
-        or ( ((.latest_version|vnum) > (.installed_version|vnum))
-             and ( $autoup or ($n | IN($inuse[]) | not) ) ) );'
 
-        JQ_ARGS=(--argjson sub "$SUB" --argjson autoup "$AUTOUP" --argjson excluded "$EXCLUDED_JSON" --argjson inuse "$INUSE_JSON")
+                        PENDING_UPDATE=true
+                    else
+                        results=$(compare_versions "$latest_version" "$installed_version")
+                        if [ $results == "greater" ]; then
+                            {#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations  #}
+                            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                            if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then
+                            {%- endif %}
+                                echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
+                                jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
 
-        # (a) Per-package status messages (parity with the previous echo output).
+                                PENDING_UPDATE=true
-        jq -r "${JQ_ARGS[@]}" "$JQ_DECISION"'
+                            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
-          .packages[]
+                            else
-          | .name as $n
+                                echo "skipping available upgrade for in use integration - $package_name."
-          | if ($n|IN($excluded[])) then "Skipping \($n)..."
+                            fi
-            elif (($sub|not) and (.subscription!=null and .subscription!="basic" and .subscription!="")) then
+                            {%- endif %}
-                 "\($n) integration requires an Elastic license of \(.subscription) or greater... skipping"
+                        fi
-            elif (.installed_version==null or .installed_version=="") then
+                    fi
-                 "\($n) is not installed... Adding to next update."
+                fi
-            elif ((.latest_version|vnum) > (.installed_version|vnum)) then
+            {% else %}
-                 (if ($autoup or ($n|IN($inuse[])|not))
+                if [[ "$installed_version" == "null" || -z "$installed_version" ]]; then
-                  then "\($n) is at version \(.installed_version) latest version is \(.latest_version)... Adding to next update."
+                    echo "$package_name is not installed... Adding to next update."
-                  else "skipping available upgrade for in use integration - \($n)." end)
+                    jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
-            else empty end
+                    PENDING_UPDATE=true
-        ' "$INSTALLED_PACKAGE_LIST"
+                else
-
+                    results=$(compare_versions "$latest_version" "$installed_version")
-        # (b) The bulk install list, built in a single pass.
+                    if [ $results == "greater" ]; then
-        jq "${JQ_ARGS[@]}" "$JQ_DECISION"'
+                        {#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations  #}
-          {packages: [ .packages[] | select(needs($sub;$autoup;$excluded;$inuse)) | {name, version: .latest_version} ]}
+                        {%- if not AUTO_UPGRADE_INTEGRATIONS %}
-        ' "$INSTALLED_PACKAGE_LIST" > "$BULK_INSTALL_PACKAGE_LIST"
+                        if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then
-
+                        {%- endif %}
-        if jq -e '.packages | length > 0' "$BULK_INSTALL_PACKAGE_LIST" >/dev/null; then
+                            echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
-            PENDING_UPDATE=true
+                            jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
-        fi
+                            PENDING_UPDATE=true
+                        {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                        else
+                            echo "skipping available upgrade for in use integration - $package_name."
+                        fi
+                        {%- endif %}
+                    fi
+                fi
+            {% endif %}
+            else
+                echo "Skipping $package_name..."
+            fi
+        done <<< "$(jq -c '.packages[]' "$INSTALLED_PACKAGE_LIST")"
 
         if [ "$PENDING_UPDATE" = true ]; then
             # Run chunked install of packages

salt/elasticsearch/cluster.sls

@@ -133,18 +133,6 @@ so-elasticsearch-templates:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
 
-so-elasticsearch-dlm-apply:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-dlm-apply
-    - cwd: /opt/so
-    - require:
-      - docker_container: so-elasticsearch
-      - file: elasticsearch_sbin_jinja
-      - cmd: so-elasticsearch-templates
-    - retry:
-        attempts: 3
-        interval: 10
-
 so-elasticsearch-pipelines:
   cmd.run:
     - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }}
@@ -165,8 +153,7 @@ so-elasticsearch-roles-load:
 {%    set ap = "absent" %}
 {%  endif %}
 {%  if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %}
-{#    Remove so-elasticsearch-indices-delete script when using DLM #}
+{%    if ELASTICSEARCHMERGED.index_clean %}
-{%    if ELASTICSEARCHMERGED.index_clean and ELASTICSEARCHMERGED.data_retention_method == "ILM" %}
 {%      set ap = "present" %}
 {%    else %}
 {%      set ap = "absent" %}

salt/elasticsearch/defaults.yaml

@@ -2,7 +2,6 @@ elasticsearch:
   enabled: false
   version: 9.3.3
   index_clean: true
-  data_retention_method: DLM
   vm:
     max_map_count: 1048576
   config:
@@ -64,8 +63,6 @@ elasticsearch:
             verification_mode: none
   index_settings:
     global_overrides:
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         template:
           settings:
@@ -146,8 +143,6 @@ elasticsearch:
                 order: desc
     so-common:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -309,8 +304,6 @@ elasticsearch:
               number_of_shards: 1
     so-assistant-chat:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: ""
       index_template:
         composed_of:
         - assistant-chat-mappings
@@ -351,8 +344,6 @@ elasticsearch:
             min_age: 0ms
     so-assistant-session:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: ""
       index_template:
         composed_of:
         - assistant-session-mappings
@@ -506,8 +497,6 @@ elasticsearch:
             min_age: 30d
     so-idh:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -616,8 +605,6 @@ elasticsearch:
             min_age: 30d
     so-import:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -800,8 +787,6 @@ elasticsearch:
             min_age: 0ms
     so-kismet:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - kismet-mappings
@@ -851,8 +836,6 @@ elasticsearch:
             min_age: 30d
     so-kratos:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -921,8 +904,6 @@ elasticsearch:
             min_age: 30d
     so-hydra:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -1068,8 +1049,6 @@ elasticsearch:
             min_age: 0ms
     so-logs:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - so-data-streams-mappings
@@ -1150,8 +1129,6 @@ elasticsearch:
             min_age: 30d
     so-logs-detections_x_alerts:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - so-data-streams-mappings
@@ -1215,8 +1192,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1332,8 +1307,6 @@ elasticsearch:
             min_age: 30d
     so-elastic-agent-monitor:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1396,8 +1369,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_apm_server:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.apm_server@package
@@ -1462,8 +1433,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_auditbeat:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.auditbeat@package
@@ -1528,8 +1497,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_cloudbeat:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.cloudbeat@package
@@ -1594,8 +1561,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_endpoint_security:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1655,8 +1620,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_filebeat:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1716,8 +1679,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_fleet_server:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1774,8 +1735,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_heartbeat:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.heartbeat@package
@@ -1840,8 +1799,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_metricbeat:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1901,8 +1858,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_osquerybeat:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -1962,8 +1917,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elastic_agent_x_packetbeat:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-elastic_agent.packetbeat@package
@@ -2028,8 +1981,6 @@ elasticsearch:
             min_age: 30d
     so-logs-elasticsearch_x_server:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-elasticsearch.server@package
@@ -2094,13 +2045,10 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_actions:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - .logs-endpoint.actions@package
         - .logs-endpoint.actions@custom
-        - endpoint@custom
         - event-mappings
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -2110,9 +2058,8 @@ elasticsearch:
           hidden: false
         ignore_missing_component_templates:
         - .logs-endpoint.actions@custom
-        - endpoint@custom
         index_patterns:
-        - .logs-endpoint.actions-*
+        - logs-endpoint.actions-*
         priority: 501
         template:
           settings:
@@ -2157,13 +2104,10 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_action_x_responses:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - .logs-endpoint.action.responses@package
         - .logs-endpoint.action.responses@custom
-        - endpoint@custom
         - event-mappings
         - so-fleet_integrations.ip_mappings-1
         - so-fleet_globals-1
@@ -2173,15 +2117,14 @@ elasticsearch:
           hidden: false
         ignore_missing_component_templates:
         - .logs-endpoint.action.responses@custom
-        - endpoint@custom
         index_patterns:
-        - .logs-endpoint.action.responses-*
+        - logs-endpoint.action.responses-*
         priority: 501
         template:
           settings:
             index:
               lifecycle:
-                name: so-logs-endpoint.action.responses-logs
+                name: so-logs-endpoint.actions-logs
               mapping:
                 total_fields:
                   limit: 5000
@@ -2220,8 +2163,6 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_alerts:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.alerts@package
@@ -2281,8 +2222,6 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_diagnostic_x_collection:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - .logs-endpoint.diagnostic.collection@package
@@ -2358,8 +2297,6 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_api:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.api@package
@@ -2419,8 +2356,6 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_file:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.file@package
@@ -2480,8 +2415,6 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_library:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.library@package
@@ -2541,8 +2474,6 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_network:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.network@package
@@ -2602,8 +2533,6 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_process:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.process@package
@@ -2663,8 +2592,6 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_registry:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.registry@package
@@ -2724,8 +2651,6 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_events_x_security:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-endpoint.events.security@package
@@ -2785,8 +2710,6 @@ elasticsearch:
             min_age: 30d
     so-logs-endpoint_x_heartbeat:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - .logs-endpoint.heartbeat@package
@@ -2846,8 +2769,6 @@ elasticsearch:
             min_age: 30d
     so-logs-http_endpoint_x_generic:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-http_endpoint.generic@package
@@ -2896,8 +2817,6 @@ elasticsearch:
             min_age: 30d
     so-logs-httpjson_x_generic:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-httpjson.generic@package
@@ -2963,8 +2882,6 @@ elasticsearch:
               number_of_replicas: 0
     so-logs-osquery-manager_x_action_x_responses:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         _meta:
           managed: true
@@ -3036,8 +2953,6 @@ elasticsearch:
               number_of_replicas: 0
     so-logs-osquery-manager_x_result:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         _meta:
           managed: true
@@ -3090,8 +3005,6 @@ elasticsearch:
             min_age: 30d
     so-logs-soc:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -3200,8 +3113,6 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_application:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3251,8 +3162,6 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_auth:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3302,8 +3211,6 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_security:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3353,8 +3260,6 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_syslog:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3404,8 +3309,6 @@ elasticsearch:
             min_age: 30d
     so-logs-system_x_system:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - event-mappings
@@ -3455,8 +3358,6 @@ elasticsearch:
             min_age: 30d
     so-logs-windows_x_forwarded:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-windows.forwarded@package
@@ -3504,8 +3405,6 @@ elasticsearch:
             min_age: 30d
     so-logs-windows_x_powershell:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-windows.powershell@package
@@ -3553,8 +3452,6 @@ elasticsearch:
             min_age: 30d
     so-logs-windows_x_powershell_operational:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-windows.powershell_operational@package
@@ -3602,8 +3499,6 @@ elasticsearch:
             min_age: 30d
     so-logs-windows_x_sysmon_operational:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-windows.sysmon_operational@package
@@ -3651,8 +3546,6 @@ elasticsearch:
             min_age: 30d
     so-logs-winlog_x_winlog:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - logs-winlog.winlog@package
@@ -3701,8 +3594,6 @@ elasticsearch:
             min_age: 30d
     so-logstash:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -3818,8 +3709,6 @@ elasticsearch:
             min_age: 30d
     so-metrics-endpoint_x_metadata:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - metrics-endpoint.metadata@package
@@ -3867,8 +3756,6 @@ elasticsearch:
             min_age: 30d
     so-metrics-endpoint_x_metrics:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - metrics-endpoint.metrics@package
@@ -3916,8 +3803,6 @@ elasticsearch:
             min_age: 30d
     so-metrics-endpoint_x_policy:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - metrics-endpoint.policy@package
@@ -3965,8 +3850,6 @@ elasticsearch:
             min_age: 30d
     so-metrics-fleet_server_x_agent_status:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - metrics@tsdb-settings
@@ -3991,8 +3874,6 @@ elasticsearch:
               number_of_replicas: 0
     so-metrics-fleet_server_x_agent_versions:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - metrics@tsdb-settings
@@ -4017,8 +3898,6 @@ elasticsearch:
               number_of_replicas: 0
     so-redis:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4079,10 +3958,13 @@ elasticsearch:
         - vulnerability-mappings
         - common-settings
         - common-dynamic-mappings
+        - logs-redis.log@package
+        - logs-redis.log@custom
         data_stream:
           allow_custom_routing: false
           hidden: false
-        ignore_missing_component_templates: []
+        ignore_missing_component_templates:
+        - logs-redis.log@custom
         index_patterns:
         - logs-redis.log*
         priority: 501
@@ -4134,8 +4016,6 @@ elasticsearch:
             min_age: 30d
     so-strelka:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4253,8 +4133,6 @@ elasticsearch:
             min_age: 30d
     so-suricata:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4371,8 +4249,6 @@ elasticsearch:
             min_age: 30d
     so-suricata_x_alerts:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4489,8 +4365,6 @@ elasticsearch:
             min_age: 30d
     so-syslog:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings
@@ -4607,8 +4481,6 @@ elasticsearch:
             min_age: 30d
     so-zeek:
       index_sorting: false
-      data_stream_lifecycle:
-        data_retention: 90d
       index_template:
         composed_of:
         - agent-mappings

salt/elasticsearch/soc_elasticsearch.yaml

@@ -4,13 +4,6 @@ elasticsearch:
     forcedType: bool
     advanced: True
     helpLink: elasticsearch
-  data_retention_method:
-    description: Method for data retention. Options are ILM or DLM. For single node deployments and most distributed grid users, DLM will be the recommended option for simplified management. Those with more complex use cases may prefer ILM. The latter allows for more granular control, but requires more management overhead.
-    options:
-    - ILM
-    - DLM
-    forcedType: string
-    global: True
   version:
     description: "This specifies the version of the following containers: so-elastic-fleet-package-registry, so-elastic-agent, so-elastic-fleet, so-kibana, so-logstash and so-elasticsearch. Modifying this value in the Elasticsearch defaults.yaml will result in catastrophic grid failure."
     readonly: True
@@ -20,7 +13,7 @@ elasticsearch:
     description: Specify the memory heap size in (m)egabytes for Elasticsearch.
     helpLink: elasticsearch
   index_clean:
-    description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, data is retained by the configured lifecycle settings. This setting only applies to EVAL, STANDALONE, and HEAVY NODE installations. Other installations use lifecycle settings only.
+    description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, indices will only be deleted by the age defined in the ILM settings. This setting only applies to EVAL, STANDALONE, and HEAVY NODE installations. Other installations can only use ILM settings.
     forcedType: bool
     helpLink: elasticsearch
   vm:
@@ -146,23 +139,6 @@ elasticsearch:
     custom010: *pipelines
   index_settings:
     global_overrides:
-      data_stream_lifecycle:
-        data_retention:
-          description: |
-            The retention period for all data streams. Retention does not define the period that the data will be removed, but the minimum time period they will be kept.
-
-            Use a number followed by a time unit, such as 7d. Leave blank for indefinite retention where supported.
-
-            Configured retention period also affects the frequency of rolling over data streams.
-              - If retention is less than or equal to 1 day, max_age will be 1 hour
-              - If retention is less than or equal to 14 days, max_age will be 1 day
-              - If retention is less than or equal to 90 days, max_age will be 7 days
-              - If retention is greater than 90 days, max_age will be 30 days
-          forcedType: string
-          allowedNodeTypes:
-            - heavynode
-          regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$
-          regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d.
       index_template:
         template:
           settings:
@@ -335,30 +311,13 @@ elasticsearch:
               forcedType: string
               global: True
               helpLink: elasticsearch
-    so-logs: &dataStreamSettings
+    so-logs: &indexSettings
       index_sorting:
         description: Sorts the index by event time, at the cost of additional processing resource consumption.
         forcedType: bool
         global: True
         advanced: True
         helpLink: elasticsearch
-      data_stream_lifecycle:
-        data_retention:
-          description: |
-            The retention period for this data stream. Retention does not define the period that the data will be removed, but the minimum time period it will be kept.
-
-            Use a number followed by a time unit, such as 7d. Leave blank for indefinite retention where supported.
-
-            Configured retention period also affects the frequency of rolling over this data stream.
-              - If retention is less than or equal to 1 day, max_age will be 1 hour
-              - If retention is less than or equal to 14 days, max_age will be 1 day
-              - If retention is less than or equal to 90 days, max_age will be 7 days
-              - If retention is greater than 90 days, max_age will be 30 days
-          forcedType: string
-          allowedNodeTypes:
-            - heavynode
-          regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$
-          regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d.
       index_template:
         index_patterns:
           description: Patterns for matching multiple indices or tables.
@@ -376,14 +335,6 @@ elasticsearch:
                 global: True
                 advanced: True
                 helpLink: elasticsearch
-              auto_expand_replicas:
-                description: Automatically expand the number of replicas based on the number of data nodes in the cluster. This can help ensure high availability as the cluster scales up or down.
-                forcedType: string
-                regex: "^(0-[1-9]|1-[2-9]|2-[3-9]|3-[4-9]|4-[5-9]|5-[6-9]|6-[7-9]|7-[89]|8-9|[0-9]-all|false)$"
-                regexFailureMessage: Must be in the format of "x-y" where x is minimum number of replicas and y is maximum number of replicas, or "0-all" to specify a minimum of 0 and no maximum, or "false" to disable automatic replica expansion.
-                global: True
-                advanced: True
-                helpLink: elasticsearch
               mapping:
                 total_fields:
                   limit:
@@ -645,349 +596,65 @@ elasticsearch:
             global: True
             advanced: True
             helpLink: elasticsearch
-    so-logs-system_x_auth: *dataStreamSettings
+    so-logs-system_x_auth: *indexSettings
-    so-logs-system_x_syslog: *dataStreamSettings
+    so-logs-system_x_syslog: *indexSettings
-    so-logs-system_x_system: *dataStreamSettings
+    so-logs-system_x_system: *indexSettings
-    so-logs-system_x_application: *dataStreamSettings
+    so-logs-system_x_application: *indexSettings
-    so-logs-system_x_security: *dataStreamSettings
+    so-logs-system_x_security: *indexSettings
-    so-logs-windows_x_forwarded: *dataStreamSettings
+    so-logs-windows_x_forwarded: *indexSettings
-    so-logs-windows_x_powershell: *dataStreamSettings
+    so-logs-windows_x_powershell: *indexSettings
-    so-logs-windows_x_powershell_operational: *dataStreamSettings
+    so-logs-windows_x_powershell_operational: *indexSettings
-    so-logs-windows_x_sysmon_operational: *dataStreamSettings
+    so-logs-windows_x_sysmon_operational: *indexSettings
-    so-logs-winlog_x_winlog: *dataStreamSettings
+    so-logs-winlog_x_winlog: *indexSettings
-    so-logs-detections_x_alerts: *dataStreamSettings
+    so-logs-detections_x_alerts: *indexSettings
-    so-logs-http_endpoint_x_generic: *dataStreamSettings
+    so-logs-http_endpoint_x_generic: *indexSettings
-    so-logs-httpjson_x_generic: *dataStreamSettings
+    so-logs-httpjson_x_generic: *indexSettings
-    so-logs-osquery-manager-actions: *dataStreamSettings
+    so-logs-osquery-manager-actions: *indexSettings
-    so-logs-osquery-manager-action_x_responses: *dataStreamSettings
+    so-logs-osquery-manager-action_x_responses: *indexSettings
-    so-logs-osquery-manager_x_action_x_responses: *dataStreamSettings
+    so-logs-osquery-manager_x_action_x_responses: *indexSettings
-    so-logs-osquery-manager_x_result: *dataStreamSettings
+    so-logs-osquery-manager_x_result: *indexSettings
-    so-logs-elastic_agent_x_apm_server: *dataStreamSettings
+    so-logs-elastic_agent_x_apm_server: *indexSettings
-    so-logs-elastic_agent_x_auditbeat: *dataStreamSettings
+    so-logs-elastic_agent_x_auditbeat: *indexSettings
-    so-logs-elastic_agent_x_cloudbeat: *dataStreamSettings
+    so-logs-elastic_agent_x_cloudbeat: *indexSettings
-    so-logs-elastic_agent_x_endpoint_security: *dataStreamSettings
+    so-logs-elastic_agent_x_endpoint_security: *indexSettings
-    so-logs-endpoint_x_alerts: *dataStreamSettings
+    so-logs-endpoint_x_alerts: *indexSettings
-    so-logs-endpoint_x_events_x_api: *dataStreamSettings
+    so-logs-endpoint_x_events_x_api: *indexSettings
-    so-logs-endpoint_x_events_x_file: *dataStreamSettings
+    so-logs-endpoint_x_events_x_file: *indexSettings
-    so-logs-endpoint_x_events_x_library: *dataStreamSettings
+    so-logs-endpoint_x_events_x_library: *indexSettings
-    so-logs-endpoint_x_events_x_network: *dataStreamSettings
+    so-logs-endpoint_x_events_x_network: *indexSettings
-    so-logs-endpoint_x_events_x_process: *dataStreamSettings
+    so-logs-endpoint_x_events_x_process: *indexSettings
-    so-logs-endpoint_x_events_x_registry: *dataStreamSettings
+    so-logs-endpoint_x_events_x_registry: *indexSettings
-    so-logs-endpoint_x_events_x_security: *dataStreamSettings
+    so-logs-endpoint_x_events_x_security: *indexSettings
-    so-logs-elastic_agent_x_filebeat: *dataStreamSettings
+    so-logs-elastic_agent_x_filebeat: *indexSettings
-    so-logs-elastic_agent_x_fleet_server: *dataStreamSettings
+    so-logs-elastic_agent_x_fleet_server: *indexSettings
-    so-logs-elastic_agent_x_heartbeat: *dataStreamSettings
+    so-logs-elastic_agent_x_heartbeat: *indexSettings
-    so-logs-elastic_agent: *dataStreamSettings
+    so-logs-elastic_agent: *indexSettings
-    so-logs-elastic_agent_x_metricbeat: *dataStreamSettings
+    so-logs-elastic_agent_x_metricbeat: *indexSettings
-    so-logs-elastic_agent_x_osquerybeat: *dataStreamSettings
+    so-logs-elastic_agent_x_osquerybeat: *indexSettings
-    so-logs-elastic_agent_x_packetbeat: *dataStreamSettings
+    so-logs-elastic_agent_x_packetbeat: *indexSettings
-    so-logs-elasticsearch_x_server: *dataStreamSettings
+    so-logs-elasticsearch_x_server: *indexSettings
-    so-metrics-endpoint_x_metadata: *dataStreamSettings
+    so-metrics-endpoint_x_metadata: *indexSettings
-    so-metrics-endpoint_x_metrics: *dataStreamSettings
+    so-metrics-endpoint_x_metrics: *indexSettings
-    so-metrics-endpoint_x_policy: *dataStreamSettings
+    so-metrics-endpoint_x_policy: *indexSettings
-    so-metrics-nginx_x_stubstatus: *dataStreamSettings
+    so-metrics-nginx_x_stubstatus: *indexSettings
-    so-metrics-vsphere_x_datastore: *dataStreamSettings
+    so-metrics-vsphere_x_datastore: *indexSettings
-    so-metrics-vsphere_x_host: *dataStreamSettings
+    so-metrics-vsphere_x_host: *indexSettings
-    so-metrics-vsphere_x_virtualmachine: *dataStreamSettings
+    so-metrics-vsphere_x_virtualmachine: *indexSettings
-    so-common: *dataStreamSettings
+    so-case: *indexSettings
-    so-endgame: *dataStreamSettings
+    so-common: *indexSettings
-    so-idh: *dataStreamSettings
+    so-endgame: *indexSettings
-    so-suricata: *dataStreamSettings
+    so-idh: *indexSettings
-    so-suricata_x_alerts: *dataStreamSettings
+    so-suricata: *indexSettings
-    so-import: *dataStreamSettings
+    so-suricata_x_alerts: *indexSettings
-    so-kratos: *dataStreamSettings
+    so-import: *indexSettings
-    so-hydra: *dataStreamSettings
+    so-kratos: *indexSettings
-    so-kismet: *dataStreamSettings
+    so-hydra: *indexSettings
-    so-logstash: *dataStreamSettings
+    so-kismet: *indexSettings
-    so-redis: *dataStreamSettings
+    so-logstash: *indexSettings
-    so-strelka: *dataStreamSettings
+    so-redis: *indexSettings
-    so-syslog: *dataStreamSettings
+    so-strelka: *indexSettings
-    so-zeek: *dataStreamSettings
+    so-syslog: *indexSettings
-    # Managed SOC integration annotations are inserted below this line. Referencing '*dataStreamSettings'
+    so-zeek: *indexSettings
-    so-case: &indexSettings
-      index_sorting:
-        description: Sorts the index by event time, at the cost of additional processing resource consumption.
-        forcedType: bool
-        global: True
-        advanced: True
-        helpLink: elasticsearch
-      index_template:
-        index_patterns:
-          description: Patterns for matching multiple indices or tables.
-          forcedType: "[]string"
-          multiline: True
-          global: True
-          advanced: True
-          helpLink: elasticsearch
-        template:
-          settings:
-            index:
-              number_of_replicas:
-                description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs.
-                forcedType: int
-                global: True
-                advanced: True
-                helpLink: elasticsearch
-              auto_expand_replicas:
-                description: Automatically expand the number of replicas based on the number of data nodes in the cluster. This can help ensure high availability as the cluster scales up or down.
-                forcedType: string
-                regex: "^(0-[1-9]|1-[2-9]|2-[3-9]|3-[4-9]|4-[5-9]|5-[6-9]|6-[7-9]|7-[89]|8-9|[0-9]-all|false)$"
-                regexFailureMessage: Must be in the format of "x-y" where x is minimum number of replicas and y is maximum number of replicas, or "0-all" to specify a minimum of 0 and no maximum, or "false" to disable automatic replica expansion.
-                global: True
-                advanced: True
-                helpLink: elasticsearch
-              mapping:
-                total_fields:
-                  limit:
-                    description: Max number of fields that can exist on a single index. Larger values will consume more resources.
-                    global: True
-                    advanced: True
-                    helpLink: elasticsearch
-              refresh_interval:
-                description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation.
-                global: True
-                advanced: True
-                helpLink: elasticsearch
-              number_of_shards:
-                description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs.
-                global: True
-                advanced: True
-                helpLink: elasticsearch
-              sort:
-                field:
-                  description: The field to sort by. Must set index_sorting to True.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-                order:
-                  description: The order to sort by. Must set index_sorting to True.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-          mappings:
-            _meta:
-              package:
-                name:
-                  description: Meta settings for the mapping.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-              managed_by:
-                  description: Meta settings for the mapping.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-              managed:
-                  description: Meta settings for the mapping.
-                  forcedType: bool
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-        composed_of:
-          description: The index template is composed of these component templates.
-          forcedType: "[]string"
-          global: True
-          advanced: True
-          helpLink: elasticsearch
-        priority:
-          description: The priority of the index template.
-          forcedType: int
-          global: True
-          advanced: True
-          helpLink: elasticsearch
-      policy:
-        phases:
-          hot:
-            min_age:
-              description: Minimum age of index. This determines when the index should be moved to the hot tier.
-              global: True
-              advanced: True
-              helpLink: elasticsearch
-            actions:
-              set_priority:
-                priority:
-                  description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
-                  forcedType: int
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-              rollover:
-                max_age:
-                  description: Maximum age of index.  Once an index reaches this limit, it will be rolled over into a new index.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-                max_primary_shard_size:
-                  description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-              shrink:
-                method:
-                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
-                  options:
-                  - COUNT
-                  - SIZE
-                  global: True
-                  advanced: True
-                  forcedType: string
-                number_of_shards:
-                  title: shard count
-                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
-                  global: True
-                  forcedType: int
-                  advanced: True
-                max_primary_shard_size:
-                  title: max shard size
-                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
-                  regex: ^[0-9]+(?:gb|tb|pb)$
-                  global: True
-                  forcedType: string
-                  advanced: True
-                allow_write_after_shrink:
-                  description: Allow writes after shrink.
-                  global: True
-                  forcedType: bool
-                  default: False
-                  advanced: True
-              forcemerge:
-                max_num_segments:
-                  description: Reduce the number of segments in each index shard and clean up deleted documents.
-                  global: True
-                  forcedType: int
-                  advanced: True
-                index_codec:
-                  title: compression
-                  description: Use higher compression for stored fields at the cost of slower performance.
-                  forcedType: bool
-                  global: True
-                  default: False
-                  advanced: True
-          warm:
-            min_age:
-              description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
-              regex: ^[0-9]{1,5}d$
-              forcedType: string
-              global: True
-              advanced: True
-              helpLink: elasticsearch
-            actions:
-              set_priority:
-                priority:
-                  description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
-                  forcedType: int
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-              rollover:
-                max_age:
-                  description: Maximum age of index.  Once an index reaches this limit, it will be rolled over into a new index.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-                max_primary_shard_size:
-                  description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-              shrink:
-                method:
-                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
-                  options:
-                  - COUNT
-                  - SIZE
-                  global: True
-                  advanced: True
-                number_of_shards:
-                  title: shard count
-                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
-                  global: True
-                  forcedType: int
-                  advanced: True
-                max_primary_shard_size:
-                  title: max shard size
-                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
-                  regex: ^[0-9]+(?:gb|tb|pb)$
-                  global: True
-                  forcedType: string
-                  advanced: True
-                allow_write_after_shrink:
-                  description: Allow writes after shrink.
-                  global: True
-                  forcedType: bool
-                  default: False
-                  advanced: True
-              forcemerge:
-                max_num_segments:
-                  description: Reduce the number of segments in each index shard and clean up deleted documents.
-                  global: True
-                  forcedType: int
-                  advanced: True
-                index_codec:
-                  title: compression
-                  description: Use higher compression for stored fields at the cost of slower performance.
-                  forcedType: bool
-                  global: True
-                  default: False
-                  advanced: True
-              allocate:
-                number_of_replicas:
-                  description: Set the number of replicas. Remains the same as the previous phase by default.
-                  forcedType: int
-                  global: True
-                  advanced: True
-          cold:
-            min_age:
-              description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
-              regex: ^[0-9]{1,5}d$
-              forcedType: string
-              global: True
-              advanced: True
-              helpLink: elasticsearch
-            actions:
-              set_priority:
-                priority:
-                  description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
-                  forcedType: int
-                  global: True
-                  advanced: True
-                  helpLink: elasticsearch
-              allocate:
-                number_of_replicas:
-                  description: Set the number of replicas. Remains the same as the previous phase by default.
-                  forcedType: int
-                  global: True
-                  advanced: True
-          delete:
-            min_age:
-              description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.
-              regex: ^[0-9]{1,5}d$
-              forcedType: string
-              global: True
-              advanced: True
-              helpLink: elasticsearch
-        _meta:
-          package:
-            name:
-              description: Meta settings for the mapping.
-              global: True
-              advanced: True
-              helpLink: elasticsearch
-          managed_by:
-            description: Meta settings for the mapping.
-            global: True
-            advanced: True
-            helpLink: elasticsearch
-          managed:
-            description: Meta settings for the mapping.
-            forcedType: bool
-            global: True
-            advanced: True
-            helpLink: elasticsearch
-    sos-backup: *indexSettings
-    so-detection: *indexSettings
-    so-assistant-chat: *indexSettings
-    so-assistant-session: *indexSettings
     so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings
       index_sorting:
         description: Sorts the index by event time, at the cost of additional processing resource consumption.

salt/elasticsearch/template.map.jinja

@@ -4,11 +4,7 @@
    Elastic License 2.0. #}
 
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
-{# ELASTICSEARCHMERGED only used here to collect data_retention_method. This file intentionally works with ELASTICSEARCHDEFAULTS #}
-{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
-
 {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %}
-{% set DATA_RETENTION_METHOD = ELASTICSEARCHMERGED.data_retention_method %}
 
 {% set PILLAR_GLOBAL_OVERRIDES = {} %}
 {% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings', {}) %}
@@ -109,17 +105,6 @@
 {%     if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %}
 {%       do settings.index_template.template.settings.index.pop('sort') %}
 {%     endif %}
-{%     if DATA_RETENTION_METHOD == 'DLM' and settings.index_template.data_stream is defined and settings.data_stream_lifecycle is defined %}
-{%       if settings.data_stream_lifecycle.data_retention is defined and settings.data_stream_lifecycle.data_retention %}
-{%         do settings.index_template.template.update({'lifecycle': {'data_retention': settings.data_stream_lifecycle.data_retention}}) %}
-{%       else %}
-{%         do settings.index_template.template.update({'lifecycle': {}}) %}
-{%       endif %}
-{%       if settings.index_template.template.settings.index.lifecycle is not defined %}
-{%         do settings.index_template.template.settings.index.update({'lifecycle': {}}) %}
-{%       endif %}
-{%       do settings.index_template.template.settings.index.lifecycle.update({'prefer_ilm': false}) %}
-{%     endif %}
 {%   endif %}
 
 {# advanced ilm actions #}

salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load

@@ -11,8 +11,10 @@ ADDON_STATEFILE_SUCCESS=/opt/so/state/addon_estemplates.txt
 ELASTICSEARCH_TEMPLATES_DIR="/opt/so/conf/elasticsearch/templates"
 SO_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/index"
 ADDON_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/addon-index"
-FAILED_NAMES=()
+SO_LOAD_FAILURES=0
-FAILED_COUNT=0
+ADDON_LOAD_FAILURES=0
+SO_LOAD_FAILURES_NAMES=()
+ADDON_LOAD_FAILURES_NAMES=()
 IS_HEAVYNODE="false"
 FORCE="false"
 VERBOSE="false"
@@ -44,86 +46,20 @@ while [[ $# -gt 0 ]]; do
     shift
 done
 
-# Max number of concurrent template PUT jobs. Override via env if needed.
-MAX_TEMPLATE_JOBS=${MAX_TEMPLATE_JOBS:-10}
-
-# Block until fewer than MAX_TEMPLATE_JOBS background jobs are running.
-template_throttle() {
-    while (( $(jobs -rp | wc -l) >= MAX_TEMPLATE_JOBS )); do
-        wait -n
-    done
-}
-
-# Per-job failure markers and an output lock for serializing parallel job output.
-# Each failed load drops one file (named after the template) into FAIL_DIR; the
-# output of each job is flushed as a single block under flock so concurrent jobs
-# never interleave their (chatty) retry output.
-FAIL_DIR=$(mktemp -d)
-OUTPUT_LOCK="${FAIL_DIR}/.output.lock"
-: > "$OUTPUT_LOCK"
-trap 'rm -rf "$FAIL_DIR"' EXIT
-
-# Record a failure: $1 = the template name/path to report later. Slashes are
-# encoded so the path becomes a safe single filename.
-record_failure() {
-    local marker="${1//\//__}"
-    : > "${FAIL_DIR}/fail.${marker}"
-}
-
-# Populate FAILED_NAMES and FAILED_COUNT from the current phase's markers.
-# Must run in the current shell (not a command substitution) so the array sticks.
-collect_failures() {
-    FAILED_NAMES=()
-    FAILED_COUNT=0
-    local f name
-    shopt -s nullglob
-    for f in "${FAIL_DIR}"/fail.*; do
-        name="${f##*/fail.}"
-        name="${name//__//}"
-        FAILED_NAMES+=("$name")
-        FAILED_COUNT=$((FAILED_COUNT + 1))
-    done
-    shopt -u nullglob
-}
-
-# Clear markers and names between phases so SO and addon counts stay independent.
-reset_failures() {
-    shopt -s nullglob
-    rm -f "${FAIL_DIR}"/fail.*
-    shopt -u nullglob
-    FAILED_NAMES=()
-    FAILED_COUNT=0
-}
-
-# Print a block of text atomically (under the shared output lock) so the output
-# of concurrent background jobs is not interleaved.
-locked_echo() {
-    { flock 9; printf '%s\n' "$1"; } 9>>"$OUTPUT_LOCK"
-}
-
-# Loads one template file via PUT. Intended to be dispatched as a background job.
-#   $1 uri          - e.g. _component_template/foo or _index_template/foo
-#   $2 file         - path to the template JSON
-#   $3 report_name  - name/path to record if this load fails
 load_template() {
     local uri="$1"
     local file="$2"
-    local report_name="$3"
-    local out rc=0 block
 
-    # Capture everything (including retry's diagnostic chatter) into one block so
+    echo "Loading template file $file"
-    # concurrent jobs never interleave; the whole block is flushed under one flock.
+    if ! output=$(retry 3 3 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"); then
-    block="Loading template file $file"$'\n'
+        echo "$output"
-    if ! out=$(retry 3 3 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}" 2>&1); then
+
-        block+="$out"$'\n'
+        return 1
-        rc=1
+
     elif [[ "$VERBOSE" == "true" ]]; then
-        block+="$out"$'\n'
+        echo "$output"
     fi
 
-    { flock 9; printf '%s' "$block"; } 9>>"$OUTPUT_LOCK"
-
-    (( rc != 0 )) && record_failure "$report_name"
 }
 
 check_required_component_template_exists() {
@@ -174,9 +110,6 @@ load_component_templates() {
         return
     fi
 
-    # Dispatch loads as throttled background jobs. The barrier (wait) happens in
-    # the caller after all component groups have been dispatched, since index
-    # templates must not load until every component template is in place.
     for component in "$pattern"/*.json; do
         tmpl_name=$(basename "${component%.json}")
 
@@ -185,11 +118,21 @@ load_component_templates() {
             tmpl_name="${tmpl_name%-mappings}-mappings"
         fi
 
-        template_throttle
+        if ! load_template "_component_template/${tmpl_name}" "$component"; then
-        load_template "_component_template/${tmpl_name}" "$component" "$component" &
+            SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
+            SO_LOAD_FAILURES_NAMES+=("$component")
+        fi
     done
 }
 
+check_elasticsearch_responsive() {
+    # Cannot load templates if Elasticsearch is not responding.
+    #  NOTE: Slightly faster exit w/ failure than previous "retry 240 1" if there is a problem with Elasticsearch the
+    #    script should exit sooner rather than hang at the 'so-elasticsearch-templates' salt state.
+    retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" ||
+        fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot."
+}
+
 index_templates_exist() {
     local templates_dir="$1"
 
@@ -237,9 +180,6 @@ if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]] && index_templates_e
     load_component_templates "Elastic Agent" "elastic-agent"
     load_component_templates "Security Onion" "so"
 
-    # Barrier: every component template PUT must complete before we snapshot the
-    # component template list and start loading index templates that depend on them.
-    wait
     component_templates=$(so-elasticsearch-component-templates-list)
     echo -e "Loading Security Onion index templates...\n"
     for so_idx_tmpl in "${SO_TEMPLATES_DIR}"/*.json; do
@@ -249,7 +189,7 @@ if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]] && index_templates_e
             # TODO: Better way to load only heavynode specific templates
             if ! check_heavynode_compatiable_index_template "$tmpl_name"; then
                 if [[ "$VERBOSE" == "true" ]]; then
-                    locked_echo "Skipping over $so_idx_tmpl, template is not a heavynode specific index template."
+                    echo "Skipping over $so_idx_tmpl, template is not a heavynode specific index template."
                 fi
 
                 continue
@@ -257,34 +197,32 @@ if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]] && index_templates_e
         fi
 
         if check_required_component_template_exists "$so_idx_tmpl"; then
-            template_throttle
+            if ! load_template "_index_template/$tmpl_name" "$so_idx_tmpl"; then
-            load_template "_index_template/$tmpl_name" "$so_idx_tmpl" "$so_idx_tmpl" &
+                SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
+                SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl")
+            fi
         else
-            locked_echo "Skipping over $so_idx_tmpl due to missing required component template(s)."
+            echo "Skipping over $so_idx_tmpl due to missing required component template(s)."
-            record_failure "$so_idx_tmpl"
+            SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1))
+            SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl")
 
             continue
         fi
     done
 
-    # Barrier: all SO index template PUTs must finish before tallying failures.
+    if [[ $SO_LOAD_FAILURES -eq 0 ]]; then
-    wait
-
-    collect_failures
-    if [[ $FAILED_COUNT -eq 0 ]]; then
         echo "All Security Onion core templates loaded successfully."
 
         touch "$SO_STATEFILE_SUCCESS"
     else
-        echo "Encountered $FAILED_COUNT failure(s) loading templates:"
+        echo "Encountered $SO_LOAD_FAILURES failure(s) loading templates:"
-        for failed_template in "${FAILED_NAMES[@]}"; do
+        for failed_template in "${SO_LOAD_FAILURES_NAMES[@]}"; do
             echo "  - $failed_template"
         done
         if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then
             fail "Failed to load all Security Onion core templates successfully."
         fi
     fi
-    reset_failures
 elif ! index_templates_exist "$SO_TEMPLATES_DIR"; then
     echo "No Security Onion core index templates found in ${SO_TEMPLATES_DIR}, skipping."
 elif [[ -f "$SO_STATEFILE_SUCCESS" ]]; then
@@ -303,27 +241,26 @@ if should_load_addon_templates; then
         tmpl_name=$(basename "${addon_idx_tmpl%-template.json}")
 
         if check_required_component_template_exists "$addon_idx_tmpl"; then
-            template_throttle
+            if ! load_template "_index_template/${tmpl_name}" "$addon_idx_tmpl"; then
-            load_template "_index_template/${tmpl_name}" "$addon_idx_tmpl" "$addon_idx_tmpl" &
+                ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1))
+                ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl")
+            fi
         else
-            locked_echo "Skipping over $addon_idx_tmpl due to missing required component template(s)."
+            echo "Skipping over $addon_idx_tmpl due to missing required component template(s)."
-            record_failure "$addon_idx_tmpl"
+            ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1))
+            ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl")
 
             continue
         fi
     done
 
-    # Barrier: all addon index template PUTs must finish before tallying failures.
+    if [[ $ADDON_LOAD_FAILURES -eq 0 ]]; then
-    wait
-
-    collect_failures
-    if [[ $FAILED_COUNT -eq 0 ]]; then
         echo "All addon integration templates loaded successfully."
 
         touch "$ADDON_STATEFILE_SUCCESS"
     else
-        echo "Encountered $FAILED_COUNT failure(s) loading addon integration templates:"
+        echo "Encountered $ADDON_LOAD_FAILURES failure(s) loading addon integration templates:"
-        for failed_template in "${FAILED_NAMES[@]}"; do
+        for failed_template in "${ADDON_LOAD_FAILURES_NAMES[@]}"; do
             echo "  - $failed_template"
         done
         if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply

@@ -1,175 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-
-{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
-{%- set DATA_RETENTION_METHOD = ELASTICSEARCHMERGED.data_retention_method %}
-
-ELASTICSEARCH_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR:-/opt/so/conf/elasticsearch/templates}"
-TEMPLATE_DIRS=(
-    "${ELASTICSEARCH_TEMPLATES_DIR}/index"
-    "${ELASTICSEARCH_TEMPLATES_DIR}/addon-index"
-)
-DATA_RETENTION_METHOD=$(cat <<'EOF'
-{{ DATA_RETENTION_METHOD }}
-EOF
-)
-DLM_FAILURES=0
-DLM_FAILURE_NAMES=()
-
-if [[ "$DATA_RETENTION_METHOD" != "DLM" && "$DATA_RETENTION_METHOD" != "ILM" ]]; then
-    echo "Unsupported data retention method $DATA_RETENTION_METHOD. Expected DLM or ILM."
-    exit 1
-fi
-
-validate_template_file() {
-    local template_file="$1"
-
-    if ! jq -e 'type == "object" and (.data_stream == null or (.data_stream | type == "object")) and (.template.lifecycle == null or (.template.lifecycle | type == "object")) and (.template.lifecycle.data_retention == null or (.template.lifecycle.data_retention | type == "string"))' >/dev/null 2>&1 "$template_file"; then
-        echo "Invalid index template JSON: $template_file"
-        return 1
-    fi
-}
-
-is_data_stream_template() {
-    jq -e '.data_stream | type == "object"' >/dev/null 2>&1 "$1"
-}
-
-has_data_stream_lifecycle() {
-    jq -e '.template.lifecycle | type == "object"' >/dev/null 2>&1 "$1"
-}
-
-get_data_retention() {
-    jq -r '.template.lifecycle.data_retention // ""' "$1"
-}
-
-find_template_file() {
-    local template="$1"
-    local template_dir
-    local template_file
-
-    for template_dir in "${TEMPLATE_DIRS[@]}"; do
-        template_file="${template_dir}/${template}-template.json"
-
-        if [[ -f "$template_file" ]]; then
-            echo "$template_file"
-            return 0
-        fi
-    done
-
-    return 1
-}
-
-set_data_stream_lifecycle() {
-    local data_stream="$1"
-    local data_retention="$2"
-    local body
-    local output
-
-    if [[ -n "$data_retention" ]]; then
-        if jq -e --arg data_stream "$data_stream" --arg data_retention "$data_retention" '.data_streams[]? | select(.name == $data_stream and .lifecycle.enabled == true and .lifecycle.data_retention == $data_retention)' >/dev/null 2>&1 <<< "$data_streams"; then
-            echo "DLM lifecycle already set for $data_stream with data_retention $data_retention, skipping."
-            return 0
-        fi
-    elif jq -e --arg data_stream "$data_stream" '.data_streams[]? | select(.name == $data_stream and .lifecycle.enabled == true and (.lifecycle.data_retention == null))' >/dev/null 2>&1 <<< "$data_streams"; then
-        echo "DLM lifecycle already set for $data_stream with indefinite retention, skipping."
-        return 0
-    fi
-
-    if [[ -n "$data_retention" ]]; then
-        body=$(jq -cn --arg data_retention "$data_retention" '{data_retention: $data_retention}')
-    else
-        # Setting indefinite retention
-        body='{}'
-    fi
-
-    if ! output=$(so-elasticsearch-query "_data_stream/${data_stream}/_lifecycle" -XPUT -d "$body" --retry 3 --retry-delay 5 --fail); then
-        echo "Failed to set data stream lifecycle for $data_stream."
-        return 1
-    fi
-
-    if [[ -n "$data_retention" ]]; then
-        echo "Set DLM lifecycle for $data_stream with data_retention $data_retention."
-    else
-        echo "Set DLM lifecycle for $data_stream with indefinite retention."
-    fi
-}
-
-disable_data_stream_lifecycle() {
-    local data_stream="$1"
-    local body='{"enabled":false}'
-    local output
-
-    if ! jq -e --arg data_stream "$data_stream" '.data_streams[]? | select(.name == $data_stream and .lifecycle != null and .lifecycle.enabled != false)' >/dev/null 2>&1 <<< "$data_streams"; then
-        # No action needed
-        return 0
-    fi
-
-    if ! output=$(so-elasticsearch-query "_data_stream/${data_stream}/_lifecycle" -XPUT -d "$body" --retry 3 --retry-delay 5 --fail); then
-        echo "Failed to disable data stream lifecycle for $data_stream."
-        return 1
-    fi
-
-    echo "Disabled DLM lifecycle for $data_stream."
-}
-
-process_data_stream() {
-    local data_stream="$1"
-    local data_retention="$2"
-
-    if [[ "$DATA_RETENTION_METHOD" == "DLM" ]]; then
-        set_data_stream_lifecycle "$data_stream" "$data_retention"
-    else
-        disable_data_stream_lifecycle "$data_stream"
-    fi
-}
-
-check_elasticsearch_responsive
-
-if ! data_streams=$(so-elasticsearch-query "_data_stream?format=json" --retry 3 --retry-delay 5 --fail); then
-    echo "Failed to retrieve data streams."
-    exit 1
-fi
-
-while read -r data_stream_config; do
-    data_stream=$(jq -r '.name' <<< "$data_stream_config")
-    template=$(jq -r '.template' <<< "$data_stream_config")
-
-    if ! template_file=$(find_template_file "$template"); then
-        echo "Skipping $data_stream: index template file not found for $template."
-        continue
-    fi
-
-    validate_template_file "$template_file" || exit 1
-
-    if ! is_data_stream_template "$template_file"; then
-        echo "Skipping $data_stream: $template_file is not a data stream template."
-        continue
-    fi
-
-    if [[ "$DATA_RETENTION_METHOD" == "DLM" ]] && ! has_data_stream_lifecycle "$template_file"; then
-        echo "Skipping $data_stream: $template_file does not define data stream lifecycle."
-        continue
-    fi
-
-    data_retention=$(get_data_retention "$template_file")
-
-    if ! process_data_stream "$data_stream" "$data_retention"; then
-        DLM_FAILURES=$((DLM_FAILURES + 1))
-        DLM_FAILURE_NAMES+=("$data_stream")
-    fi
-done < <(jq -c '.data_streams[]' <<< "$data_streams")
-
-if [[ $DLM_FAILURES -eq 0 ]]; then
-    echo "Data stream lifecycle updates completed successfully."
-else
-    echo "Encountered $DLM_FAILURES failure(s) updating data stream lifecycle:"
-    for failed_data_stream in "${DLM_FAILURE_NAMES[@]}"; do
-        echo "  - $failed_data_stream"
-    done
-    exit 1
-fi

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load

@@ -6,37 +6,6 @@
 
 . /usr/sbin/so-common
 
-MAX_JOBS=10
-
-# Lock used to serialize block writes so concurrent jobs never interleave their output.
-ILM_OUTPUT_LOCK=$(mktemp)
-trap 'rm -f "$ILM_OUTPUT_LOCK"' EXIT
-
-# Policies are loaded concurrently (up to MAX_JOBS at a time) for speed. Each policy's block is
-# printed the moment its curl returns, so output appears in COMPLETION ORDER, not the order
-# policies are defined in configuration.
-echo "Loading ILM policies concurrently; output below appears in completion order, not configuration order."
-echo
-
-put_policy() {
-  local desc="$1" policyname="$2" data="$3" result
-  result=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L \
-    -X PUT "https://localhost:9200/_ilm/policy/${policyname}" \
-    -H 'Content-Type: application/json' -d"${data}")
-  # curl above ran in parallel; serialize just this block write so concurrent jobs never interleave.
-  {
-    flock 200
-    printf 'Setting up %s policy...\n%s\n\n' "${desc}" "${result}"
-  } 200>>"${ILM_OUTPUT_LOCK}"
-}
-
-# Block until fewer than MAX_JOBS background curls are running.
-throttle() {
-  while (( $(jobs -rp | wc -l) >= MAX_JOBS )); do
-    wait -n
-  done
-}
-
 {%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
 {%- if GLOBALS.role != "so-heavynode" %}
 {%-   from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
@@ -45,26 +14,35 @@ throttle() {
 {%- for index, settings in ES_INDEX_SETTINGS.items() %}
 {%-   if settings.policy is defined %}
 {%-     if index == 'so-logs-detections.alerts' %}
-  throttle
+  echo
-  put_policy "so-logs-detections.alerts-so" "{{ index }}-so" '{ "policy": {{ settings.policy | tojson(true) }} }' &
+  echo "Setting up so-logs-detections.alerts-so policy..."
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-so" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+  echo
 {%-     elif index == 'so-logs-soc' %}
-  throttle
+  echo
-  put_policy "so-soc-logs" "so-soc-logs" '{ "policy": {{ settings.policy | tojson(true) }} }' &
+  echo "Setting up so-soc-logs policy..."
-  throttle
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/so-soc-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
-  put_policy "{{ index }}-logs" "{{ index }}-logs" '{ "policy": {{ settings.policy | tojson(true) }} }' &
+  echo
+  echo
+  echo "Setting up {{ index }}-logs policy..."
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+  echo
 {%-     else %}
-  throttle
+  echo
-  put_policy "{{ index }}-logs" "{{ index }}-logs" '{ "policy": {{ settings.policy | tojson(true) }} }' &
+  echo "Setting up {{ index }}-logs policy..."
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+  echo
 {%-     endif %}
 {%-   endif %}
 {%- endfor %}
+echo
 {%- if GLOBALS.role != "so-heavynode" %}
 {%-   for index, settings in ALL_ADDON_SETTINGS.items() %}
 {%-     if settings.policy is defined %}
-  throttle
+  echo
-  put_policy "{{ index }}-logs" "{{ index }}-logs" '{ "policy": {{ settings.policy | tojson(true) }} }' &
+  echo "Setting up {{ index }}-logs policy..."
+  curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+  echo
 {%-     endif %}
 {%-   endfor %}
 {%- endif %}
-
-wait

salt/manager/managed_soc_annotations.sls

@@ -16,35 +16,40 @@
 {%       endif %}
 {%     endfor %}
 {%   endfor %}
-{%   set soc_annotation_lines = [] %}
-{%   set defaults_lines = [] %}
-{%   for k in matched_integration_names %}
-{%     do soc_annotation_lines.append('    ' ~ k ~ ': *dataStreamSettings') %}
-{%     do defaults_lines.append('    ' ~ k ~ ':') %}
-{%     set defaults_yaml = salt['slsutil.serialize']('yaml', ADDON_INTEGRATION_DEFAULTS[k], default_flow_style=False).strip() %}
-{%     for line in defaults_yaml.splitlines() %}
-{%       do defaults_lines.append('      ' ~ line) %}
-{%     endfor %}
-{%   endfor %}
 {%   set es_soc_annotations = '/opt/so/saltstack/default/salt/elasticsearch/soc_elasticsearch.yaml' %}
-manage_soc_annotations:
+{{   es_soc_annotations }}:
-  file.blockreplace:
+     file.serialize:
-    - name: {{ es_soc_annotations }}
+       - dataset:
-    - marker_start: '    # START managed SOC integration annotations'
+           {% set data = salt['file.read'](es_soc_annotations) | load_yaml %}
-    - marker_end: '    # END managed SOC integration annotations'
+           {% set es = data.get('elasticsearch', {}) %}
-    - content: {{ soc_annotation_lines | join('\n') | tojson }}
+           {% set index_settings = es.get('index_settings', {}) %}
-    - insert_after_match: '^    # Managed SOC integration annotations are inserted below this line\.'
+           {% set input = index_settings.get('so-logs', {}) %}
-    - append_if_not_found: False
+           {% for k in matched_integration_names %}
-    - show_changes: True
+           {%   do index_settings.update({k: input}) %}
+           {% endfor %}
+           {% for k in addon_integration_keys %}
+           {%   if k not in matched_integration_names and k in index_settings %}
+           {%     do index_settings.pop(k) %}
+           {%   endif %}
+           {% endfor %}
+           {{ data }}
 
 {#   Managed elasticsearch/defaults.yaml file for enabling 'Revert to default' via SOC UI for newly added config items #}
 {%   set es_defaults = '/opt/so/saltstack/default/salt/elasticsearch/defaults.yaml' %}
 {{   es_defaults }}:
-  file.blockreplace:
+     file.serialize:
-    - marker_start: '    # START managed SOC integration defaults'
+       - dataset:
-    - marker_end: '    # END managed SOC integration defaults'
+           {% set data = salt['file.read'](es_defaults) | load_yaml %}
-    - content: {{ defaults_lines | join('\n') | tojson }}
+           {% set es = data.get('elasticsearch', {}) %}
-    - insert_after_match: '^  index_settings:$'
+           {% set index_settings = es.get('index_settings', {}) %}
-    - append_if_not_found: False
+           {% for k in matched_integration_names %}
-    - show_changes: True
+           {%   set input = ADDON_INTEGRATION_DEFAULTS[k] %}
-{% endif %}
+           {%     do index_settings.update({k: input})%}
+           {% endfor %}
+           {% for k in addon_integration_keys %}
+           {%   if k not in matched_integration_names and k in index_settings %}
+           {%     do index_settings.pop(k) %}
+           {%   endif %}
+           {% endfor %}
+           {{ data }}
+{% endif %}

salt/manager/tools/sbin/soup

@@ -16,7 +16,6 @@ POSTVERSION=$INSTALLEDVERSION
 INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
 BATCHSIZE=5
 SOUP_LOG=/root/soup.log
-SOUP_DEBUG_LOG=/root/soup-debug.log
 WHATWOULDYOUSAYYAHDOHERE=soup
 whiptail_title='Security Onion UPdater'
 NOTIFYCUSTOMELASTICCONFIG=false
@@ -35,7 +34,6 @@ if [[ -f /etc/salt/cloud.profiles.d/socloud.conf ]]; then
 fi
 # used to display messages to the user at the end of soup
 declare -a FINAL_MESSAGE_QUEUE=()
-SOUP_ERR_CONTEXT=
 
 
 check_err() {
@@ -116,50 +114,11 @@ check_err() {
       echo "$err_msg"
     fi
 
-    if [[ -n $SOUP_ERR_CONTEXT ]]; then
-      echo ""
-      printf '%s\n' "$SOUP_ERR_CONTEXT"
-    fi
-
-    echo "SOUP XTRACE debug log (if enabled) at $SOUP_DEBUG_LOG. Re-run soup with SOUP_DEBUG=1 to create $SOUP_DEBUG_LOG"
-
     exit $exit_code
   fi
 
 }
 
-# Collect bash error context before passing off to check_err()
-on_err() {
-  local exit_code=$?
-  # turn off xtrace to prevent added noise in debug log
-  set +x 2>/dev/null || true
-
-  # Use first error context, multiple errors can happen with command substitutions or nested functions. We just need context from the initial error.
-  [[ -n $SOUP_ERR_CONTEXT ]] && return $exit_code
-
-  local cmd=$BASH_COMMAND
-  local line=${BASH_LINENO[0]}
-  local function=${FUNCNAME[1]:-main}
-  local source=${BASH_SOURCE[1]##*/}
-  local -a err_lines=(
-    "ERROR on: ${cmd}"
-    "  source: ${source}:${line} in ${function}()"
-  )
-  local i caller_line caller_src caller_func
-
-  for ((i=2; i<${#FUNCNAME[@]}-1; i++)); do
-    caller_line=${BASH_LINENO[$((i-1))]}
-    [[ -n $caller_line && $caller_line -gt 0 ]] || continue
-    caller_src=${BASH_SOURCE[$i]##*/}
-    caller_func=${FUNCNAME[$i]:-main}
-    err_lines+=("  called by: ${caller_src}:${caller_line} in ${caller_func}()")
-  done
-
-  SOUP_ERR_CONTEXT=$(printf '%s\n' "${err_lines[@]}")
-
-  return $exit_code
-}
-
 airgap_mounted() {
   # Let's see if the ISO is already mounted.
   if [[ -f /tmp/soagupdate/SecurityOnion/VERSION ]]; then
@@ -384,11 +343,10 @@ highstate() {
 masterlock() {
   echo "Locking Salt Master"
   mv -v $TOPFILE $BACKUPTOPFILE
-  # Render the real top file only for the host running soup; every other
+  echo "base:" > $TOPFILE
-  # minion gets an empty top (no states) while the master is upgrading.
+  echo "  $MINIONID:" >> $TOPFILE
-  echo "{% if grains['id'] == '$MINIONID' %}" > $TOPFILE
+  echo "    - ca" >> $TOPFILE
-  cat $BACKUPTOPFILE >> $TOPFILE
+  echo "    - elasticsearch" >> $TOPFILE
-  echo "{% endif %}" >> $TOPFILE
 }
 
 masterunlock() {
@@ -803,56 +761,9 @@ bootstrap_so_soc_database() {
   echo "so_soc bootstrap complete."
 }
 
-# Existing grids should keep ILM unless an admin explicitly opts in to DLM.
-pin_elasticsearch_data_retention_method() {
-  local elasticsearch_file=/opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls
-  mkdir -p "$(dirname "$elasticsearch_file")"
-  [[ -f "$elasticsearch_file" ]] || touch "$elasticsearch_file"
-
-  if so-yaml.py get -r "$elasticsearch_file" elasticsearch.data_retention_method >/dev/null 2>&1; then
-    echo "elasticsearch.data_retention_method already set; leaving as-is."
-    return 0
-  fi
-
-  echo "Pinning existing grid to ILM data retention."
-  so-yaml.py add "$elasticsearch_file" elasticsearch.data_retention_method ILM
-  chown socore:socore "$elasticsearch_file"
-}
-
-# Addes auto_expand_replicas setting to .kibana_streams index template
-#
-# In Kibana 9.3.3 the auto_expand_replicas setting was not added to the .kibana_streams index template. Causing single node deployments to be stuck in yellow state (unable to assign replica). Here we update the template in place using the so_kibana system user (system managed index template) to include the auto_expand_replicas setting
-#
-# Reference: https://github.com/elastic/kibana/issues/263048
-kibana_backport_streams_index_template() {
-    local current_template updated_template
-
-    set +e
-    if ! current_template=$(so-elasticsearch-query "_index_template/.kibana_streams" --retry 3 --retry-delay 5 --fail); then
-        echo "Index template .kibana_streams does not exist, skipping backport."
-        return 0
-    fi
-    set -e
-
-    updated_template=$(jq '.index_templates[0].index_template | .template.settings += {"index.auto_expand_replicas": "0-1"} | del(.created_date_millis, .modified_date_millis)' <<< "$current_template")
-
-    if ! kibana_user_pass=$(/usr/sbin/so-yaml.py get -r /opt/so/saltstack/local/pillar/elasticsearch/auth.sls elasticsearch.auth.users.so_kibana_user.pass); then
-        echo "Unable to retrieve so_kibana_user password, skipping .kibana_streams index template backport."
-        return 0
-    fi
-
-    if ! so-elasticsearch-query "_index_template/.kibana_streams" -XPUT -d "$updated_template" -u "so_kibana:$kibana_user_pass" --retry 3 --retry-delay 5 --fail; then
-        echo "Unable to automatically update .kibana_streams index template"
-        return 0
-    fi
-
-}
-
 up_to_3.2.0() {
   fix_logstash_0013_lumberjack_pipeline_name
 
-  pin_elasticsearch_data_retention_method
-
   INSTALLEDVERSION=3.2.0
 }
 
@@ -863,8 +774,6 @@ post_to_3.2.0() {
   echo "Regenerating Elastic Agent Installers"
   /sbin/so-elastic-agent-gen-installers
 
-  kibana_backport_streams_index_template
-
   POSTVERSION=3.2.0
 }
 
@@ -2024,20 +1933,4 @@ EOF
   read -r input
 fi
 
-set -o errtrace
+main "$@" | tee -a $SOUP_LOG
-trap on_err ERR
-
-if [[ $SOUP_DEBUG == 1 ]]; then
-  if [ -f $SOUP_DEBUG_LOG ]; then
-    current_time=$(date +%Y%m%d.%H%M%S)
-    mv $SOUP_DEBUG_LOG $SOUP_DEBUG_LOG.$INSTALLEDVERSION.$current_time
-  fi
-  exec {SOUP_XTRACE_FD}>>"$SOUP_DEBUG_LOG"
-  export SOUP_XTRACE_FD
-  BASH_XTRACEFD=$SOUP_XTRACE_FD
-  PS4='+ [${BASH_SOURCE##*/}:${LINENO} ${FUNCNAME[0]:-main}()] | '
-  set -x
-  export SOUP_DEBUG
-fi
-
-main "$@" 2>&1 | tee -a $SOUP_LOG

salt/soc/defaults.yaml

@@ -1464,7 +1464,6 @@ soc:
           sigmaRulePackages:
             - core
             - emerging_threats_addon
-          useEsql: false
         elastic:
           hostUrl:
           remoteHostUrls: []
@@ -1509,6 +1508,8 @@ soc:
         assistant:
           systemPromptAddendum: ""
           systemPromptAddendumMaxLength: 50000
+          maxSubSessionTokens: 0
+          maxDelegationDepth: 0
           adapters:
             - name: SOAI
               protocol: securityonion_ai_cloud

salt/soc/soc_soc.yaml

@@ -383,11 +383,6 @@ soc:
             global: True
             advanced: False
             helpLink: sigma
-          useEsql:
-            description: "(Pre-release) Use Elasticsearch Piped Query Language (ES|QL) instead of EQL (Elastic Query Language) for Elasticsearch queries. The Sigma converter will output ES|QL instead of EQL, allowing support for correlations."
-            global: True
-            advanced: True
-            forcedType: bool
         elastic:
           index:
             description: Comma-separated list of indices or index patterns (wildcard "*" supported) that SOC will search for records.
@@ -719,6 +714,16 @@ soc:
             description: Maximum length of the system prompt addendum. Longer prompts will be truncated.
             global: True
             advanced: True
+          maxSubSessionTokens:
+            description: Maximum number of output tokens a delegated sub-session may generate across all of its turns. When the budget is reached, the sub-agent is halted and its result is returned to the parent agent. Set to 0 to disable the limit.
+            global: True
+            advanced: True
+            forcedType: int
+          maxDelegationDepth:
+            description: Maximum delegation nesting depth for sub-agents. For example, a value of 2 lets the main agent delegate to a sub-agent that may itself delegate one level deeper. Any deeper delegation is refused and the requesting agent continues without it. Set to 0 to disable the limit.
+            global: True
+            advanced: True
+            forcedType: int
           adapters:
             description: Configuration for AI adapters used by the Onion AI assistant. Please see documentation for help on which fields are required for which protocols.
             global: True