From 35f545a858736a7e9b7697d06276a681c1e9374f Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 19 Aug 2026 16:30:38 -0500
Subject: [PATCH] find known problematic system indices and add missing
 auto_expand_replicas configuration to prevent yellow cluster

---
 salt/elasticsearch/cluster.sls                |   9 ++
 salt/elasticsearch/config.sls                 |  10 ++
 .../so-elasticsearch-system-indices-patch     | 123 ++++++++++++++++++
 3 files changed, 142 insertions(+)
 create mode 100644 salt/elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch

diff --git a/salt/elasticsearch/cluster.sls b/salt/elasticsearch/cluster.sls
index a8ccd3780..86407bae3 100644
--- a/salt/elasticsearch/cluster.sls
+++ b/salt/elasticsearch/cluster.sls
@@ -98,6 +98,15 @@ so-es-cluster-settings:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
       - http: wait_for_so-elasticsearch
+      
+so-elasticsearch-system-indices-patch:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-system-indices-patch
+    - require:
+      - http: wait_for_so-elasticsearch
+      - file: so-elasticsearch-system-indices-patch-script
+    - onchanges:
+      - file: so-elasticsearch-system-indices-patch-script
 {%  endif %}
 
 # heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)
diff --git a/salt/elasticsearch/config.sls b/salt/elasticsearch/config.sls
index 8a4674c71..cbfdca406 100644
--- a/salt/elasticsearch/config.sls
+++ b/salt/elasticsearch/config.sls
@@ -42,6 +42,16 @@ elasticsearch_sbin:
     - file_mode: 755
     - exclude_pat:
       - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state
+      - so-elasticsearch-system-indices-patch
+    - show_changes: False
+
+so-elasticsearch-system-indices-patch-script:
+  file.managed:
+    - name: /usr/sbin/so-elasticsearch-system-indices-patch
+    - source: salt://elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch
+    - user: 930
+    - group: 939
+    - mode: 755
     - show_changes: False
 
 elasticsearch_sbin_jinja:
diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch b/salt/elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch
new file mode 100644
index 000000000..72f887194
--- /dev/null
+++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch
@@ -0,0 +1,123 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+set -eo pipefail
+
+SETTINGS='{"index":{"auto_expand_replicas":"0-1"}}'
+KIBANA_PASSWORD=
+INDEX_PATTERNS=(
+    '.entity_analytics.risk_score.lookup-*'
+    '.entity_analytics.watchlists.*'
+    '.workflows-executions'
+    '.workflows-step-executions'
+    '.entities.v2.latest.security_*'
+    '.entities.v2.history.security_*'
+    'risk-score.risk-score-latest-*'
+)
+DATA_STREAM_PATTERNS=(
+    '.entities.v2.updates.security_*'
+    'risk-score.risk-score-*'
+)
+TEMPLATE_PATTERNS=(
+    'entities_v2_latest_security_default_index_template'
+    'entities_v2_history_security_default_index_template'
+    '.entities_v2_updates_security_default_index_template'
+    '.risk-score.risk-score-default-index-template'
+)
+
+query_es() {
+    if so-elasticsearch-query "$@" --fail --retry 3 --retry-delay 5; then
+        return 0
+    fi
+    
+    # retry failed attempts with so_kibana user (system managed indices reject so_elastic user)
+    local query_path="$1"
+    shift
+
+    if [[ -z "$KIBANA_PASSWORD" ]]; then
+        KIBANA_PASSWORD=$(salt-call pillar.get elasticsearch:auth:users:so_kibana_user:pass --out=newline_values_only)
+    fi
+    [[ -n "$KIBANA_PASSWORD" ]] || return 1
+
+    echo "Retrying ${query_path} as so_kibana." >&2
+    curl -K /opt/so/conf/elasticsearch/curl.config --user "so_kibana:${KIBANA_PASSWORD}" \
+        -s -k -L --fail --retry 3 --retry-delay 5 -H 'Content-Type: application/json' "https://localhost:9200/${query_path}" "$@"
+}
+
+# add auto_expand_replicas=0-1 to given index
+set_auto_expand_replicas() {
+    local index="$1"
+
+    echo "Setting auto_expand_replicas to 0-1 on ${index}."
+    query_es "${index}/_settings" -XPUT -d "$SETTINGS" >/dev/null
+}
+
+# resolve index patterns and run set_auto_expand_replicas on each index
+update_system_indices() {
+    local pattern="$1"
+    local response index
+
+    if ! response=$(query_es "_resolve/index/${pattern}?expand_wildcards=all" 2>/dev/null); then
+        return 0
+    fi
+
+    while read -r index; do
+        [[ -n "$index" ]] && set_auto_expand_replicas "$index"
+    done < <(jq -r '.indices[]?.name' <<<"$response")
+}
+
+# get data stream backing indices and run set_auto_expand_replicas on each index
+update_system_ds() {
+    local pattern="$1"
+    local response index
+
+    if ! response=$(query_es "_data_stream/${pattern}?expand_wildcards=all" 2>/dev/null); then
+        return 0
+    fi
+    # find backing indices for each data stream and update with $SETTINGS
+    while read -r index; do
+        [[ -n "$index" ]] && set_auto_expand_replicas "$index"
+    done < <(jq -r '.data_streams[]?.indices[]?.index_name' <<<"$response")
+}
+
+# get index templates, update with auto_expand_replicas=0-1, and PUT back. Keeping mappings/settings/aliases in-place
+update_system_templates() {
+    local pattern="$1"
+    local templates name response template auto_expand_replicas
+
+    if ! templates=$(query_es "_index_template/${pattern}" 2>/dev/null); then
+        return 0
+    fi
+    while read -r name; do
+        response=$(query_es "_index_template/${name}")
+        template=$(jq -c '.index_templates[0].index_template' <<<"$response")
+        auto_expand_replicas=$(jq -r '.template.settings["index.auto_expand_replicas"] // .template.settings.index.auto_expand_replicas // empty' <<<"$template")
+        [[ "$auto_expand_replicas" == "0-1" ]] && continue
+
+        template=$(jq '
+            if (.template.settings.index | type) == "object" then
+                .template.settings.index.auto_expand_replicas = "0-1"
+            else
+                .template.settings["index.auto_expand_replicas"] = "0-1"
+            end
+            | del(.created_date_millis, .modified_date_millis)
+        ' <<<"$template")
+        echo "Setting auto_expand_replicas to 0-1 on index template ${name}."
+        query_es "_index_template/${name}" -XPUT -d "$template" >/dev/null
+    done < <(jq -r '.index_templates[]?.name' <<<"$templates")
+}
+
+for pattern in "${INDEX_PATTERNS[@]}"; do
+    update_system_indices "$pattern"
+done
+
+for pattern in "${DATA_STREAM_PATTERNS[@]}"; do
+    update_system_ds "$pattern"
+done
+
+for pattern in "${TEMPLATE_PATTERNS[@]}"; do
+    update_system_templates "$pattern"
+done