From 99a8ad2d4bd4da3989d4d0db19b596c391fbe97e Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:14:12 -0600 Subject: [PATCH 1/2] Wait for Elasticsearch and retry ILM policy loads Require wait_for_so-elasticsearch, check ES is responsive before dispatch, and retry each policy PUT so soup no longer fails with empty output when ES is not ready yet. Fixes https://github.com/Security-Onion-Solutions/securityonion/issues/16369 --- salt/elasticsearch/cluster.sls | 1 + .../so-elasticsearch-ilm-policy-load | 22 ++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/salt/elasticsearch/cluster.sls b/salt/elasticsearch/cluster.sls index 45dcf4111..1c036ea4a 100644 --- a/salt/elasticsearch/cluster.sls +++ b/salt/elasticsearch/cluster.sls @@ -115,6 +115,7 @@ so-elasticsearch-ilm-policy-load: - require: - docker_container: so-elasticsearch - file: so-elasticsearch-ilm-policy-load-script + - http: wait_for_so-elasticsearch - onchanges: - file: so-elasticsearch-ilm-policy-load-script diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load index 9b748ce59..109b002e5 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load @@ -8,10 +8,14 @@ MAX_JOBS=${MAX_ILM_JOBS:-10} +check_elasticsearch_responsive + +ILM_TMP_DIR=$(mktemp -d) +trap 'rm -rf "$ILM_TMP_DIR"' EXIT # Lock used to serialize block writes so concurrent jobs never interleave their output. -ILM_OUTPUT_LOCK=$(mktemp) -ILM_FAIL_FILE=$(mktemp) -trap 'rm -f "$ILM_OUTPUT_LOCK" "$ILM_FAIL_FILE"' EXIT +ILM_OUTPUT_LOCK="$ILM_TMP_DIR/output.lock" +ILM_FAIL_FILE="$ILM_TMP_DIR/failed" +touch "$ILM_FAIL_FILE" # Policies are loaded concurrently (up to MAX_JOBS at a time) for speed. Each policy's block is # printed the moment its curl returns, so output appears in COMPLETION ORDER, not the order @@ -21,11 +25,13 @@ echo put_policy() { local desc="$1" policyname="$2" data="$3" result rc=0 - if ! result=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L --fail \ - -X PUT "https://localhost:9200/_ilm/policy/${policyname}" \ - -H 'Content-Type: application/json' -d"${data}" 2>&1); then - rc=1 - elif ! jq -e '.acknowledged == true' <<<"$result" >/dev/null 2>&1; then + # retry evals and echoes its command, so pass the body by file rather than inline. + local tmpdata="$ILM_TMP_DIR/${policyname}.json" + printf '%s' "$data" >"$tmpdata" + + if result=$(retry 3 3 "so-elasticsearch-query _ilm/policy/${policyname} -d@${tmpdata} -XPUT" "{\"acknowledged\":true}" 2>&1); then + result='{"acknowledged":true}' + else rc=1 fi From 274da18929a63f851fcc8798f8e1b4f577a3f578 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 4 Nov 2026 13:18:31 -0600 Subject: [PATCH 2/2] Surface ILM policy validation errors in highstate Fail 4xx ILM PUTs immediately (retry 429/5xx/connect), print the Elasticsearch reason in the salt summary, and retry the load on the next highstate via /opt/so/state/esilm.txt when the last run failed. Fixes https://github.com/Security-Onion-Solutions/sos/issues/277 --- salt/elasticsearch/cluster.sls | 10 ++++- .../so-elasticsearch-ilm-policy-load | 38 +++++++++++++++---- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/salt/elasticsearch/cluster.sls b/salt/elasticsearch/cluster.sls index 1c036ea4a..df3314ba1 100644 --- a/salt/elasticsearch/cluster.sls +++ b/salt/elasticsearch/cluster.sls @@ -107,6 +107,12 @@ so-elasticsearch-system-indices-patch: - file: so-elasticsearch-system-indices-patch-script {% endif %} +so-elasticsearch-ilm-policy-reload: + file.absent: + - name: /opt/so/state/esilm.txt + - onchanges: + - file: so-elasticsearch-ilm-policy-load-script + # heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml) so-elasticsearch-ilm-policy-load: cmd.run: @@ -116,8 +122,8 @@ so-elasticsearch-ilm-policy-load: - docker_container: so-elasticsearch - file: so-elasticsearch-ilm-policy-load-script - http: wait_for_so-elasticsearch - - onchanges: - - file: so-elasticsearch-ilm-policy-load-script + - file: so-elasticsearch-ilm-policy-reload + - unless: test -f /opt/so/state/esilm.txt so-elasticsearch-templates-reload: file.absent: diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load index 109b002e5..b2f277669 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load @@ -7,6 +7,8 @@ . /usr/sbin/so-common MAX_JOBS=${MAX_ILM_JOBS:-10} +ILM_STATEFILE=/opt/so/state/esilm.txt +rm -f "$ILM_STATEFILE" check_elasticsearch_responsive @@ -24,15 +26,30 @@ echo "Loading ILM policies concurrently; output below appears in completion orde echo put_policy() { - local desc="$1" policyname="$2" data="$3" result rc=0 - # retry evals and echoes its command, so pass the body by file rather than inline. - local tmpdata="$ILM_TMP_DIR/${policyname}.json" + local desc="$1" policyname="$2" data="$3" rc=1 attempt out http_code body result reason tmpdata + tmpdata=$(mktemp "$ILM_TMP_DIR/${policyname}.XXXXXX.json") printf '%s' "$data" >"$tmpdata" - if result=$(retry 3 3 "so-elasticsearch-query _ilm/policy/${policyname} -d@${tmpdata} -XPUT" "{\"acknowledged\":true}" 2>&1); then + for attempt in 1 2 3; do + out=$(so-elasticsearch-query "_ilm/policy/${policyname}" -d@"${tmpdata}" -XPUT -w '\n%{http_code}' 2>/dev/null) + http_code=${out##*$'\n'} body=${out%$'\n'*} + + if [[ "$http_code" == 200 ]] && jq -e '.acknowledged == true' <<<"$body" >/dev/null 2>&1; then + rc=0 + break + fi + # 429 = ES rejected/throttled (may succeed later); other 4xx will not + [[ "$http_code" == 4?? && "$http_code" != 429 ]] && break + (( attempt < 3 )) && sleep 3 + done + + if (( rc == 0 )); then result='{"acknowledged":true}' else - rc=1 + result=${body:-"HTTP ${http_code:-000}"} + # flatten tabs/newlines so the TAB fail-file stays one record + reason=$(jq -r '(if (.error|type) == "object" then (.error.reason // .error.type) elif (.error|type) == "string" then .error else empty end) // empty | tostring | gsub("[\t\r\n]+"; " ")' <<<"$body" 2>/dev/null) + reason="HTTP ${http_code:-000}${reason:+: ${reason}}" fi # curl above ran in parallel; serialize just this block write so concurrent jobs never interleave. @@ -40,7 +57,7 @@ put_policy() { flock 200 printf 'Setting up %s policy...\n%s\n\n' "${desc}" "${result}" if (( rc != 0 )); then - printf '%s\n' "${policyname}" >>"$ILM_FAIL_FILE" + printf '%s\t%s\n' "${policyname}" "${reason}" >>"$ILM_FAIL_FILE" fi } 200>>"${ILM_OUTPUT_LOCK}" @@ -88,10 +105,15 @@ wait || true if [[ -s "$ILM_FAIL_FILE" ]]; then echo "ERROR: Failed to load ILM policy(s):" - while read -r POLICY; do - echo " - $POLICY" + while IFS=$'\t' read -r POLICY REASON; do + if [[ -n "$REASON" ]]; then + echo " - ${POLICY}: ${REASON}" + else + echo " - ${POLICY}" + fi done < "$ILM_FAIL_FILE" exit 1 else echo "Successfully loaded all ILM policies." + touch "$ILM_STATEFILE" fi