Compare commits

..
Author SHA1 Message Date
reyesj2 274da18929 Surface ILM policy validation errors in highstate
Fail 4xx ILM PUTs immediately (retry 429/5xx/connect), print the
Elasticsearch reason in the salt summary, and retry the load on the
next highstate via /opt/so/state/esilm.txt when the last run failed.

Fixes https://github.com/Security-Onion-Solutions/sos/issues/277
2026-11-04 13:18:31 -06:00
reyesj2 99a8ad2d4b Wait for Elasticsearch and retry ILM policy loads
Require wait_for_so-elasticsearch, check ES is responsive before
dispatch, and retry each policy PUT so soup no longer fails with
empty output when ES is not ready yet.

Fixes https://github.com/Security-Onion-Solutions/securityonion/issues/16369
2026-11-04 11:14:12 -06:00
Mike Reeves 021b069706 Merge pull request #16367 from Security-Onion-Solutions/TOoSmOotH-patch-1
Bump version from 3.4.0 to 3.5.0
2026-11-02 11:52:06 -05:00
Mike Reeves 824d674eec Add version 3.5.0 to discussion template 2026-11-02 11:50:49 -05:00
Mike Reeves d43791c63e Bump version from 3.4.0 to 3.5.0 2026-11-02 11:50:08 -05:00
4 changed files with 52 additions and 16 deletions

No files matched your search

+1
View File
@@ -14,6 +14,7 @@ body:
- 3.2.0 - 3.2.0
- 3.3.0 - 3.3.0
- 3.4.0 - 3.4.0
- 3.5.0
- Other (please provide detail below) - Other (please provide detail below)
validations: validations:
required: true required: true
+1 -1
View File
@@ -1 +1 @@
3.4.0 3.5.0
+9 -2
View File
@@ -107,6 +107,12 @@ so-elasticsearch-system-indices-patch:
- file: so-elasticsearch-system-indices-patch-script - file: so-elasticsearch-system-indices-patch-script
{% endif %} {% endif %}
so-elasticsearch-ilm-policy-reload:
file.absent:
- name: /opt/so/state/esilm.txt
- onchanges:
- file: so-elasticsearch-ilm-policy-load-script
# heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml) # heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)
so-elasticsearch-ilm-policy-load: so-elasticsearch-ilm-policy-load:
cmd.run: cmd.run:
@@ -115,8 +121,9 @@ so-elasticsearch-ilm-policy-load:
- require: - require:
- docker_container: so-elasticsearch - docker_container: so-elasticsearch
- file: so-elasticsearch-ilm-policy-load-script - file: so-elasticsearch-ilm-policy-load-script
- onchanges: - http: wait_for_so-elasticsearch
- file: so-elasticsearch-ilm-policy-load-script - file: so-elasticsearch-ilm-policy-reload
- unless: test -f /opt/so/state/esilm.txt
so-elasticsearch-templates-reload: so-elasticsearch-templates-reload:
file.absent: file.absent:
@@ -7,11 +7,17 @@
. /usr/sbin/so-common . /usr/sbin/so-common
MAX_JOBS=${MAX_ILM_JOBS:-10} MAX_JOBS=${MAX_ILM_JOBS:-10}
ILM_STATEFILE=/opt/so/state/esilm.txt
rm -f "$ILM_STATEFILE"
check_elasticsearch_responsive
ILM_TMP_DIR=$(mktemp -d)
trap 'rm -rf "$ILM_TMP_DIR"' EXIT
# Lock used to serialize block writes so concurrent jobs never interleave their output. # Lock used to serialize block writes so concurrent jobs never interleave their output.
ILM_OUTPUT_LOCK=$(mktemp) ILM_OUTPUT_LOCK="$ILM_TMP_DIR/output.lock"
ILM_FAIL_FILE=$(mktemp) ILM_FAIL_FILE="$ILM_TMP_DIR/failed"
trap 'rm -f "$ILM_OUTPUT_LOCK" "$ILM_FAIL_FILE"' EXIT touch "$ILM_FAIL_FILE"
# Policies are loaded concurrently (up to MAX_JOBS at a time) for speed. Each policy's block is # Policies are loaded concurrently (up to MAX_JOBS at a time) for speed. Each policy's block is
# printed the moment its curl returns, so output appears in COMPLETION ORDER, not the order # printed the moment its curl returns, so output appears in COMPLETION ORDER, not the order
@@ -20,13 +26,30 @@ echo "Loading ILM policies concurrently; output below appears in completion orde
echo echo
put_policy() { put_policy() {
local desc="$1" policyname="$2" data="$3" result rc=0 local desc="$1" policyname="$2" data="$3" rc=1 attempt out http_code body result reason tmpdata
if ! result=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L --fail \ tmpdata=$(mktemp "$ILM_TMP_DIR/${policyname}.XXXXXX.json")
-X PUT "https://localhost:9200/_ilm/policy/${policyname}" \ printf '%s' "$data" >"$tmpdata"
-H 'Content-Type: application/json' -d"${data}" 2>&1); then
rc=1 for attempt in 1 2 3; do
elif ! jq -e '.acknowledged == true' <<<"$result" >/dev/null 2>&1; then out=$(so-elasticsearch-query "_ilm/policy/${policyname}" -d@"${tmpdata}" -XPUT -w '\n%{http_code}' 2>/dev/null)
rc=1 http_code=${out##*$'\n'} body=${out%$'\n'*}
if [[ "$http_code" == 200 ]] && jq -e '.acknowledged == true' <<<"$body" >/dev/null 2>&1; then
rc=0
break
fi
# 429 = ES rejected/throttled (may succeed later); other 4xx will not
[[ "$http_code" == 4?? && "$http_code" != 429 ]] && break
(( attempt < 3 )) && sleep 3
done
if (( rc == 0 )); then
result='{"acknowledged":true}'
else
result=${body:-"HTTP ${http_code:-000}"}
# flatten tabs/newlines so the TAB fail-file stays one record
reason=$(jq -r '(if (.error|type) == "object" then (.error.reason // .error.type) elif (.error|type) == "string" then .error else empty end) // empty | tostring | gsub("[\t\r\n]+"; " ")' <<<"$body" 2>/dev/null)
reason="HTTP ${http_code:-000}${reason:+: ${reason}}"
fi fi
# curl above ran in parallel; serialize just this block write so concurrent jobs never interleave. # curl above ran in parallel; serialize just this block write so concurrent jobs never interleave.
@@ -34,7 +57,7 @@ put_policy() {
flock 200 flock 200
printf 'Setting up %s policy...\n%s\n\n' "${desc}" "${result}" printf 'Setting up %s policy...\n%s\n\n' "${desc}" "${result}"
if (( rc != 0 )); then if (( rc != 0 )); then
printf '%s\n' "${policyname}" >>"$ILM_FAIL_FILE" printf '%s\t%s\n' "${policyname}" "${reason}" >>"$ILM_FAIL_FILE"
fi fi
} 200>>"${ILM_OUTPUT_LOCK}" } 200>>"${ILM_OUTPUT_LOCK}"
@@ -82,10 +105,15 @@ wait || true
if [[ -s "$ILM_FAIL_FILE" ]]; then if [[ -s "$ILM_FAIL_FILE" ]]; then
echo "ERROR: Failed to load ILM policy(s):" echo "ERROR: Failed to load ILM policy(s):"
while read -r POLICY; do while IFS=$'\t' read -r POLICY REASON; do
echo " - $POLICY" if [[ -n "$REASON" ]]; then
echo " - ${POLICY}: ${REASON}"
else
echo " - ${POLICY}"
fi
done < "$ILM_FAIL_FILE" done < "$ILM_FAIL_FILE"
exit 1 exit 1
else else
echo "Successfully loaded all ILM policies." echo "Successfully loaded all ILM policies."
touch "$ILM_STATEFILE"
fi fi