Surface ILM policy validation errors in highstate

Fail 4xx ILM PUTs immediately (retry 429/5xx/connect), print the
Elasticsearch reason in the salt summary, and retry the load on the
next highstate via /opt/so/state/esilm.txt when the last run failed.

Fixes https://github.com/Security-Onion-Solutions/sos/issues/277

salt/elasticsearch/cluster.sls

@@ -107,6 +107,12 @@ so-elasticsearch-system-indices-patch:
       - file: so-elasticsearch-system-indices-patch-script
 {%  endif %}
 
+so-elasticsearch-ilm-policy-reload:
+  file.absent:
+    - name: /opt/so/state/esilm.txt
+    - onchanges:
+      - file: so-elasticsearch-ilm-policy-load-script
+
 # heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)
 so-elasticsearch-ilm-policy-load:
   cmd.run:
@@ -116,8 +122,8 @@ so-elasticsearch-ilm-policy-load:
       - docker_container: so-elasticsearch
       - file: so-elasticsearch-ilm-policy-load-script
       - http: wait_for_so-elasticsearch
-    - onchanges:
-      - file: so-elasticsearch-ilm-policy-load-script
+      - file: so-elasticsearch-ilm-policy-reload
+    - unless: test -f /opt/so/state/esilm.txt
 
 so-elasticsearch-templates-reload:
   file.absent:

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load

@@ -7,6 +7,8 @@
 . /usr/sbin/so-common
 
 MAX_JOBS=${MAX_ILM_JOBS:-10}
+ILM_STATEFILE=/opt/so/state/esilm.txt
+rm -f "$ILM_STATEFILE"
 
 check_elasticsearch_responsive
 
@@ -24,15 +26,30 @@ echo "Loading ILM policies concurrently; output below appears in completion orde
 echo
 
 put_policy() {
-  local desc="$1" policyname="$2" data="$3" result rc=0
-  # retry evals and echoes its command, so pass the body by file rather than inline.
-  local tmpdata="$ILM_TMP_DIR/${policyname}.json"
+  local desc="$1" policyname="$2" data="$3" rc=1 attempt out http_code body result reason tmpdata
+  tmpdata=$(mktemp "$ILM_TMP_DIR/${policyname}.XXXXXX.json")
   printf '%s' "$data" >"$tmpdata"
 
-  if result=$(retry 3 3 "so-elasticsearch-query _ilm/policy/${policyname} -d@${tmpdata} -XPUT" "{\"acknowledged\":true}" 2>&1); then
+  for attempt in 1 2 3; do
+    out=$(so-elasticsearch-query "_ilm/policy/${policyname}" -d@"${tmpdata}" -XPUT -w '\n%{http_code}' 2>/dev/null)
+    http_code=${out##*$'\n'} body=${out%$'\n'*}
+
+    if [[ "$http_code" == 200 ]] && jq -e '.acknowledged == true' <<<"$body" >/dev/null 2>&1; then
+      rc=0
+      break
+    fi
+    # 429 = ES rejected/throttled (may succeed later); other 4xx will not
+    [[ "$http_code" == 4?? && "$http_code" != 429 ]] && break
+    (( attempt < 3 )) && sleep 3
+  done
+
+  if (( rc == 0 )); then
     result='{"acknowledged":true}'
   else
-    rc=1
+    result=${body:-"HTTP ${http_code:-000}"}
+    # flatten tabs/newlines so the TAB fail-file stays one record
+    reason=$(jq -r '(if (.error|type) == "object" then (.error.reason // .error.type) elif (.error|type) == "string" then .error else empty end) // empty | tostring | gsub("[\t\r\n]+"; " ")' <<<"$body" 2>/dev/null)
+    reason="HTTP ${http_code:-000}${reason:+: ${reason}}"
   fi
 
   # curl above ran in parallel; serialize just this block write so concurrent jobs never interleave.
@@ -40,7 +57,7 @@ put_policy() {
     flock 200
     printf 'Setting up %s policy...\n%s\n\n' "${desc}" "${result}"
     if (( rc != 0 )); then
-      printf '%s\n' "${policyname}" >>"$ILM_FAIL_FILE"
+      printf '%s\t%s\n' "${policyname}" "${reason}" >>"$ILM_FAIL_FILE"
     fi
   } 200>>"${ILM_OUTPUT_LOCK}"
 
@@ -88,10 +105,15 @@ wait || true
 
 if [[ -s "$ILM_FAIL_FILE" ]]; then
   echo "ERROR: Failed to load ILM policy(s):"
-  while read -r POLICY; do
-    echo " - $POLICY"
+  while IFS=$'\t' read -r POLICY REASON; do
+    if [[ -n "$REASON" ]]; then
+      echo " - ${POLICY}: ${REASON}"
+    else
+      echo " - ${POLICY}"
+    fi
   done < "$ILM_FAIL_FILE"
   exit 1
 else
   echo "Successfully loaded all ILM policies."
+  touch "$ILM_STATEFILE"
 fi