From 96fcc0ec3848060ce84ec9b2dde4ac0234704bee Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Mon, 22 Jun 2026 14:25:46 -0500
Subject: [PATCH] wip

---
 .../tools/sbin/so-elastic-fleet-common        | 20 ++++++++---
 .../so-elasticsearch-ilm-policy-load          | 35 +++++++++++++++----
 2 files changed, 44 insertions(+), 11 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
index e8ded916f..855a28510 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
@@ -36,7 +36,7 @@ MAX_FLEET_JOBS=${MAX_FLEET_JOBS:-10}
 # Block until fewer than MAX_FLEET_JOBS background jobs are running.
 elastic_fleet_throttle() {
     while (( $(jobs -rp | wc -l) >= MAX_FLEET_JOBS )); do
-        wait -n
+        wait -n || true
     done
 }
 
@@ -47,7 +47,7 @@ elastic_fleet_throttle() {
 #   $2 DIR              - directory of integration *.json files
 #   $3 LABEL           - human-readable label for log output
 #   $4 SKIP_CREATE_NAME - (optional) integration name to skip when creating (still updated if present)
-# Returns 1 if any integration failed to create/update.
+# Returns 1 if the policy cannot be fetched or if any integration failed to create/update.
 elastic_fleet_load_integrations_dir() {
     local AGENT_POLICY=$1
     local DIR=$2
@@ -62,7 +62,19 @@ elastic_fleet_load_integrations_dir() {
     i=0
 
     # Fetch the agent policy a single time; we look up integration ids locally below.
-    POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY")
+    if ! POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY"); then
+        echo "Error: Failed to retrieve agent policy '$AGENT_POLICY'."
+        rm -f "$FAIL_FILE"
+        rm -rf "$OUT_DIR"
+        return 1
+    fi
+
+    if ! jq -e '.item.package_policies' <<<"$POLICY_JSON" >/dev/null 2>&1; then
+        echo "Error: Invalid agent policy response for '$AGENT_POLICY'."
+        rm -f "$FAIL_FILE"
+        rm -rf "$OUT_DIR"
+        return 1
+    fi
 
     for INTEGRATION in "$DIR"/*.json; do
         [ -e "$INTEGRATION" ] || continue
@@ -90,7 +102,7 @@ elastic_fleet_load_integrations_dir() {
         } >"$OUT_DIR/$(printf '%03d' "$i")" 9>>"$FAIL_FILE" &
         i=$((i+1))
     done
-    wait
+    wait || true
 
     # Emit per-integration output grouped and in submission order (glob sorts numerically).
     cat "$OUT_DIR"/* 2>/dev/null
diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load
index a884f2e2f..9b748ce59 100755
--- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load
+++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load
@@ -6,11 +6,12 @@
 
 . /usr/sbin/so-common
 
-MAX_JOBS=10
+MAX_JOBS=${MAX_ILM_JOBS:-10}
 
 # Lock used to serialize block writes so concurrent jobs never interleave their output.
 ILM_OUTPUT_LOCK=$(mktemp)
-trap 'rm -f "$ILM_OUTPUT_LOCK"' EXIT
+ILM_FAIL_FILE=$(mktemp)
+trap 'rm -f "$ILM_OUTPUT_LOCK" "$ILM_FAIL_FILE"' EXIT
 
 # Policies are loaded concurrently (up to MAX_JOBS at a time) for speed. Each policy's block is
 # printed the moment its curl returns, so output appears in COMPLETION ORDER, not the order
@@ -19,21 +20,31 @@ echo "Loading ILM policies concurrently; output below appears in completion orde
 echo
 
 put_policy() {
-  local desc="$1" policyname="$2" data="$3" result
-  result=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L \
+  local desc="$1" policyname="$2" data="$3" result rc=0
+  if ! result=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L --fail \
     -X PUT "https://localhost:9200/_ilm/policy/${policyname}" \
-    -H 'Content-Type: application/json' -d"${data}")
+    -H 'Content-Type: application/json' -d"${data}" 2>&1); then
+    rc=1
+  elif ! jq -e '.acknowledged == true' <<<"$result" >/dev/null 2>&1; then
+    rc=1
+  fi
+
   # curl above ran in parallel; serialize just this block write so concurrent jobs never interleave.
   {
     flock 200
     printf 'Setting up %s policy...\n%s\n\n' "${desc}" "${result}"
+    if (( rc != 0 )); then
+      printf '%s\n' "${policyname}" >>"$ILM_FAIL_FILE"
+    fi
   } 200>>"${ILM_OUTPUT_LOCK}"
+
+  return "$rc"
 }
 
 # Block until fewer than MAX_JOBS background curls are running.
 throttle() {
   while (( $(jobs -rp | wc -l) >= MAX_JOBS )); do
-    wait -n
+    wait -n || true
   done
 }
 
@@ -67,4 +78,14 @@ throttle() {
 {%-   endfor %}
 {%- endif %}
 
-wait
+wait || true
+
+if [[ -s "$ILM_FAIL_FILE" ]]; then
+  echo "ERROR: Failed to load ILM policy(s):"
+  while read -r POLICY; do
+    echo " - $POLICY"
+  done < "$ILM_FAIL_FILE"
+  exit 1
+else
+  echo "Successfully loaded all ILM policies."
+fi