From 3394e9aab78bb832392682593b1a05c2c45190cf Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 9 Jul 2026 11:29:43 -0500
Subject: [PATCH 1/6] increase agent health timeout and add logging for elastic
 agent status output

---
 salt/elasticfleet/tools/sbin/so-elastic-agent-install | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install
index c6a499e94..93889c002 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-agent-install
+++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install
@@ -9,14 +9,19 @@
 
 # passed in as arg from elasticfleet/install_agent_grid.sls, else pulled from pillar later
 GRIDNODETOKEN="$1"
+LOGFILE="/opt/so/SO-Elastic-Agent_Installer_Health.log"
 
 check_agent_health() {
-    timeout=180
+    timeout=300
     interval=10
     start=$SECONDS
 
+    truncate -s 0 "$LOGFILE"
+
     while (( SECONDS - start < timeout )); do
-        if elastic-agent status 2>/dev/null | grep -A1 'elastic-agent$' | grep -q 'status: (HEALTHY)'; then
+        agent_status=$(elastic-agent status 2>&1)
+        echo -e "\n$(date)\n$agent_status\n" >> "$LOGFILE"
+        if echo "$agent_status" | grep -A1 'elastic-agent$' | grep -q 'status: (HEALTHY)'; then
             return 0
         fi
         echo "The Elastic Agent is not yet healthy. Waiting for ${interval} seconds before checking again..."

From 6fa0d327cbafd9a18d38a4f3bb51b1f145e74cfe Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 9 Jul 2026 14:14:58 -0500
Subject: [PATCH 2/6] cause so-setup to fail if there are issues setting up
 fleet

---
 .../tools/sbin_jinja/so-elastic-fleet-setup   | 28 +++++++++++++++----
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
index b62310375..2b677606a 100755
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
@@ -244,11 +244,27 @@ printf '%s\n'\
     "" >> "$global_pillar_file"
 
 # Call Elastic-Fleet Salt State
-printf "\nApplying elasticfleet state"
-salt-call state.apply elasticfleet queue=True
+printf "\nApplying elasticfleet state\n"
+if ! salt-call state.apply elasticfleet queue=True; then
+    printf "\nFailure(s) in elasticfleet state... Exiting...\n"
+    exit 1
+fi
 
 # Generate installers & install Elastic Agent on the node
-so-elastic-agent-gen-installers
-printf "\nApplying elasticfleet.install_agent_grid state"
-salt-call state.apply elasticfleet.install_agent_grid queue=True
-exit 0
+for agent_gen_attempt in {1..3}; do
+    if so-elastic-agent-gen-installers; then
+        break
+    elif [[ $agent_gen_attempt -lt 3 ]]; then
+        printf "\nUnable to generate Elastic Agent installers... Attempt (%s/3). Retrying...\n" "$agent_gen_attempt"
+        sleep 10
+    else
+        printf "\nFailed to generate Elastic Agent installers after 3 attempts. Exiting...\n"
+        exit 1
+    fi
+done
+
+printf "\nApplying elasticfleet.install_agent_grid state\n"
+if ! salt-call state.apply elasticfleet.install_agent_grid queue=True; then
+    printf "\nFailure(s) in elasticfleet.install_agent_grid state... Exiting...\n"
+    exit 1
+fi
\ No newline at end of file

From 8b0759866ea8d2c121c0eb32b9bdf581fed6c7e8 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 9 Jul 2026 14:15:22 -0500
Subject: [PATCH 3/6] verify installers are generated

---
 .../so-elastic-agent-gen-installers           | 39 +++++++++++++++----
 1 file changed, 31 insertions(+), 8 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers
index d25c18e29..de342657f 100755
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers
@@ -30,7 +30,7 @@ done
 if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then 
   printf "\nFleet Host URL, Enrollment Token or Elastic Version empty - exiting..."
   printf "\nFleet Host: $FLEETHOST, Enrollment Token: $ENROLLMENTOKEN\n"
-  exit 
+  exit 1
 fi
 
 OSARCH=( "linux-x86_64" "windows-x86_64" "darwin-x86_64" "darwin-aarch64" )
@@ -62,31 +62,54 @@ do
 done 
 
 GOTARGETOS=( "linux" "windows" "darwin" "darwin/arm64" )
-GOARCH="amd64"
 printf "\n### Generating OS packages using the cleaned up tarballs"
-for GOOS in "${GOTARGETOS[@]}"
-do
+for GOOS in "${GOTARGETOS[@]}"; do
+    GOARCH="amd64"
     if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi
     printf "\n\n### Generating $GOOS/$GOARCH Installer...\n"
     docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \
     --mount type=bind,source=/etc/pki/tls/certs/,target=/workspace/files/cert/ \
     --mount type=bind,source=/nsm/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \
-    --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \
+    --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ \
     {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN"  -o /output/so-elastic-agent_${GOOS}_${GOARCH}
     printf "\n### $GOOS/$GOARCH Installer Generated...\n"
 done
 
 printf "\n\n### Generating MSI...\n"
-cp /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe
+cp /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe
 docker run \
---mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ -w /output \
+--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ -w /output \
 {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs
 printf "\n### MSI Generated...\n"
 
+# Verify installers were created
+for GOOS in "${GOTARGETOS[@]}"; do
+    GOARCH="amd64"
+    if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin"; GOARCH="arm64"; fi
+    if [[ ! -f /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_${GOOS}_${GOARCH} ]]; then
+        printf "\n### ERROR: Installer for %s/%s was not generated. Exiting...\n" "$GOOS" "$GOARCH"
+        exit 1
+    fi
+    # After verifying new installer was generated, move it to so_agent-installers directory
+    mv /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_${GOOS}_${GOARCH} /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/
+done
+
+# Verify MSI installer
+if [[ ! -f /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64_msi ]]; then
+    printf "\n### ERROR: Installer MSI was not generated. Exiting...\n"
+    exit 1
+else
+    # After verifying new installer MSI was generated, move it to so_agent-installers directory
+    mv /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64_msi /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/
+fi
+
 printf "\n### Cleaning up temp files \n"
 rm -rf /nsm/elastic-agent-workspace
-rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe
+rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe
 
 printf "\n### Copying so_agent-installers to /nsm/elastic-fleet/ for nginx.\n"
 \cp -vr /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ /nsm/elastic-fleet/
 chmod 644 /nsm/elastic-fleet/so_agent-installers/*
+
+# if we got here all installers have been generated successfully
+exit 0

From 2959dc95644a4372dadd425fd72929d32e093177 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 9 Jul 2026 14:18:08 -0500
Subject: [PATCH 4/6] keep logs for all 3 attempts

---
 salt/elasticfleet/tools/sbin/so-elastic-agent-install | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install
index 93889c002..8f57d217c 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-agent-install
+++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install
@@ -16,8 +16,6 @@ check_agent_health() {
     interval=10
     start=$SECONDS
 
-    truncate -s 0 "$LOGFILE"
-
     while (( SECONDS - start < timeout )); do
         agent_status=$(elastic-agent status 2>&1)
         echo -e "\n$(date)\n$agent_status\n" >> "$LOGFILE"
@@ -70,6 +68,8 @@ if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then
     attempts=0
     cd /opt/so/ || exit 1
 
+    truncate -s 0 "$LOGFILE"
+
     uninstall_agent
 
     while [[ $attempts -lt 3 ]]; do

From 0b078c48041a30947efbb6dc8bf4945e26272339 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 9 Jul 2026 14:23:45 -0500
Subject: [PATCH 5/6] give the elasticfleet state a few chances to complete
 successfully before exiting 1 causing so-setup to fail

---
 .../tools/sbin_jinja/so-elastic-fleet-setup       | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
index 2b677606a..21d74cf6f 100755
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
@@ -245,10 +245,17 @@ printf '%s\n'\
 
 # Call Elastic-Fleet Salt State
 printf "\nApplying elasticfleet state\n"
-if ! salt-call state.apply elasticfleet queue=True; then
-    printf "\nFailure(s) in elasticfleet state... Exiting...\n"
-    exit 1
-fi
+for state_attempt in {1..3}; do
+    if salt-call state.apply elasticfleet queue=True; then
+        break
+    elif [[ $state_attempt -lt 3 ]]; then
+        printf "\nElasticfleet state did not complete successfully... Attempt (%s/3). Retrying...\n" "$state_attempt"
+        sleep 10
+    else
+        printf "\nFailure(s) in elasticfleet state... Exiting...\n"
+        exit 1
+    fi
+done
 
 # Generate installers & install Elastic Agent on the node
 for agent_gen_attempt in {1..3}; do

From f6a2758321891c9e12f0d4efe29a76ee446067e5 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 9 Jul 2026 14:44:42 -0500
Subject: [PATCH 6/6] add so-elastic-agent-install script to minions

---
 salt/elasticfleet/install_agent_grid.sls | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/salt/elasticfleet/install_agent_grid.sls b/salt/elasticfleet/install_agent_grid.sls
index 431aa6f97..7ed727b48 100644
--- a/salt/elasticfleet/install_agent_grid.sls
+++ b/salt/elasticfleet/install_agent_grid.sls
@@ -10,6 +10,15 @@
 {% set AGENT_STATUS = salt['service.available']('elastic-agent') %}
 {% set AGENT_EXISTS = salt['file.file_exists']('/opt/Elastic/Agent/elastic-agent') %}
 
+so-elastic-agent-install:
+  file.managed:
+    - name: /usr/sbin/so-elastic-agent-install
+    - source: salt://elasticfleet/tools/sbin/so-elastic-agent-install
+    - user: 947
+    - group: 939
+    - mode: 755
+    - show_changes: False
+
 {% if not AGENT_STATUS or not AGENT_EXISTS %}
 
 pull_agent_installer: