From f588a80ec71d0b38f74858360714668fd7c29a6b Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 16 Sep 2025 10:37:26 -0500
Subject: [PATCH 01/32] fix jq error when indices don't exist (seen on fresh
 installs when fleet hasn't ever been installed)

---
 .../tools/sbin_jinja/so-elastic-fleet-setup     | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
index deb16dadf..1ceec9c95 100755
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
@@ -23,18 +23,17 @@ if [[ "$RETURN_CODE" != "0" ]]; then
     exit 1
 fi
 
-ALIASES=".fleet-servers .fleet-policies-leader .fleet-policies .fleet-agents .fleet-artifacts .fleet-enrollment-api-keys .kibana_ingest"
-for ALIAS in ${ALIASES}
-do
+ALIASES=(.fleet-servers .fleet-policies-leader .fleet-policies .fleet-agents .fleet-artifacts .fleet-enrollment-api-keys .kibana_ingest)
+for ALIAS in "${ALIASES[@]}"; do
     # Get all concrete indices from alias
-    INDXS=$(curl -K /opt/so/conf/kibana/curl.config -s -k -L -H "Content-Type: application/json" "https://localhost:9200/_resolve/index/${ALIAS}"  | jq -r '.aliases[].indices[]')
-
-    # Delete all resolved indices
-    for INDX in ${INDXS}
-    do
+    if INDXS_RAW=$(curl -sK /opt/so/conf/kibana/curl.config -s -k -L -H "Content-Type: application/json" "https://localhost:9200/_resolve/index/${ALIAS}" --fail 2>/dev/null); then
+        INDXS=$(echo "$INDXS_RAW" | jq -r '.aliases[].indices[]')
+        # Delete all resolved indices
+        for INDX in ${INDXS}; do
             status "Deleting $INDX"
             curl -K /opt/so/conf/kibana/curl.config -s -k -L -H "Content-Type: application/json" "https://localhost:9200/${INDX}" -XDELETE
-    done
+        done
+    fi
 done
 
 # Restarting Kibana...

From fdb5ad810a717c04f926176bb5a64cc3ccf5d591 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 16 Sep 2025 20:10:48 -0500
Subject: [PATCH 02/32] add err check and retries around func
 elastic_fleet_policy_create

---
 .../tools/sbin/so-elastic-fleet-common         | 18 ++++++++++--------
 .../tools/sbin_jinja/so-elastic-fleet-setup    | 15 ++++++++++++---
 2 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
index 9780c8b12..8f42a9f9a 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
@@ -190,17 +190,19 @@ elastic_fleet_policy_create() {
     NAME=$1
     DESC=$2
     FLEETSERVER=$3
-        TIMEOUT=$4
+    TIMEOUT=$4
 
     JSON_STRING=$( jq -n \
-                    --arg NAME "$NAME" \
-                    --arg DESC "$DESC" \
-                    --arg TIMEOUT $TIMEOUT \
-                    --arg FLEETSERVER "$FLEETSERVER" \
-                    '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER}'
-                    )
+        --arg NAME "$NAME" \
+        --arg DESC "$DESC" \
+        --arg TIMEOUT $TIMEOUT \
+        --arg FLEETSERVER "$FLEETSERVER" \
+            '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER}'
+        )
     # Create Fleet Policy
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+    if ! curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" --retry 3 --fail 2>/dev/null; then
+        return 1
+    fi
 
 }
 
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
index 1ceec9c95..4e84987f6 100755
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
@@ -116,13 +116,22 @@ printf "\n\n"
 /usr/sbin/so-elasticsearch-templates-load
 
 # Initial Endpoints Policy
-elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" "1209600"
+if ! elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" "1209600"; then
+    echo -e "Failed to create endpoints-initial policy..."
+    exit 1
+fi
 
 # Grid Nodes - General Policy
-elastic_fleet_policy_create "so-grid-nodes_general" "SO Grid Nodes - General Purpose" "false" "1209600"
+if ! elastic_fleet_policy_create "so-grid-nodes_general" "SO Grid Nodes - General Purpose" "false" "1209600"; then
+    echo -e "Failed to create so-grid-nodes_general policy..."
+    exit 1
+fi
 
 # Grid Nodes - Heavy Node Policy
-elastic_fleet_policy_create "so-grid-nodes_heavy" "SO Grid Nodes - Heavy Node" "false" "1209600"
+if ! elastic_fleet_policy_create "so-grid-nodes_heavy" "SO Grid Nodes - Heavy Node" "false" "1209600"; then
+    echo -e "Failed to create so-grid-nodes_heavy policy..."
+    exit 1
+fi
 
 # Load Integrations for default policies
 so-elastic-fleet-integration-policy-load

From bdeb92ab05817b86425aaf2257292dc81704daa9 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 16 Sep 2025 20:30:45 -0500
Subject: [PATCH 03/32] add err check and retries for
 elastic_fleet_integration_create

---
 .../tools/sbin/so-elastic-fleet-common        |  4 +++-
 ...ic-fleet-integration-policy-elastic-defend |  5 ++++-
 .../so-elastic-fleet-integration-policy-load  | 20 +++++++++++++++----
 .../tools/sbin_jinja/so-elastic-fleet-setup   |  5 ++++-
 4 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
index 8f42a9f9a..66f1dcf7a 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
@@ -39,7 +39,9 @@ elastic_fleet_integration_create() {
 
     JSON_STRING=$1
 
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+    if ! curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" --retry 3 --fail 2>/dev/null; then
+        return 1
+    fi
 }
 
 
diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-defend b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-defend
index 636942490..312c84be6 100755
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-defend
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-defend
@@ -18,6 +18,9 @@ do
       elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"
   else
     printf "\n\nIntegration does not exist - Creating integration\n"
-    elastic_fleet_integration_create "@$INTEGRATION"
+    if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+        exit 1
+    fi
   fi
 done
diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load
index 26414a94b..b26b79695 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load
@@ -28,7 +28,10 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
       elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
     else
       printf "\n\nIntegration does not exist - Creating integration\n"
-      elastic_fleet_integration_create "@$INTEGRATION"
+      if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+        exit 1
+      fi
     fi
   done
 
@@ -42,7 +45,10 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
       elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
     else
       printf "\n\nIntegration does not exist - Creating integration\n"
-      elastic_fleet_integration_create "@$INTEGRATION"
+      if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+        exit 1
+      fi
     fi
   done
   if [[ "$RETURN_CODE" != "1" ]]; then
@@ -60,7 +66,10 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
     else
       printf "\n\nIntegration does not exist - Creating integration\n"
       if [ "$NAME" != "elasticsearch-logs" ]; then
-        elastic_fleet_integration_create "@$INTEGRATION"
+        if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+          echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+          exit 1
+        fi
       fi
     fi
   done
@@ -81,7 +90,10 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
       else
         printf "\n\nIntegration does not exist - Creating integration\n"
         if [ "$NAME" != "elasticsearch-logs" ]; then
-          elastic_fleet_integration_create "@$INTEGRATION"
+          if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+            echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+            exit 1
+          fi
         fi
       fi
     fi
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
index 4e84987f6..0510b6bfe 100755
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
@@ -73,7 +73,10 @@ UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "FleetServer_{{ GLOBALS.hostname
 .name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json)
 
 # Add the Fleet Server Integration to the new Fleet Policy
-elastic_fleet_integration_create "$UPDATED_INTEGRATION_POLICY"
+if ! elastic_fleet_integration_create "$UPDATED_INTEGRATION_POLICY"; then
+    echo -e "\nFailed to create Fleet server integration for Manager.."
+    exit 1
+fi
 
 # Now we can create the Logstash Output and set it to to be the default Output
 printf "\n\nCreate Logstash Output Config if node is not an Import or Eval install\n"

From 948d72c282abe86c29999cb711ec6979383bc424 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 16 Sep 2025 21:07:02 -0500
Subject: [PATCH 04/32] add error check and retry to
 elastic_fleet_integration_update

---
 .../tools/sbin/so-elastic-fleet-common        |  6 +++--
 ...et-integration-policy-elastic-fleet-server |  6 ++++-
 .../so-elastic-fleet-integration-policy-load  | 22 ++++++++++++++-----
 3 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
index 66f1dcf7a..42e563562 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
@@ -39,7 +39,7 @@ elastic_fleet_integration_create() {
 
     JSON_STRING=$1
 
-    if ! curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" --retry 3 --fail 2>/dev/null; then
+    if ! curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" --retry 3 --fail 2>/dev/null; then
         return 1
     fi
 }
@@ -67,7 +67,9 @@ elastic_fleet_integration_update() {
 
     JSON_STRING=$2
 
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+    if ! curl -sK /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" --retry 3 --fail 2>/dev/null; then
+        return 1
+    fi
 }
 
 elastic_fleet_integration_policy_upgrade() {
diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server
index 8f7c8b8b4..caa684829 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server
@@ -25,5 +25,9 @@ for POLICYNAME in $POLICY; do
     .name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json)
 
     # Now update the integration policy using the modified JSON
-    elastic_fleet_integration_update "$INTEGRATION_ID" "$UPDATED_INTEGRATION_POLICY"
+    if ! elastic_fleet_integration_update "$INTEGRATION_ID" "$UPDATED_INTEGRATION_POLICY"; then
+        # exit 1 on failure to update fleet integration policies, let salt handle retries
+        echo "Failed to update $POLICYNAME.."
+        exit 1
+    fi
 done
\ No newline at end of file
diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load
index b26b79695..8427b47bc 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load
@@ -13,7 +13,7 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
   /usr/sbin/so-elastic-fleet-package-upgrade
 
   # Second, update Fleet Server policies
-  /sbin/so-elastic-fleet-integration-policy-elastic-fleet-server
+  /usr/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server
 
   # Third, configure Elastic Defend Integration seperately
   /usr/sbin/so-elastic-fleet-integration-policy-elastic-defend
@@ -25,7 +25,10 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
     elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
     if [ -n "$INTEGRATION_ID" ]; then
       printf "\n\nIntegration $NAME exists - Updating integration\n"
-      elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
+      if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+        echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+        exit 1
+      fi
     else
       printf "\n\nIntegration does not exist - Creating integration\n"
       if ! elastic_fleet_integration_create "@$INTEGRATION"; then
@@ -42,7 +45,10 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
     elastic_fleet_integration_check "so-grid-nodes_general" "$INTEGRATION"
     if [ -n "$INTEGRATION_ID" ]; then
       printf "\n\nIntegration $NAME exists - Updating integration\n"
-      elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
+      if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+        echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+        exit 1
+      fi
     else
       printf "\n\nIntegration does not exist - Creating integration\n"
       if ! elastic_fleet_integration_create "@$INTEGRATION"; then
@@ -62,7 +68,10 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
     elastic_fleet_integration_check "so-grid-nodes_heavy" "$INTEGRATION"
     if [ -n "$INTEGRATION_ID" ]; then
       printf "\n\nIntegration $NAME exists - Updating integration\n"
-      elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
+      if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+        echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+        exit 1
+      fi
     else
       printf "\n\nIntegration does not exist - Creating integration\n"
       if [ "$NAME" != "elasticsearch-logs" ]; then
@@ -86,7 +95,10 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
       elastic_fleet_integration_check "$FLEET_POLICY" "$INTEGRATION"
       if [ -n "$INTEGRATION_ID" ]; then
         printf "\n\nIntegration $NAME exists - Updating integration\n"
-        elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
+        if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+          echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+          exit 1
+        fi
       else
         printf "\n\nIntegration does not exist - Creating integration\n"
         if [ "$NAME" != "elasticsearch-logs" ]; then

From 94e8cd84e6f2984ea703fea432b546ebc17cbf00 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 16 Sep 2025 21:07:33 -0500
Subject: [PATCH 05/32] because of more aggressive exits use salt to rerun
 script as needed

---
 salt/elasticfleet/enabled.sls | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls
index 0ca54ccb8..797291dfe 100644
--- a/salt/elasticfleet/enabled.sls
+++ b/salt/elasticfleet/enabled.sls
@@ -141,6 +141,9 @@ so-elastic-fleet-package-upgrade:
 so-elastic-fleet-integrations:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-integration-policy-load
+    - retry:
+        attempts: 3
+        interval: 10
 
 so-elastic-agent-grid-upgrade:
   cmd.run:

From bcd2e95fbe3a77a55b98a8487aba43f190b70f5e Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 16 Sep 2025 21:22:03 -0500
Subject: [PATCH 06/32] add error checking and retries to
 elastic_fleet_integration_policy_upgrade

---
 salt/elasticfleet/tools/sbin/so-elastic-fleet-common         | 4 +++-
 .../sbin/so-elastic-fleet-integration-policy-elastic-defend  | 5 ++++-
 .../tools/sbin_jinja/so-elastic-fleet-integration-upgrade    | 3 +--
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
index 42e563562..f76c6d64e 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
@@ -81,7 +81,9 @@ elastic_fleet_integration_policy_upgrade() {
                     '{"packagePolicyIds":[$INTEGRATIONID]}'
                     )
 
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies/upgrade" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+    if ! curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies/upgrade" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" --retry 3 --fail 2>/dev/null; then
+        return 1
+    fi
 }
 
 
diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-defend b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-defend
index 312c84be6..9769f2f79 100755
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-defend
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-defend
@@ -15,7 +15,10 @@ do
   elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
   if [ -n "$INTEGRATION_ID" ]; then
       printf "\n\nIntegration $NAME exists - Upgrading integration policy\n"
-      elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"
+      if ! elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"; then
+        echo -e "\nFailed to upgrade integration policy for ${INTEGRATION##*/}"
+        exit 1
+      fi
   else
     printf "\n\nIntegration does not exist - Creating integration\n"
     if ! elastic_fleet_integration_create "@$INTEGRATION"; then
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade
index 68a644798..f25059f39 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade
@@ -62,8 +62,7 @@ for AGENT_POLICY in $agent_policies; do
                     # If no errors with dry run, proceed with actual upgrade
                     if [[ "$DRYRUN_ERRORS" == "false" ]]; then
                         echo "No errors detected. Proceeding with upgrade..."
-                        elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"
-                        if [ $? -ne 0 ]; then
+                        if ! elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"; then
                             echo "Error: Upgrade failed for $PACKAGE_NAME with integration ID '$INTEGRATION_ID'."
                             exit 1
                         fi

From 063a2b33488a81304e7c06fe9ca04e89ebee0350 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 16 Sep 2025 21:56:53 -0500
Subject: [PATCH 07/32] update elastic_fleet_package_version_check &
 elastic_fleet_package_install to add error checking + retries. Update related
 scripts

---
 salt/elasticfleet/enabled.sls                      |  3 +++
 .../tools/sbin/so-elastic-fleet-common             | 14 +++++++++++---
 .../tools/sbin_jinja/so-elastic-fleet-package-load | 12 ++++++++++--
 .../sbin_jinja/so-elastic-fleet-package-upgrade    | 11 +++++++++--
 4 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls
index 797291dfe..58409fb48 100644
--- a/salt/elasticfleet/enabled.sls
+++ b/salt/elasticfleet/enabled.sls
@@ -135,6 +135,9 @@ so-elastic-fleet-package-statefile:
 so-elastic-fleet-package-upgrade:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-package-upgrade
+    - retry:
+        attempts: 3
+        interval: 10
     - onchanges:
       - file: /opt/so/state/elastic_fleet_packages.txt
 
diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
index f76c6d64e..b4ac496df 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
@@ -89,24 +89,32 @@ elastic_fleet_integration_policy_upgrade() {
 
 elastic_fleet_package_version_check() {
     PACKAGE=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.version'
+
+    if output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" --retry 3 --fail 2>/dev/null); then
+        echo $output | jq -r '.item.version'
+    else
+        return 1
+    fi
 }
 
 elastic_fleet_package_latest_version_check() {
     PACKAGE=$1
-    if output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" --fail); then
+    if output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" --retry 3 --fail 2>/dev/null); then
         if version=$(jq -e -r '.item.latestVersion' <<< $output); then
             echo "$version"
         fi
     else
         echo "Error: Failed to get latest version for $PACKAGE"
+        return 1
     fi
 }
 
 elastic_fleet_package_install() {
     PKG=$1
     VERSION=$2
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  -d '{"force":true}' "localhost:5601/api/fleet/epm/packages/$PKG/$VERSION"
+    if ! curl -sK /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  -d '{"force":true}' "localhost:5601/api/fleet/epm/packages/$PKG/$VERSION" --retry 3 --fail 2>/dev/null; then
+        return 1
+    fi
 }
 
 elastic_fleet_bulk_package_install() {
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load
index 819d7ecff..52fa96cd5 100755
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load
@@ -10,8 +10,16 @@
 
 {%- for PACKAGE in SUPPORTED_PACKAGES %}
 echo "Setting up {{ PACKAGE }} package..."
-VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}")
-elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"
+if VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}"); then
+    if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
+        # packages loaded by this script should never fail to install and REQUIRED before an installation of SO can be considered successful
+        echo -e "\nERROR: Failed to install default integration package -- $PACKAGE $VERSION"
+        exit 1
+    fi
+else
+    echo -e "\nERROR: Failed to get version information for integration $PACKAGE"
+    exit 1
+fi
 echo
 {%- endfor %}
 echo
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
index a092e3ecb..18211a7c6 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
@@ -10,8 +10,15 @@
 
 {%- for PACKAGE in SUPPORTED_PACKAGES %}
 echo "Upgrading {{ PACKAGE }} package..."
-VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}")
-elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"
+if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
+    if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
+        # exit 1 on failure to upgrade a default package, allow salt to handle retries
+        echo -e "\nERROR: Failed to upgrade $PACKAGE to version: $VERSION"
+        exit 1
+    fi
+else
+    echo -e "\nERROR: Failed to get version information for integration $PACKAGE"
+fi
 echo
 {%- endfor %}
 echo

From 5806999f63c21e9bb1eec7a7f9e98e6f4783eb84 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 17 Sep 2025 11:39:06 -0500
Subject: [PATCH 08/32] add error check & retries to
 elastic_fleet_bulk_package_install

---
 salt/elasticfleet/tools/sbin/so-elastic-fleet-common        | 4 +++-
 .../sbin_jinja/so-elastic-fleet-optional-integrations-load  | 6 +++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
index b4ac496df..873cb6e0d 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
@@ -119,7 +119,9 @@ elastic_fleet_package_install() {
 
 elastic_fleet_bulk_package_install() {
     BULK_PKG_LIST=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d@$1 "localhost:5601/api/fleet/epm/packages/_bulk"
+    if ! curl -sK /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d@$1 "localhost:5601/api/fleet/epm/packages/_bulk" --retry 3 --fail 2>/dev/null; then
+        return 1
+    fi
 }
 
 elastic_fleet_package_is_installed() {
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
index 886bbf75c..22ab543ac 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
@@ -160,7 +160,11 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
 
             for file in "${pkg_filename}_"*.json; do
                 [ -e "$file" ] || continue
-                elastic_fleet_bulk_package_install $file >> $BULK_INSTALL_OUTPUT
+                if ! elastic_fleet_bulk_package_install $file >> $BULK_INSTALL_OUTPUT; then
+                    # integrations loaded my this script are non-essential and shouldn't cause exit, skip them for now next highstate run can retry
+                    echo "Failed to complete a chunk of bulk package installs -- $file "
+                    continue
+                fi
             done
             # cleanup any temp files for chunked package install
             rm -f ${pkg_filename}_*.json $BULK_INSTALL_PACKAGE_LIST

From 9e24d21282e86f2fa8bfcc5f04f9998199fa841e Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 17 Sep 2025 11:41:27 -0500
Subject: [PATCH 09/32] remove unused functions from so-elastic-fleet-common

---
 .../tools/sbin/so-elastic-fleet-common        | 22 -------------------
 1 file changed, 22 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
index 873cb6e0d..ee174e159 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
@@ -124,11 +124,6 @@ elastic_fleet_bulk_package_install() {
     fi
 }
 
-elastic_fleet_package_is_installed() {
-    PACKAGE=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.status'
-}
-
 elastic_fleet_installed_packages() {
     curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' -H 'Content-Type: application/json' "localhost:5601/api/fleet/epm/packages/installed?perPage=500"
 }
@@ -141,14 +136,6 @@ elastic_fleet_agent_policy_ids() {
     fi
 }
 
-elastic_fleet_agent_policy_names() {
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq -r .items[].name
-    if [ $? -ne 0 ]; then
-        echo "Error: Failed to retrieve agent policies."
-        exit 1
-    fi
-}
-
 elastic_fleet_integration_policy_names() {
     AGENT_POLICY=$1
     curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r .item.package_policies[].name
@@ -221,12 +208,3 @@ elastic_fleet_policy_create() {
     fi
 
 }
-
-elastic_fleet_policy_update() {
-
-    POLICYID=$1
-    JSON_STRING=$2
-
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/$POLICYID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
-}
-

From d0e875928db9131799e93e7d3777227d699caf0f Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 17 Sep 2025 14:59:13 -0500
Subject: [PATCH 10/32] add error checking and retries for
 elastic_fleet_installed_packages & associated script

---
 salt/elasticfleet/tools/sbin/so-elastic-fleet-common         | 4 +++-
 .../sbin_jinja/so-elastic-fleet-optional-integrations-load   | 5 +++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
index ee174e159..8945c37d9 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
@@ -125,7 +125,9 @@ elastic_fleet_bulk_package_install() {
 }
 
 elastic_fleet_installed_packages() {
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' -H 'Content-Type: application/json' "localhost:5601/api/fleet/epm/packages/installed?perPage=500"
+    if ! curl -sK /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' -H 'Content-Type: application/json' "localhost:5601/api/fleet/epm/packages/installed?perPage=500" --retry 3 --fail 2>/dev/null; then
+        return 1
+    fi
 }
 
 elastic_fleet_agent_policy_ids() {
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
index 22ab543ac..833f3255d 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
@@ -172,8 +172,9 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
             echo "Elastic integrations don't appear to need installation/updating..."
         fi
         # Write out file for generating index/component/ilm templates
-        latest_installed_package_list=$(elastic_fleet_installed_packages)
-        echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
+        if latest_installed_package_list=$(elastic_fleet_installed_packages); then
+            echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
+        fi
         if retry 3 1 "so-elasticsearch-query / --fail --output /dev/null"; then
             # Refresh installed component template list
             latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.')

From f3aaee1e414379fc22443d262f5231ec041e6a71 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 17 Sep 2025 14:59:41 -0500
Subject: [PATCH 11/32] update elastic_fleet_agent_policy_ids scripts already
 check rc

---
 salt/elasticfleet/tools/sbin/so-elastic-fleet-common | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
index 8945c37d9..bbc8dd85f 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
@@ -131,10 +131,11 @@ elastic_fleet_installed_packages() {
 }
 
 elastic_fleet_agent_policy_ids() {
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq -r .items[].id
-    if [ $? -ne 0 ]; then
+    if output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" --retry 3 --fail 2>/dev/null);
+        echo "$output" | jq -r .items[].id
+    else
         echo "Error: Failed to retrieve agent policies."
-        exit 1
+        return 1
     fi
 }
 

From 5b70398c0ab6320f4affe6e9cd48966d6ee718e5 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 17 Sep 2025 15:35:20 -0500
Subject: [PATCH 12/32] add error check & retries to
 elastic_fleet_integration_policy_names and associated scripts

---
 salt/elasticfleet/enabled.sls                              | 3 +++
 salt/elasticfleet/tools/sbin/so-elastic-fleet-common       | 7 ++++---
 .../tools/sbin_jinja/so-elastic-fleet-integration-upgrade  | 5 ++++-
 .../sbin_jinja/so-elastic-fleet-optional-integrations-load | 7 ++++++-
 4 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls
index 58409fb48..6b8be2f28 100644
--- a/salt/elasticfleet/enabled.sls
+++ b/salt/elasticfleet/enabled.sls
@@ -158,6 +158,9 @@ so-elastic-agent-grid-upgrade:
 so-elastic-fleet-integration-upgrade:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-integration-upgrade
+    - retry:
+        attempts: 3
+        interval: 10
 
 so-elastic-fleet-addon-integrations:
   cmd.run:
diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
index bbc8dd85f..9dbfbf752 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
@@ -141,10 +141,11 @@ elastic_fleet_agent_policy_ids() {
 
 elastic_fleet_integration_policy_names() {
     AGENT_POLICY=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r .item.package_policies[].name
-    if [ $? -ne 0 ]; then
+    if output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" --retry 3 --fail 2>/dev/null); then
+        echo "$output" | jq -r .item.package_policies[].name
+    else
         echo "Error: Failed to retrieve integrations for '$AGENT_POLICY'."
-        exit 1
+        return 1
     fi
 }
 
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade
index f25059f39..ec0011ea1 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade
@@ -25,7 +25,10 @@ fi
 default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %})
 
 for AGENT_POLICY in $agent_policies; do
-    integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY")
+    if ! integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY")
+        # this script upgrades default integration packages, exit 1 and let salt handle retrying
+        exit 1
+    fi
     for INTEGRATION in $integrations; do
         if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then
             # Get package name so we know what package to look for when checking the current and latest available version
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
index 833f3255d..a3e62df20 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
@@ -62,7 +62,12 @@ default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.l
 in_use_integrations=()
 
 for AGENT_POLICY in $agent_policies; do
-    integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY")
+
+    if ! integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY"); then
+        # skip the agent policy if we can't get required info, let salt retry. Integrations loaded by this script are non-default integrations.
+        echo "Skipping $AGENT_POLICY.. "
+        continue
+    fi
     for INTEGRATION in $integrations; do
         PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION")
         # non-default integrations that are in-use in any policy

From a5011b398d8accf80efb3f1bcf37aa18b819045e Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 18 Sep 2025 09:39:56 -0500
Subject: [PATCH 13/32] add err check and retries to
 elastic_fleet_integration_policy_package_name and associated scripts

---
 salt/elasticfleet/enabled.sls                              | 1 +
 salt/elasticfleet/tools/sbin/so-elastic-fleet-common       | 7 ++++---
 .../tools/sbin_jinja/so-elastic-fleet-integration-upgrade  | 4 +++-
 .../sbin_jinja/so-elastic-fleet-optional-integrations-load | 5 ++++-
 4 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls
index 6b8be2f28..cef47168f 100644
--- a/salt/elasticfleet/enabled.sls
+++ b/salt/elasticfleet/enabled.sls
@@ -162,6 +162,7 @@ so-elastic-fleet-integration-upgrade:
         attempts: 3
         interval: 10
 
+{# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
 so-elastic-fleet-addon-integrations:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
index 9dbfbf752..32e407487 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
@@ -152,10 +152,11 @@ elastic_fleet_integration_policy_names() {
 elastic_fleet_integration_policy_package_name() {
     AGENT_POLICY=$1
     INTEGRATION=$2
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.name'
-    if [ $? -ne 0 ]; then
+    if output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" --retry 3 --fail 2>/dev/null); then
+        echo "$output" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.name'
+    else
         echo "Error: Failed to retrieve package name for '$INTEGRATION' in '$AGENT_POLICY'."
-        exit 1
+        return 1
     fi
 }
 
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade
index ec0011ea1..8b9cf3639 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade
@@ -32,7 +32,9 @@ for AGENT_POLICY in $agent_policies; do
     for INTEGRATION in $integrations; do
         if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then
             # Get package name so we know what package to look for when checking the current and latest available version
-            PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION")
+            if ! PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION"); then
+                exit 1
+            fi
             {%- if not AUTO_UPGRADE_INTEGRATIONS %}
             if [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then
             {%- endif %}
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
index a3e62df20..896f8adef 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
@@ -69,7 +69,10 @@ for AGENT_POLICY in $agent_policies; do
         continue
     fi
     for INTEGRATION in $integrations; do
-        PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION")
+        if ! PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION")
+            echo  "Not adding $INTEGRATION, couldn't get package name"
+            continue
+        fi
         # non-default integrations that are in-use in any policy
         if ! [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then
             in_use_integrations+=("$PACKAGE_NAME")

From 24a0fa3f6d2a5784dc1fe5ef51d3a45f953bda68 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 18 Sep 2025 10:15:57 -0500
Subject: [PATCH 14/32] add fleet_api wrapper for curl retries

---
 .../tools/sbin/so-elastic-fleet-common        | 29 ++++++++++++-------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
index 32e407487..7334c409a 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
@@ -23,6 +23,13 @@ fi
 # Define a banner to separate sections
 banner="========================================================================="
 
+fleet_api() {
+    local QUERYPATH=$1
+    shift
+
+    curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/${QUERYPATH}" "$@" --retry 3 --fail 2>/dev/null
+}
+
 elastic_fleet_integration_check() {
 
     AGENT_POLICY=$1
@@ -39,7 +46,7 @@ elastic_fleet_integration_create() {
 
     JSON_STRING=$1
 
-    if ! curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" --retry 3 --fail 2>/dev/null; then
+    if ! fleet_api "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPOST -d "$JSON_STRING"; then
         return 1
     fi
 }
@@ -67,7 +74,7 @@ elastic_fleet_integration_update() {
 
     JSON_STRING=$2
 
-    if ! curl -sK /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" --retry 3 --fail 2>/dev/null; then
+    if ! fleet_api "localhost:5601/api/fleet/package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPUT -d "$JSON_STRING"; then
         return 1
     fi
 }
@@ -81,7 +88,7 @@ elastic_fleet_integration_policy_upgrade() {
                     '{"packagePolicyIds":[$INTEGRATIONID]}'
                     )
 
-    if ! curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies/upgrade" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" --retry 3 --fail 2>/dev/null; then
+    if ! fleet_api "localhost:5601/api/fleet/package_policies/upgrade" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
         return 1
     fi
 }
@@ -90,7 +97,7 @@ elastic_fleet_integration_policy_upgrade() {
 elastic_fleet_package_version_check() {
     PACKAGE=$1
 
-    if output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" --retry 3 --fail 2>/dev/null); then
+    if output=$(fleet_api "localhost:5601/api/fleet/epm/packages/$PACKAGE"); then
         echo $output | jq -r '.item.version'
     else
         return 1
@@ -99,7 +106,7 @@ elastic_fleet_package_version_check() {
 
 elastic_fleet_package_latest_version_check() {
     PACKAGE=$1
-    if output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" --retry 3 --fail 2>/dev/null); then
+    if output=$(fleet_api "localhost:5601/api/fleet/epm/packages/$PACKAGE"); then
         if version=$(jq -e -r '.item.latestVersion' <<< $output); then
             echo "$version"
         fi
@@ -112,26 +119,26 @@ elastic_fleet_package_latest_version_check() {
 elastic_fleet_package_install() {
     PKG=$1
     VERSION=$2
-    if ! curl -sK /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  -d '{"force":true}' "localhost:5601/api/fleet/epm/packages/$PKG/$VERSION" --retry 3 --fail 2>/dev/null; then
+    if ! fleet_api "localhost:5601/api/fleet/epm/packages/$PKG/$VERSION" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"force":true}'; then
         return 1
     fi
 }
 
 elastic_fleet_bulk_package_install() {
     BULK_PKG_LIST=$1
-    if ! curl -sK /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d@$1 "localhost:5601/api/fleet/epm/packages/_bulk" --retry 3 --fail 2>/dev/null; then
+    if ! fleet_api "localhost:5601/api/fleet/epm/packages/_bulk" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d@$BULK_PKG_LIST; then
         return 1
     fi
 }
 
 elastic_fleet_installed_packages() {
-    if ! curl -sK /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' -H 'Content-Type: application/json' "localhost:5601/api/fleet/epm/packages/installed?perPage=500" --retry 3 --fail 2>/dev/null; then
+    if ! fleet_api "localhost:5601/api/fleet/epm/packages/installed?perPage=500"; then
         return 1
     fi
 }
 
 elastic_fleet_agent_policy_ids() {
-    if output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" --retry 3 --fail 2>/dev/null);
+    if output=$(fleet_api "localhost:5601/api/fleet/agent_policies");
         echo "$output" | jq -r .items[].id
     else
         echo "Error: Failed to retrieve agent policies."
@@ -141,7 +148,7 @@ elastic_fleet_agent_policy_ids() {
 
 elastic_fleet_integration_policy_names() {
     AGENT_POLICY=$1
-    if output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" --retry 3 --fail 2>/dev/null); then
+    if output=$(fleet_api "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY"); then
         echo "$output" | jq -r .item.package_policies[].name
     else
         echo "Error: Failed to retrieve integrations for '$AGENT_POLICY'."
@@ -152,7 +159,7 @@ elastic_fleet_integration_policy_names() {
 elastic_fleet_integration_policy_package_name() {
     AGENT_POLICY=$1
     INTEGRATION=$2
-    if output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" --retry 3 --fail 2>/dev/null); then
+    if output=$(fleet_api "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY"); then
         echo "$output" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.name'
     else
         echo "Error: Failed to retrieve package name for '$INTEGRATION' in '$AGENT_POLICY'."

From 8b07ff453d781329534bdd632c1de5b266b2f508 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 18 Sep 2025 10:21:07 -0500
Subject: [PATCH 15/32] elastic_fleet_integration_policy_package_version

---
 salt/elasticfleet/tools/sbin/so-elastic-fleet-common | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
index 7334c409a..115055916 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
@@ -171,13 +171,13 @@ elastic_fleet_integration_policy_package_version() {
     AGENT_POLICY=$1
     INTEGRATION=$2
 
-    if output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" --fail); then
-        if version=$(jq -e -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.version' <<< $output); then
+    if output=$(fleet_api "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY"); then
+        if version=$(jq -e -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.version' <<< "$output"); then
             echo "$version"
         fi
     else
         echo "Error: Failed to retrieve agent policy $AGENT_POLICY"
-        exit 1
+        return 1
     fi
 }
 

From f663f22628ecf0d755d2882c60f0bfb8df5f63cd Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 18 Sep 2025 10:27:54 -0500
Subject: [PATCH 16/32] elastic_fleet_integration_id

---
 salt/elasticfleet/tools/sbin/so-elastic-fleet-common       | 7 ++++---
 .../tools/sbin_jinja/so-elastic-fleet-integration-upgrade  | 4 +++-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
index 115055916..80a016908 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
@@ -184,10 +184,11 @@ elastic_fleet_integration_policy_package_version() {
 elastic_fleet_integration_id() {
     AGENT_POLICY=$1
     INTEGRATION=$2
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .id'
-    if [ $? -ne 0 ]; then
+    if output=$(fleet_api "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY"); then
+        echo "$output" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .id'
+    else
         echo "Error: Failed to retrieve integration ID for '$INTEGRATION' in '$AGENT_POLICY'."
-        exit 1
+        return 1
     fi
 }
 
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade
index 8b9cf3639..f24d17210 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade
@@ -53,7 +53,9 @@ for AGENT_POLICY in $agent_policies; do
                 fi
 
                 # Get integration ID
-                INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION")
+                if ! INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION");
+                    exit 1
+                fi
 
                 if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then
                     # Dry run of the upgrade

From faa112eddf80754ce873f87129b04a1d1a65f329 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 18 Sep 2025 12:18:16 -0500
Subject: [PATCH 17/32] update last so-elastic-fleet-common functions

---
 .../tools/sbin/so-elastic-fleet-common        | 45 ++++++++++---------
 .../so-elastic-fleet-integration-upgrade      |  4 +-
 2 files changed, 27 insertions(+), 22 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
index 80a016908..5d8781ba8 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
@@ -46,7 +46,7 @@ elastic_fleet_integration_create() {
 
     JSON_STRING=$1
 
-    if ! fleet_api "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPOST -d "$JSON_STRING"; then
+    if ! fleet_api "package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPOST -d "$JSON_STRING"; then
         return 1
     fi
 }
@@ -65,7 +65,10 @@ elastic_fleet_integration_remove() {
                     '{"packagePolicyIds":[$INTEGRATIONID]}'
                     )
 
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies/delete" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+    if ! fleet_api "package_policies/delete" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
+        echo "Error: Unable to delete '$NAME' from '$AGENT_POLICY'"
+        return 1
+    fi
 }
 
 elastic_fleet_integration_update() {
@@ -74,7 +77,7 @@ elastic_fleet_integration_update() {
 
     JSON_STRING=$2
 
-    if ! fleet_api "localhost:5601/api/fleet/package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPUT -d "$JSON_STRING"; then
+    if ! fleet_api "package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPUT -d "$JSON_STRING"; then
         return 1
     fi
 }
@@ -88,7 +91,7 @@ elastic_fleet_integration_policy_upgrade() {
                     '{"packagePolicyIds":[$INTEGRATIONID]}'
                     )
 
-    if ! fleet_api "localhost:5601/api/fleet/package_policies/upgrade" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
+    if ! fleet_api "package_policies/upgrade" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
         return 1
     fi
 }
@@ -97,21 +100,22 @@ elastic_fleet_integration_policy_upgrade() {
 elastic_fleet_package_version_check() {
     PACKAGE=$1
 
-    if output=$(fleet_api "localhost:5601/api/fleet/epm/packages/$PACKAGE"); then
-        echo $output | jq -r '.item.version'
+    if output=$(fleet_api "epm/packages/$PACKAGE"); then
+        echo "$output" | jq -r '.item.version'
     else
+        echo "Error: Failed to get current package version for '$PACKAGE'"
         return 1
     fi
 }
 
 elastic_fleet_package_latest_version_check() {
     PACKAGE=$1
-    if output=$(fleet_api "localhost:5601/api/fleet/epm/packages/$PACKAGE"); then
+    if output=$(fleet_api "epm/packages/$PACKAGE"); then
         if version=$(jq -e -r '.item.latestVersion' <<< $output); then
             echo "$version"
         fi
     else
-        echo "Error: Failed to get latest version for $PACKAGE"
+        echo "Error: Failed to get latest version for '$PACKAGE'"
         return 1
     fi
 }
@@ -119,26 +123,26 @@ elastic_fleet_package_latest_version_check() {
 elastic_fleet_package_install() {
     PKG=$1
     VERSION=$2
-    if ! fleet_api "localhost:5601/api/fleet/epm/packages/$PKG/$VERSION" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"force":true}'; then
+    if ! fleet_api "epm/packages/$PKG/$VERSION" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"force":true}'; then
         return 1
     fi
 }
 
 elastic_fleet_bulk_package_install() {
     BULK_PKG_LIST=$1
-    if ! fleet_api "localhost:5601/api/fleet/epm/packages/_bulk" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d@$BULK_PKG_LIST; then
+    if ! fleet_api "epm/packages/_bulk" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d@$BULK_PKG_LIST; then
         return 1
     fi
 }
 
 elastic_fleet_installed_packages() {
-    if ! fleet_api "localhost:5601/api/fleet/epm/packages/installed?perPage=500"; then
+    if ! fleet_api "epm/packages/installed?perPage=500"; then
         return 1
     fi
 }
 
 elastic_fleet_agent_policy_ids() {
-    if output=$(fleet_api "localhost:5601/api/fleet/agent_policies");
+    if output=$(fleet_api "agent_policies");
         echo "$output" | jq -r .items[].id
     else
         echo "Error: Failed to retrieve agent policies."
@@ -148,7 +152,7 @@ elastic_fleet_agent_policy_ids() {
 
 elastic_fleet_integration_policy_names() {
     AGENT_POLICY=$1
-    if output=$(fleet_api "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY"); then
+    if output=$(fleet_api "agent_policies/$AGENT_POLICY"); then
         echo "$output" | jq -r .item.package_policies[].name
     else
         echo "Error: Failed to retrieve integrations for '$AGENT_POLICY'."
@@ -159,7 +163,7 @@ elastic_fleet_integration_policy_names() {
 elastic_fleet_integration_policy_package_name() {
     AGENT_POLICY=$1
     INTEGRATION=$2
-    if output=$(fleet_api "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY"); then
+    if output=$(fleet_api "agent_policies/$AGENT_POLICY"); then
         echo "$output" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.name'
     else
         echo "Error: Failed to retrieve package name for '$INTEGRATION' in '$AGENT_POLICY'."
@@ -171,12 +175,12 @@ elastic_fleet_integration_policy_package_version() {
     AGENT_POLICY=$1
     INTEGRATION=$2
 
-    if output=$(fleet_api "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY"); then
+    if output=$(fleet_api "agent_policies/$AGENT_POLICY"); then
         if version=$(jq -e -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.version' <<< "$output"); then
             echo "$version"
         fi
     else
-        echo "Error: Failed to retrieve agent policy $AGENT_POLICY"
+        echo "Error: Failed to retrieve integration version for '$INTEGRATION' in policy '$AGENT_POLICY'"
         return 1
     fi
 }
@@ -184,7 +188,7 @@ elastic_fleet_integration_policy_package_version() {
 elastic_fleet_integration_id() {
     AGENT_POLICY=$1
     INTEGRATION=$2
-    if output=$(fleet_api "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY"); then
+    if output=$(fleet_api "agent_policies/$AGENT_POLICY"); then
         echo "$output" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .id'
     else
         echo "Error: Failed to retrieve integration ID for '$INTEGRATION' in '$AGENT_POLICY'."
@@ -194,10 +198,9 @@ elastic_fleet_integration_id() {
 
 elastic_fleet_integration_policy_dryrun_upgrade() {
     INTEGRATION_ID=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -H "Content-Type: application/json" -H 'kbn-xsrf: true' -L -X POST "localhost:5601/api/fleet/package_policies/upgrade/dryrun" -d "{\"packagePolicyIds\":[\"$INTEGRATION_ID\"]}"
-    if [ $? -ne 0 ]; then
+    if ! fleet_api "package_policies/upgrade/dryrun" -H "Content-Type: application/json" -H 'kbn-xsrf: true' -XPOST -d "{\"packagePolicyIds\":[\"$INTEGRATION_ID\"]}"; then
         echo "Error: Failed to complete dry run for '$INTEGRATION_ID'."
-        exit 1
+        return 1
     fi
 }
 
@@ -216,7 +219,7 @@ elastic_fleet_policy_create() {
             '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER}'
         )
     # Create Fleet Policy
-    if ! curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" --retry 3 --fail 2>/dev/null; then
+    if ! fleet_api "agent_policies" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
         return 1
     fi
 
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade
index f24d17210..318ce45e6 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade
@@ -63,7 +63,9 @@ for AGENT_POLICY in $agent_policies; do
                     echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..."
                     echo "Upgrading $INTEGRATION..."
                     echo "Starting dry run..."
-                    DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID")
+                    if ! DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID")
+                        exit 1
+                    fi
                     DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors)
 
                     # If no errors with dry run, proceed with actual upgrade

From cd5483623b789613bf343d51a4585e8f9a249c80 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 18 Sep 2025 14:33:34 -0500
Subject: [PATCH 18/32] update import/eval fleet output config -- try to
 prevent corrupt dual 'default' output polices from having a successful
 installation

---
 .../tools/sbin_jinja/so-elastic-fleet-setup   | 90 +++++++++++++++----
 salt/manager/tools/sbin/soup                  | 28 ++++++
 2 files changed, 102 insertions(+), 16 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
index 0510b6bfe..066edd2da 100755
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
@@ -50,22 +50,54 @@ if [[ "$RETURN_CODE" != "0" ]]; then
 fi
 
 printf "\n### Create ES Token ###\n"
-ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' |  jq -r .value)
+if ESTOKEN_RAW=$(fleet_api "service_tokens" -XPOST-H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
+    ESTOKEN=$(echo "$ESTOKEN_RAW" | jq -r .value)
+else
+    echo -e "\nFailed to create ES token..."
+    exit 1
+fi
 
 ### Create Outputs, Fleet Policy and Fleet URLs ###
 # Create the Manager Elasticsearch Output first and set it as the default output
 printf "\nAdd Manager Elasticsearch Output...\n"
-ESCACRT=$(openssl x509 -in  $INTCA)
-JSON_STRING=$( jq -n \
-                  --arg ESCACRT "$ESCACRT" \
-                  '{"name":"so-manager_elasticsearch","id":"so-manager_elasticsearch","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200","https://{{ GLOBALS.manager }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate_authorities": [$ESCACRT]}}' )
-curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+ESCACRT=$(openssl x509 -in "$INTCA" -outform DER | sha256sum | cut -d' ' -f1 | tr '[:lower:]' '[:upper:]')
+JSON_STRING=$(jq -n \
+    --arg ESCACRT "$ESCACRT" \
+        '{"name":"so-manager_elasticsearch","id":"so-manager_elasticsearch","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200","https://{{ GLOBALS.manager }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ca_trusted_fingerprint": $ESCACRT}')
+
+if ! fleet_api "outputs" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
+    echo -e "\nFailed to create so-elasticsearch_manager policy..."
+    exit 1
+fi
 printf "\n\n"
 
+# At this point there should only be two policies. fleet-default-output & so-manager_elasticsearch
+status "Verifying so-manager_elasticsearch policy is configured as the current default"
+
+# Grab the fleet-default-output policy instead of so-manager_elasticsearch, because a weird state can exist where both fleet-default-output & so-elasticsearch_manager can be set as the active default output for logs / metrics. Resulting in logs not ingesting on import/eval nodes
+#   Check that fleet-default-output isn't configured as a default for anything
+if DEFAULTPOLICY=$(fleet_api "outputs/fleet-default-output"); then
+    fleet_default=$(echo "$DEFAULTPOLICY" | jq -er '.item.is_default')
+    fleet_default_monitoring=$(echo "$DEFAULTPOLICY" | jq -er '.item.is_default_monitoring')
+    if [[ ! $fleet_default ]] && [[ ! $fleet_default_monitoring ]]; then
+        echo -e "\nso-manager_elasticsearch is configured as the current default policy..."
+    else
+        echo -e "\nVerification of so-manager_elasticsearch policy failed... The default 'fleet-default-output' output is still active..."
+        exit 1
+    fi
+else
+    # fleet-output-policy is created automatically by fleet when started. Should always exist on any installation type
+    echo -e "\nDefault fleet-default-output policy doesn't exist...\n"
+    exit 1
+fi
+
 # Create the Manager Fleet Server Host Agent Policy
 # This has to be done while the Elasticsearch Output is set to the default Output
 printf "Create Manager Fleet Server Policy...\n"
-elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "false" "120"
+if ! elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "false" "120"; then
+    echo -e "\n Failed to create Manager fleet server policy..."
+    exit 1
+fi
 
 # Modify the default integration policy to update the policy_id with the correct naming
 UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "FleetServer_{{ GLOBALS.hostname }}" --arg name "fleet_server-{{ GLOBALS.hostname }}" '
@@ -90,7 +122,10 @@ JSON_STRING=$( jq -n \
                   --arg LOGSTASHCA "$LOGSTASHCA" \
                   '{"name":"grid-logstash","is_default":true,"is_default_monitoring":true,"id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055", "{{ GLOBALS.manager }}:5055"],"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]},"proxy_id":null}'
                   )
-curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+if ! fleet_api "outputs" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
+    echo -e "\nFailed to create logstash fleet output"
+    exit 1
+fi
 printf "\n\n"
 {%- endif %}
 
@@ -108,7 +143,10 @@ else
 fi
 
 ## This array replaces whatever URLs are currently configured
-curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/fleet_server_hosts" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d  "$JSON_STRING"
+if ! fleet_api "fleet_server_hosts" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d  "$JSON_STRING"; then
+    echo -e "\nFailed to add manager fleet URL"
+    exit 1
+fi
 printf "\n\n"
 
 ### Create Policies & Associated Integration Configuration ###
@@ -120,19 +158,19 @@ printf "\n\n"
 
 # Initial Endpoints Policy
 if ! elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" "1209600"; then
-    echo -e "Failed to create endpoints-initial policy..."
+    echo -e "\nFailed to create endpoints-initial policy..."
     exit 1
 fi
 
 # Grid Nodes - General Policy
 if ! elastic_fleet_policy_create "so-grid-nodes_general" "SO Grid Nodes - General Purpose" "false" "1209600"; then
-    echo -e "Failed to create so-grid-nodes_general policy..."
+    echo -e "\nFailed to create so-grid-nodes_general policy..."
     exit 1
 fi
 
 # Grid Nodes - Heavy Node Policy
 if ! elastic_fleet_policy_create "so-grid-nodes_heavy" "SO Grid Nodes - Heavy Node" "false" "1209600"; then
-    echo -e "Failed to create so-grid-nodes_heavy policy..."
+    echo -e "\nFailed to create so-grid-nodes_heavy policy..."
     exit 1
 fi
 
@@ -146,14 +184,34 @@ JSON_STRING=$( jq -n \
                 '{"name":$NAME,"host":$URL,"is_default":true}'
                 )
 
-curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_download_sources" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+if ! fleet_api "agent_download_sources" -XPOST-H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
+    echo -e "\nFailed to update Elastic Agent artifact URL"
+    exit 1
+fi
 
 ### Finalization ###
 
 # Query for Enrollment Tokens for default policies
-ENDPOINTSENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
-GRIDNODESENROLLMENTOKENGENERAL=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_general")) | .api_key')
-GRIDNODESENROLLMENTOKENHEAVY=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_heavy")) | .api_key')
+if ENDPOINTSENROLLMENTOKEN_RAW=$(fleet_api "/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
+    ENDPOINTSENROLLMENTOKEN=$(echo "$ENDPOINTSENROLLMENTOKEN_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
+else
+    echo -e "\nFailed to query for Endpoints enrollment token"
+    exit 1
+fi
+
+if GRIDNODESENROLLMENTOKENGENERAL_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
+    GRIDNODESENROLLMENTOKENGENERAL=$(echo "$GRIDNODESENROLLMENTOKENGENERAL_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_general")) | .api_key')
+else
+    echo -e "\nFailed to query for Grid nodes - General enrollment token"
+    exit 1
+fi
+
+if GRIDNODESENROLLMENTOKENHEAVY_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
+    GRIDNODESENROLLMENTOKENHEAVY=$(echo "$GRIDNODESENROLLMENTOKENHEAVY_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_heavy")) | .api_key')
+else
+    echo -e "\nFailed to query for Grid nodes - Heavy enrollment token"
+    exit 1
+fi
 
 # Store needed data in minion pillar
 pillar_file=/opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls
diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 86595c162..46010da61 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -450,6 +450,7 @@ postupgrade_changes() {
     [[ "$POSTVERSION"  == 2.4.150 ]] && post_to_2.4.160
     [[ "$POSTVERSION"  == 2.4.160 ]] && post_to_2.4.170
     [[ "$POSTVERSION"  == 2.4.170 ]] && post_to_2.4.180
+    [[ "$POSTVERSION"  == 2.4.180 ]] && post_to_2.4.190
     true
 }
 
@@ -608,6 +609,15 @@ post_to_2.4.180() {
   POSTVERSION=2.4.180
 }
 
+post_to_2.4.190() {
+    # Only need to update import / eval nodes
+    if [[ "$MINIONID" =~ "_import" ]] || [[ ! "$MINIONID" =~ "_eval" ]]; then
+        update_import_fleet_output
+    fi
+
+  POSTVERSION=2.4.190
+}
+
 repo_sync() {
   echo "Sync the local repo."
   su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
@@ -870,6 +880,11 @@ up_to_2.4.180() {
   INSTALLEDVERSION=2.4.180
 }
 
+up_to_2.4.190() {
+  echo "Nothing to do for 2.4.190"
+  INSTALLEDVERSION=2.4.190
+}
+
 add_hydra_pillars() {
   mkdir -p /opt/so/saltstack/local/pillar/hydra
   touch /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls
@@ -1143,6 +1158,19 @@ update_elasticsearch_index_settings() {
   done
 }
 
+update_import_fleet_output() {
+    if output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" --retry 3 --fail 2>/dev/null); then
+        # Update the current config of so-manager_elasticsearch output policy in place (leaving any customizations like having changed the preset value from 'balanced' to 'performance')
+        CAFINGERPRINT=$(openssl x509 -in /etc/pki/tls/certs/intca.crt -outform DER | sha256sum | cut -d' ' -f1 | tr '[:lower:]' '[:upper:]')
+        updated_policy=$(jq --args CAFINGERPRINT "$CAFINGERPRINT" '.item | (del(.id) | .ca_trusted_fingerprint = $CAFINGERPRINT)' <<< "$output")
+        if curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" -XPUT -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$updated_policy" --retry 3 --fail 2>/dev/null; then
+            echo "Successfully updated so-manager_elasticsearch fleet output policy"
+        else
+            fail "Failed to update so-manager_elasticsearch fleet output policy"
+        fi
+    fi
+}
+
 update_salt_mine() {
     echo "Populating the mine with mine_functions for each host."
     set +e

From d9eba3cd0eaacc3a8f4c516c2388c349ba163680 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 18 Sep 2025 15:17:22 -0500
Subject: [PATCH 19/32] typo

---
 salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
index 066edd2da..9033d7a0c 100755
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
@@ -50,7 +50,7 @@ if [[ "$RETURN_CODE" != "0" ]]; then
 fi
 
 printf "\n### Create ES Token ###\n"
-if ESTOKEN_RAW=$(fleet_api "service_tokens" -XPOST-H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
+if ESTOKEN_RAW=$(fleet_api "service_tokens" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
     ESTOKEN=$(echo "$ESTOKEN_RAW" | jq -r .value)
 else
     echo -e "\nFailed to create ES token..."

From 336ca0dbbdccc6e7d17268047c78e97efe2512fb Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 18 Sep 2025 15:42:25 -0500
Subject: [PATCH 20/32] typos

---
 salt/elasticfleet/tools/sbin/so-elastic-fleet-common      | 2 +-
 salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
index 5d8781ba8..4ca5030aa 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
@@ -142,7 +142,7 @@ elastic_fleet_installed_packages() {
 }
 
 elastic_fleet_agent_policy_ids() {
-    if output=$(fleet_api "agent_policies");
+    if output=$(fleet_api "agent_policies"); then
         echo "$output" | jq -r .items[].id
     else
         echo "Error: Failed to retrieve agent policies."
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
index 066edd2da..9033d7a0c 100755
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
@@ -50,7 +50,7 @@ if [[ "$RETURN_CODE" != "0" ]]; then
 fi
 
 printf "\n### Create ES Token ###\n"
-if ESTOKEN_RAW=$(fleet_api "service_tokens" -XPOST-H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
+if ESTOKEN_RAW=$(fleet_api "service_tokens" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
     ESTOKEN=$(echo "$ESTOKEN_RAW" | jq -r .value)
 else
     echo -e "\nFailed to create ES token..."

From 878a3f8962668570d0245ef3ae3cecd849c2863f Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 18 Sep 2025 16:05:34 -0500
Subject: [PATCH 21/32] flip logic to check there aren't two default policies
 and fleet-default-output is disabled

---
 salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
index 9033d7a0c..e4ce8be68 100755
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
@@ -75,11 +75,11 @@ printf "\n\n"
 status "Verifying so-manager_elasticsearch policy is configured as the current default"
 
 # Grab the fleet-default-output policy instead of so-manager_elasticsearch, because a weird state can exist where both fleet-default-output & so-elasticsearch_manager can be set as the active default output for logs / metrics. Resulting in logs not ingesting on import/eval nodes
-#   Check that fleet-default-output isn't configured as a default for anything
 if DEFAULTPOLICY=$(fleet_api "outputs/fleet-default-output"); then
     fleet_default=$(echo "$DEFAULTPOLICY" | jq -er '.item.is_default')
     fleet_default_monitoring=$(echo "$DEFAULTPOLICY" | jq -er '.item.is_default_monitoring')
-    if [[ ! $fleet_default ]] && [[ ! $fleet_default_monitoring ]]; then
+#   Check that fleet-default-output isn't configured as a default for anything ( both variables return false )
+    if [[ $fleet_default ]] && [[ $fleet_default_monitoring ]]; then
         echo -e "\nso-manager_elasticsearch is configured as the current default policy..."
     else
         echo -e "\nVerification of so-manager_elasticsearch policy failed... The default 'fleet-default-output' output is still active..."

From 29ac4f23c6166070ca91b7d1682809d20ea88883 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 18 Sep 2025 16:26:37 -0500
Subject: [PATCH 22/32] typo

---
 salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
index e4ce8be68..9fbd92bfd 100755
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
@@ -184,7 +184,7 @@ JSON_STRING=$( jq -n \
                 '{"name":$NAME,"host":$URL,"is_default":true}'
                 )
 
-if ! fleet_api "agent_download_sources" -XPOST-H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
+if ! fleet_api "agent_download_sources" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
     echo -e "\nFailed to update Elastic Agent artifact URL"
     exit 1
 fi

From 87281efc248d089804adc622b5414b6b5463c0be Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 18 Sep 2025 16:41:33 -0500
Subject: [PATCH 23/32] typo

---
 salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
index e4ce8be68..586c68a80 100755
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
@@ -184,7 +184,7 @@ JSON_STRING=$( jq -n \
                 '{"name":$NAME,"host":$URL,"is_default":true}'
                 )
 
-if ! fleet_api "agent_download_sources" -XPOST-H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
+if ! fleet_api "agent_download_sources" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
     echo -e "\nFailed to update Elastic Agent artifact URL"
     exit 1
 fi
@@ -192,7 +192,7 @@ fi
 ### Finalization ###
 
 # Query for Enrollment Tokens for default policies
-if ENDPOINTSENROLLMENTOKEN_RAW=$(fleet_api "/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
+if ENDPOINTSENROLLMENTOKEN_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
     ENDPOINTSENROLLMENTOKEN=$(echo "$ENDPOINTSENROLLMENTOKEN_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
 else
     echo -e "\nFailed to query for Endpoints enrollment token"

From 138849d25891c8376482f5bb60af7a63fe07f256 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 18 Sep 2025 17:33:42 -0500
Subject: [PATCH 24/32] more typos

---
 .../tools/sbin_jinja/so-elastic-fleet-integration-upgrade   | 6 +++---
 .../sbin_jinja/so-elastic-fleet-optional-integrations-load  | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade
index 318ce45e6..f1154af1e 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade
@@ -25,7 +25,7 @@ fi
 default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %})
 
 for AGENT_POLICY in $agent_policies; do
-    if ! integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY")
+    if ! integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY"); then
         # this script upgrades default integration packages, exit 1 and let salt handle retrying
         exit 1
     fi
@@ -53,7 +53,7 @@ for AGENT_POLICY in $agent_policies; do
                 fi
 
                 # Get integration ID
-                if ! INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION");
+                if ! INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION"); then
                     exit 1
                 fi
 
@@ -63,7 +63,7 @@ for AGENT_POLICY in $agent_policies; do
                     echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..."
                     echo "Upgrading $INTEGRATION..."
                     echo "Starting dry run..."
-                    if ! DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID")
+                    if ! DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID"); then
                         exit 1
                     fi
                     DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors)
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
index 896f8adef..01777e5da 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
@@ -69,7 +69,7 @@ for AGENT_POLICY in $agent_policies; do
         continue
     fi
     for INTEGRATION in $integrations; do
-        if ! PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION")
+        if ! PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION"); then
             echo  "Not adding $INTEGRATION, couldn't get package name"
             continue
         fi

From ba710c994462b7cd291077396822faa10d97ddf4 Mon Sep 17 00:00:00 2001
From: Jorge Reyes <94730068+reyesj2@users.noreply.github.com>
Date: Fri, 19 Sep 2025 12:26:08 -0500
Subject: [PATCH 25/32] import or eval should get updated

---
 salt/manager/tools/sbin/soup | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 46010da61..93c35ac26 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -611,7 +611,7 @@ post_to_2.4.180() {
 
 post_to_2.4.190() {
     # Only need to update import / eval nodes
-    if [[ "$MINIONID" =~ "_import" ]] || [[ ! "$MINIONID" =~ "_eval" ]]; then
+    if [[ "$MINIONID" =~ "_import" ]] || [[ "$MINIONID" =~ "_eval" ]]; then
         update_import_fleet_output
     fi
 

From f066baf6ba4c28137579153568a0ae8574c66a6b Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Fri, 19 Sep 2025 12:54:04 -0500
Subject: [PATCH 26/32] use only the characters up to the last seen '_'

---
 salt/manager/tools/sbin/soup | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 46010da61..81e5b9c18 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -611,7 +611,7 @@ post_to_2.4.180() {
 
 post_to_2.4.190() {
     # Only need to update import / eval nodes
-    if [[ "$MINIONID" =~ "_import" ]] || [[ ! "$MINIONID" =~ "_eval" ]]; then
+    if [[ "${MINIONID##*_}" == "import" ]] || [[ "${MINIONID##*_}" == "eval" ]]; then
         update_import_fleet_output
     fi
 
@@ -1429,7 +1429,7 @@ main() {
   if [ "$is_hotfix" == "true" ]; then
     echo "Applying $HOTFIXVERSION hotfix"
     # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars
-    if [[ ! "$MINIONID" =~ "_import" ]]; then
+    if [[ ! "${MINIONID##*_}" == "import" ]]; then
       backup_old_states_pillars
     fi
     copy_new_files
@@ -1492,7 +1492,7 @@ main() {
     fi
 
     # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars
-    if [[ ! "$MINIONID" =~ "_import" ]]; then
+    if [[ ! "${MINIONID##*_}" == "import" ]]; then
       echo ""
       echo "Creating snapshots of default and local Salt states and pillars and saving to /nsm/backup/"
       backup_old_states_pillars

From c92dc580a2b119850b445f289b913f78a525746e Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Fri, 19 Sep 2025 13:17:52 -0500
Subject: [PATCH 27/32] centralize MINION_ROLE lookup_role

---
 salt/common/tools/sbin/so-common | 3 +--
 salt/manager/tools/sbin/soup     | 7 ++++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index 203b54cd0..365852e63 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -441,8 +441,7 @@ lookup_grain() {
 
 lookup_role() {
 	id=$(lookup_grain id)
-	pieces=($(echo $id | tr '_' ' '))
-	echo ${pieces[1]}
+	echo "${id##*_}"
 }
 
 is_feature_enabled() {
diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 81e5b9c18..f324924cb 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -611,7 +611,7 @@ post_to_2.4.180() {
 
 post_to_2.4.190() {
     # Only need to update import / eval nodes
-    if [[ "${MINIONID##*_}" == "import" ]] || [[ "${MINIONID##*_}" == "eval" ]]; then
+    if [[ "$MINION_ROLE" == "import" ]] || [[ "$MINION_ROLE" == "eval" ]]; then
         update_import_fleet_output
     fi
 
@@ -1387,6 +1387,7 @@ main() {
   fi
 
   set_minionid
+  MINION_ROLE=$(lookup_role)
   echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
   echo ""
   if [[ $is_airgap -eq 0 ]]; then
@@ -1429,7 +1430,7 @@ main() {
   if [ "$is_hotfix" == "true" ]; then
     echo "Applying $HOTFIXVERSION hotfix"
     # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars
-    if [[ ! "${MINIONID##*_}" == "import" ]]; then
+    if [[ ! "$MINION_ROLE" == "import" ]]; then
       backup_old_states_pillars
     fi
     copy_new_files
@@ -1492,7 +1493,7 @@ main() {
     fi
 
     # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars
-    if [[ ! "${MINIONID##*_}" == "import" ]]; then
+    if [[ ! "$MINION_ROLE" == "import" ]]; then
       echo ""
       echo "Creating snapshots of default and local Salt states and pillars and saving to /nsm/backup/"
       backup_old_states_pillars

From a3401aad117c51e970718eb4874b0978b5e0cc99 Mon Sep 17 00:00:00 2001
From: Jorge Reyes <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 24 Sep 2025 08:56:40 -0500
Subject: [PATCH 28/32] typo

---
 salt/manager/tools/sbin/soup | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index f324924cb..e49be133f 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -1162,7 +1162,7 @@ update_import_fleet_output() {
     if output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" --retry 3 --fail 2>/dev/null); then
         # Update the current config of so-manager_elasticsearch output policy in place (leaving any customizations like having changed the preset value from 'balanced' to 'performance')
         CAFINGERPRINT=$(openssl x509 -in /etc/pki/tls/certs/intca.crt -outform DER | sha256sum | cut -d' ' -f1 | tr '[:lower:]' '[:upper:]')
-        updated_policy=$(jq --args CAFINGERPRINT "$CAFINGERPRINT" '.item | (del(.id) | .ca_trusted_fingerprint = $CAFINGERPRINT)' <<< "$output")
+        updated_policy=$(jq --arg CAFINGERPRINT "$CAFINGERPRINT" '.item | (del(.id) | .ca_trusted_fingerprint = $CAFINGERPRINT)' <<< "$output")
         if curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" -XPUT -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$updated_policy" --retry 3 --fail 2>/dev/null; then
             echo "Successfully updated so-manager_elasticsearch fleet output policy"
         else

From 23e12811a1d76305db5841a191a2f9e49e77c1c3 Mon Sep 17 00:00:00 2001
From: Jorge Reyes <94730068+reyesj2@users.noreply.github.com>
Date: Thu, 25 Sep 2025 09:51:32 -0500
Subject: [PATCH 29/32] make sure  fleet-default-output is not set as either
 default output policy

---
 salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
index 586c68a80..ee74d1056 100755
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
@@ -79,7 +79,7 @@ if DEFAULTPOLICY=$(fleet_api "outputs/fleet-default-output"); then
     fleet_default=$(echo "$DEFAULTPOLICY" | jq -er '.item.is_default')
     fleet_default_monitoring=$(echo "$DEFAULTPOLICY" | jq -er '.item.is_default_monitoring')
 #   Check that fleet-default-output isn't configured as a default for anything ( both variables return false )
-    if [[ $fleet_default ]] && [[ $fleet_default_monitoring ]]; then
+    if [[ ! $fleet_default ]] && [[ ! $fleet_default_monitoring ]]; then
         echo -e "\nso-manager_elasticsearch is configured as the current default policy..."
     else
         echo -e "\nVerification of so-manager_elasticsearch policy failed... The default 'fleet-default-output' output is still active..."

From d81d9a0722f2dbcc247e6f8882e94ea93a79eda8 Mon Sep 17 00:00:00 2001
From: Matthew Wright <matthew.wright@securityonionsolutions.com>
Date: Thu, 25 Sep 2025 14:45:06 -0400
Subject: [PATCH 30/32] small tweak to investigation prompt

---
 salt/soc/defaults.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml
index 58b3a3827..6caeddbe3 100644
--- a/salt/soc/defaults.yaml
+++ b/salt/soc/defaults.yaml
@@ -2545,7 +2545,7 @@ soc:
               level: 'high'  # info | low | medium | high | critical
         assistant:
           enabled: false
-          investigationPrompt: Investigate Alert ID {socid}
+          investigationPrompt: Investigate Alert ID {socId}
           contextLimitSmall: 200000
           contextLimitLarge: 1000000
           thresholdColorRatioLow: 0.5

From 3a2ceb0b6fb995d7e37d1e320268dbdf7eaf9a28 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Thu, 25 Sep 2025 15:40:00 -0400
Subject: [PATCH 31/32] retry kratos pulls since this is the first image to
 install during setup

---
 salt/kratos/enabled.sls | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/salt/kratos/enabled.sls b/salt/kratos/enabled.sls
index 31097ccf4..f0345edec 100644
--- a/salt/kratos/enabled.sls
+++ b/salt/kratos/enabled.sls
@@ -54,6 +54,9 @@ so-kratos:
       - file: kratosconfig
       - file: kratoslogdir
       - file: kratosdir
+    - retry:
+        attempts: 10
+        interval: 10
 
 delete_so-kratos_so-status.disabled:
   file.uncomment:

From 1fb558cc7764d2952d9cfa3ef5f81eeabe8c52b3 Mon Sep 17 00:00:00 2001
From: Josh Patterson <josh.patterson@securityonionsolutions.com>
Date: Thu, 25 Sep 2025 16:06:25 -0400
Subject: [PATCH 32/32] managerhype br0 setup

---
 salt/salt/map.jinja |  5 ++++-
 setup/so-functions  | 17 ++++++++++++-----
 setup/so-setup      |  2 +-
 3 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja
index 81baa100a..62b7f1b18 100644
--- a/salt/salt/map.jinja
+++ b/salt/salt/map.jinja
@@ -4,7 +4,10 @@
    Elastic License 2.0. #}
 
 {% set role = salt['grains.get']('role', '') %}
-{% if role in ['so-hypervisor','so-managerhype'] %}
+{# We are using usebr0 mostly for setup of the so-managerhype node and controlling when we use br0 vs the physical interface #}
+{% set usebr0 = salt['pillar.get']('usebr0', True) %}
+
+{% if role in ['so-hypervisor','so-managerhype'] and usebr0 %}
 {%   set interface = 'br0' %}
 {% else %}
 {%   set interface = pillar.host.mainint %}
diff --git a/setup/so-functions b/setup/so-functions
index 00f2e46c1..0d7890d17 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -541,8 +541,15 @@ configure_minion() {
 		"log_file: /opt/so/log/salt/minion"\
 		"#startup_states: highstate" >> "$minion_config"
 
-	info "Running: salt-call state.apply salt.mine_functions --local --file-root=../salt/ -l info pillar='{"host": {"mainint": "$MNIC"}}'"
-	salt-call state.apply salt.mine_functions --local --file-root=../salt/ -l info pillar="{'host': {'mainint': $MNIC}}"
+	# At the time the so-managerhype node does not yet have the bridge configured.
+	# The so-hypervisor node doesn't either, but it doesn't cause issues here.
+	local usebr0=false
+	if [ "$minion_type" == 'hypervisor' ]; then
+    	usebr0=true
+	fi
+	local pillar_json="{\"host\": {\"mainint\": \"$MNIC\"}, \"usebr0\": $usebr0}"
+	info "Running: salt-call state.apply salt.mine_functions --local --file-root=../salt/ -l info pillar='$pillar_json'"
+	salt-call state.apply salt.mine_functions --local --file-root=../salt/ -l info pillar="$pillar_json"
 
 	{
 		logCmd "systemctl enable salt-minion";
@@ -1195,9 +1202,9 @@ hypervisor_local_states() {
         logCmd "salt-call state.apply libvirt.64962 --local --file-root=../salt/ -l info queue=True"
         info "Setting up bridge for $MNIC"
 		salt-call state.apply libvirt.bridge --local --file-root=../salt/ -l info pillar='{"host": {"mainint": "'$MNIC'"}}' queue=True
-		if [ $is_managerhype ]; then
-	    	logCmd "salt-call state.apply salt.minion queue=True"
-		fi
+		#if [ $is_managerhype ]; then
+	    #	logCmd "salt-call state.apply salt.minion queue=True"
+		#fi
     fi
 }
 
diff --git a/setup/so-setup b/setup/so-setup
index 347a7165c..ab055fd2d 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -762,6 +762,7 @@ if ! [[ -f $install_opt_file ]]; then
 		fi
 		logCmd "salt-call state.apply common.packages"
 		logCmd "salt-call state.apply common"
+		hypervisor_local_states
 		# this will apply the salt.minion state first since salt.master includes salt.minion
 		logCmd "salt-call state.apply salt.master"
 		# wait here until we get a response from the salt-master since it may have just restarted
@@ -826,7 +827,6 @@ if ! [[ -f $install_opt_file ]]; then
 		checkin_at_boot
 		set_initial_firewall_access
 		logCmd "salt-call schedule.enable -linfo --local"
-		hypervisor_local_states
 		verify_setup
 	else
 		touch /root/accept_changes