From 5b70398c0ab6320f4affe6e9cd48966d6ee718e5 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 17 Sep 2025 15:35:20 -0500
Subject: [PATCH] add error check & retries to
 elastic_fleet_integration_policy_names and associated scripts

---
 salt/elasticfleet/enabled.sls                              | 3 +++
 salt/elasticfleet/tools/sbin/so-elastic-fleet-common       | 7 ++++---
 .../tools/sbin_jinja/so-elastic-fleet-integration-upgrade  | 5 ++++-
 .../sbin_jinja/so-elastic-fleet-optional-integrations-load | 7 ++++++-
 4 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls
index 58409fb48..6b8be2f28 100644
--- a/salt/elasticfleet/enabled.sls
+++ b/salt/elasticfleet/enabled.sls
@@ -158,6 +158,9 @@ so-elastic-agent-grid-upgrade:
 so-elastic-fleet-integration-upgrade:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-integration-upgrade
+    - retry:
+        attempts: 3
+        interval: 10
 
 so-elastic-fleet-addon-integrations:
   cmd.run:
diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
index bbc8dd85f..9dbfbf752 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
@@ -141,10 +141,11 @@ elastic_fleet_agent_policy_ids() {
 
 elastic_fleet_integration_policy_names() {
     AGENT_POLICY=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r .item.package_policies[].name
-    if [ $? -ne 0 ]; then
+    if output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" --retry 3 --fail 2>/dev/null); then
+        echo "$output" | jq -r .item.package_policies[].name
+    else
         echo "Error: Failed to retrieve integrations for '$AGENT_POLICY'."
-        exit 1
+        return 1
     fi
 }
 
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade
index f25059f39..ec0011ea1 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade
@@ -25,7 +25,10 @@ fi
 default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %})
 
 for AGENT_POLICY in $agent_policies; do
-    integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY")
+    if ! integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY")
+        # this script upgrades default integration packages, exit 1 and let salt handle retrying
+        exit 1
+    fi
     for INTEGRATION in $integrations; do
         if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then
             # Get package name so we know what package to look for when checking the current and latest available version
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
index 833f3255d..a3e62df20 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
@@ -62,7 +62,12 @@ default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.l
 in_use_integrations=()
 
 for AGENT_POLICY in $agent_policies; do
-    integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY")
+
+    if ! integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY"); then
+        # skip the agent policy if we can't get required info, let salt retry. Integrations loaded by this script are non-default integrations.
+        echo "Skipping $AGENT_POLICY.. "
+        continue
+    fi
     for INTEGRATION in $integrations; do
         PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION")
         # non-default integrations that are in-use in any policy