From 5806999f63c21e9bb1eec7a7f9e98e6f4783eb84 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 17 Sep 2025 11:39:06 -0500
Subject: [PATCH] add error check & retries to
 elastic_fleet_bulk_package_install

---
 salt/elasticfleet/tools/sbin/so-elastic-fleet-common        | 4 +++-
 .../sbin_jinja/so-elastic-fleet-optional-integrations-load  | 6 +++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
index b4ac496df..873cb6e0d 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
@@ -119,7 +119,9 @@ elastic_fleet_package_install() {
 
 elastic_fleet_bulk_package_install() {
     BULK_PKG_LIST=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d@$1 "localhost:5601/api/fleet/epm/packages/_bulk"
+    if ! curl -sK /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d@$1 "localhost:5601/api/fleet/epm/packages/_bulk" --retry 3 --fail 2>/dev/null; then
+        return 1
+    fi
 }
 
 elastic_fleet_package_is_installed() {
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
index 886bbf75c..22ab543ac 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
@@ -160,7 +160,11 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
 
             for file in "${pkg_filename}_"*.json; do
                 [ -e "$file" ] || continue
-                elastic_fleet_bulk_package_install $file >> $BULK_INSTALL_OUTPUT
+                if ! elastic_fleet_bulk_package_install $file >> $BULK_INSTALL_OUTPUT; then
+                    # integrations loaded my this script are non-essential and shouldn't cause exit, skip them for now next highstate run can retry
+                    echo "Failed to complete a chunk of bulk package installs -- $file "
+                    continue
+                fi
             done
             # cleanup any temp files for chunked package install
             rm -f ${pkg_filename}_*.json $BULK_INSTALL_PACKAGE_LIST