From 063a2b33488a81304e7c06fe9ca04e89ebee0350 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Tue, 16 Sep 2025 21:56:53 -0500
Subject: [PATCH] update elastic_fleet_package_version_check &
 elastic_fleet_package_install to add error checking + retries. Update related
 scripts

---
 salt/elasticfleet/enabled.sls                      |  3 +++
 .../tools/sbin/so-elastic-fleet-common             | 14 +++++++++++---
 .../tools/sbin_jinja/so-elastic-fleet-package-load | 12 ++++++++++--
 .../sbin_jinja/so-elastic-fleet-package-upgrade    | 11 +++++++++--
 4 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls
index 797291dfe..58409fb48 100644
--- a/salt/elasticfleet/enabled.sls
+++ b/salt/elasticfleet/enabled.sls
@@ -135,6 +135,9 @@ so-elastic-fleet-package-statefile:
 so-elastic-fleet-package-upgrade:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-package-upgrade
+    - retry:
+        attempts: 3
+        interval: 10
     - onchanges:
       - file: /opt/so/state/elastic_fleet_packages.txt
 
diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
index f76c6d64e..b4ac496df 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
@@ -89,24 +89,32 @@ elastic_fleet_integration_policy_upgrade() {
 
 elastic_fleet_package_version_check() {
     PACKAGE=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.version'
+
+    if output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" --retry 3 --fail 2>/dev/null); then
+        echo $output | jq -r '.item.version'
+    else
+        return 1
+    fi
 }
 
 elastic_fleet_package_latest_version_check() {
     PACKAGE=$1
-    if output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" --fail); then
+    if output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" --retry 3 --fail 2>/dev/null); then
         if version=$(jq -e -r '.item.latestVersion' <<< $output); then
             echo "$version"
         fi
     else
         echo "Error: Failed to get latest version for $PACKAGE"
+        return 1
     fi
 }
 
 elastic_fleet_package_install() {
     PKG=$1
     VERSION=$2
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  -d '{"force":true}' "localhost:5601/api/fleet/epm/packages/$PKG/$VERSION"
+    if ! curl -sK /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  -d '{"force":true}' "localhost:5601/api/fleet/epm/packages/$PKG/$VERSION" --retry 3 --fail 2>/dev/null; then
+        return 1
+    fi
 }
 
 elastic_fleet_bulk_package_install() {
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load
index 819d7ecff..52fa96cd5 100755
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load
@@ -10,8 +10,16 @@
 
 {%- for PACKAGE in SUPPORTED_PACKAGES %}
 echo "Setting up {{ PACKAGE }} package..."
-VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}")
-elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"
+if VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}"); then
+    if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
+        # packages loaded by this script should never fail to install and REQUIRED before an installation of SO can be considered successful
+        echo -e "\nERROR: Failed to install default integration package -- $PACKAGE $VERSION"
+        exit 1
+    fi
+else
+    echo -e "\nERROR: Failed to get version information for integration $PACKAGE"
+    exit 1
+fi
 echo
 {%- endfor %}
 echo
diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
index a092e3ecb..18211a7c6 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
@@ -10,8 +10,15 @@
 
 {%- for PACKAGE in SUPPORTED_PACKAGES %}
 echo "Upgrading {{ PACKAGE }} package..."
-VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}")
-elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"
+if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
+    if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
+        # exit 1 on failure to upgrade a default package, allow salt to handle retries
+        echo -e "\nERROR: Failed to upgrade $PACKAGE to version: $VERSION"
+        exit 1
+    fi
+else
+    echo -e "\nERROR: Failed to get version information for integration $PACKAGE"
+fi
 echo
 {%- endfor %}
 echo