From 944e7737599326e7028bf4a3b075f876cfe29b32 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 10 Jun 2026 14:58:49 -0500
Subject: [PATCH 1/6] save exit until all packages have been attempted

---
 .../so-elastic-fleet-package-upgrade          | 23 ++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
index 18211a7c6..a89a0409b 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
@@ -8,18 +8,35 @@
 
 . /usr/sbin/so-elastic-fleet-common
 
+PKG_LOAD_FAILURES=0
+PKG_LOAD_FAILURES_NAMES=()
+
 {%- for PACKAGE in SUPPORTED_PACKAGES %}
 echo "Upgrading {{ PACKAGE }} package..."
 if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
     if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
         # exit 1 on failure to upgrade a default package, allow salt to handle retries
-        echo -e "\nERROR: Failed to upgrade $PACKAGE to version: $VERSION"
-        exit 1
+        echo -e "\nERROR: Failed to upgrade {{ PACKAGE }} to version: $VERSION"
+        PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
+        PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
     fi
 else
-    echo -e "\nERROR: Failed to get version information for integration $PACKAGE"
+    echo -e "\nERROR: Failed to get version information for integration {{ PACKAGE }}"
+    PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
+    PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
 fi
 echo
 {%- endfor %}
+
+if [ $PKG_LOAD_FAILURES -gt 0 ]; then
+    echo "ERROR: Failed to upgrade $PKG_LOAD_FAILURES package(s):"
+    for PKG in "${PKG_LOAD_FAILURES_NAMES[@]}"; do
+        echo " - $PKG"
+    done
+    exit 1
+else
+    echo "Successfully upgraded all packages."
+fi
+
 echo
 /usr/sbin/so-elasticsearch-templates-load

From bd5e77afc58fb6f14136d067b7ae3fa0045590ab Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 10 Jun 2026 14:59:29 -0500
Subject: [PATCH 2/6] increase delay in so-elastic-fleet-package-upgrade
 attempts

---
 salt/elasticfleet/manager.sls | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/elasticfleet/manager.sls b/salt/elasticfleet/manager.sls
index 6cb672bef..04430d496 100644
--- a/salt/elasticfleet/manager.sls
+++ b/salt/elasticfleet/manager.sls
@@ -55,7 +55,7 @@ so-elastic-fleet-package-upgrade:
     - name: /usr/sbin/so-elastic-fleet-package-upgrade
     - retry:
         attempts: 3
-        interval: 10
+        interval: 30
     - onchanges:
       - file: /opt/so/state/elastic_fleet_packages.txt
 

From f905afbc6fda4a94b2090452010d7f8cabf2143b Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 10 Jun 2026 15:01:22 -0500
Subject: [PATCH 3/6] logging

---
 .../tools/sbin_jinja/so-elastic-fleet-package-upgrade         | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
index a89a0409b..8ba250c00 100644
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
@@ -15,13 +15,10 @@ PKG_LOAD_FAILURES_NAMES=()
 echo "Upgrading {{ PACKAGE }} package..."
 if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
     if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
-        # exit 1 on failure to upgrade a default package, allow salt to handle retries
-        echo -e "\nERROR: Failed to upgrade {{ PACKAGE }} to version: $VERSION"
         PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
         PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
     fi
 else
-    echo -e "\nERROR: Failed to get version information for integration {{ PACKAGE }}"
     PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
     PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
 fi
@@ -33,6 +30,7 @@ if [ $PKG_LOAD_FAILURES -gt 0 ]; then
     for PKG in "${PKG_LOAD_FAILURES_NAMES[@]}"; do
         echo " - $PKG"
     done
+    # exit 1 on failure to upgrade a default package, allow salt to handle retries
     exit 1
 else
     echo "Successfully upgraded all packages."

From 289ddda5e8054ef324f2d06a41922e86d5a38542 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 10 Jun 2026 17:06:22 -0500
Subject: [PATCH 4/6] kibana health check for fleet scripts

---
 salt/elasticfleet/enabled.sls                 | 11 ++++++++
 salt/elasticfleet/manager.sls                 | 21 +++++++++++++++
 .../so-elastic-fleet-integration-policy-load  |  7 +++--
 salt/kibana/enabled.sls                       |  3 +++
 salt/kibana/healthcheck.sls                   | 27 +++++++++++++++++++
 5 files changed, 67 insertions(+), 2 deletions(-)
 create mode 100644 salt/kibana/healthcheck.sls

diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls
index 166cb9719..60cf27deb 100644
--- a/salt/elasticfleet/enabled.sls
+++ b/salt/elasticfleet/enabled.sls
@@ -101,6 +101,17 @@ so-elastic-fleet:
       - file: trusttheca
       - x509: etc_elasticfleet_key
       - x509: etc_elasticfleet_crt
+
+wait_for_so-elastic-fleet:
+  http.wait_for_successful_query:
+    - name: "https://localhost:8220/api/status"
+    - ssl: True
+    - verify_ssl: False
+    - status: 200
+    - wait_for: 300
+    - request_interval: 15
+    - require:
+      - docker_container: so-elastic-fleet
 {%   endif %}
 
 delete_so-elastic-fleet_so-status.disabled:
diff --git a/salt/elasticfleet/manager.sls b/salt/elasticfleet/manager.sls
index 04430d496..a0aa83460 100644
--- a/salt/elasticfleet/manager.sls
+++ b/salt/elasticfleet/manager.sls
@@ -9,6 +9,7 @@
 
 include:
   - elasticfleet.config
+  - kibana.healthcheck
 
 # If enabled, automatically update Fleet Logstash Outputs
 {% if ELASTICFLEETMERGED.config.server.enable_auto_configuration %}
@@ -19,6 +20,8 @@ so-elastic-fleet-auto-configure-logstash-outputs:
     - retry:
         attempts: 4
         interval: 30
+    - require:
+      - http: wait_for_so-kibana
 {%   endif %}
 
 # If enabled, automatically update Fleet Server URLs & ES Connection
@@ -28,6 +31,8 @@ so-elastic-fleet-auto-configure-server-urls:
     - retry:
         attempts: 4
         interval: 30
+    - require:
+      - http: wait_for_so-kibana
 {% endif %}
 
 # Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
@@ -37,6 +42,8 @@ so-elastic-fleet-auto-configure-elasticsearch-urls:
     - retry:
         attempts: 4
         interval: 30
+    - require:
+      - http: wait_for_so-kibana
 
 so-elastic-fleet-auto-configure-artifact-urls:
   cmd.run:
@@ -44,6 +51,8 @@ so-elastic-fleet-auto-configure-artifact-urls:
     - retry:
         attempts: 4
         interval: 30
+    - require:
+      - http: wait_for_so-kibana
 
 so-elastic-fleet-package-statefile:
   file.managed:
@@ -56,6 +65,8 @@ so-elastic-fleet-package-upgrade:
     - retry:
         attempts: 3
         interval: 30
+    - require:
+      - http: wait_for_so-kibana
     - onchanges:
       - file: /opt/so/state/elastic_fleet_packages.txt
 
@@ -65,6 +76,8 @@ so-elastic-fleet-integrations:
     - retry:
         attempts: 3
         interval: 10
+    - require:
+      - http: wait_for_so-kibana
 
 so-elastic-agent-grid-upgrade:
   cmd.run:
@@ -72,6 +85,8 @@ so-elastic-agent-grid-upgrade:
     - retry:
         attempts: 12
         interval: 5
+    - require:
+      - http: wait_for_so-kibana
 
 so-elastic-fleet-integration-upgrade:
   cmd.run:
@@ -79,16 +94,22 @@ so-elastic-fleet-integration-upgrade:
     - retry:
         attempts: 3
         interval: 10
+    - require:
+      - http: wait_for_so-kibana
 
 {# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
 so-elastic-fleet-addon-integrations:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
+    - require:
+      - http: wait_for_so-kibana
 
 {% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
 so-elastic-defend-manage-filters-file-watch:
   cmd.run:
     - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log
+    - require:
+      - http: wait_for_so-kibana
     - onchanges:
       - file: elasticdefendcustom
       - file: elasticdefenddisabled
diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load
index e548c7f86..81a3c74be 100644
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load
@@ -108,9 +108,12 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
   done
 
   # Only create the state file if all policies were created/updated successfully
-  if [[ "$RETURN_CODE" != "1" ]]; then
+  if [[ $RETURN_CODE -eq 0 ]]; then
     touch /opt/so/state/eaintegrations.txt
+  else
+    exit 1
   fi
 else
-  exit $RETURN_CODE
+  echo "Fleet integration policies already loaded."
+  exit 0
 fi
diff --git a/salt/kibana/enabled.sls b/salt/kibana/enabled.sls
index 04f44e508..4bb5fef9c 100644
--- a/salt/kibana/enabled.sls
+++ b/salt/kibana/enabled.sls
@@ -10,6 +10,7 @@
 
 include:
   - kibana.config
+  - kibana.healthcheck
   - kibana.sostatus
 
 # Start the kibana docker
@@ -59,6 +60,8 @@ so-kibana:
     {% endif %}
     - watch:
       - file: kibanaconfig
+    - require_in:
+      - http: wait_for_so-kibana
 
 delete_so-kibana_so-status.disabled:
   file.uncomment:
diff --git a/salt/kibana/healthcheck.sls b/salt/kibana/healthcheck.sls
new file mode 100644
index 000000000..f209c3964
--- /dev/null
+++ b/salt/kibana/healthcheck.sls
@@ -0,0 +1,27 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
+
+wait_for_so-kibana:
+  http.wait_for_successful_query:
+    - name: "https://localhost:5601/api/status"
+    - username: 'so_elastic'
+    - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}'
+    - ssl: True
+    - verify_ssl: False
+    - status: 200
+    - wait_for: 300
+    - request_interval: 15
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

From 46655860e926ad0b6692b5ba11ca6218b2d5b9e8 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 10 Jun 2026 17:27:23 -0500
Subject: [PATCH 5/6] http

---
 salt/kibana/healthcheck.sls | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/kibana/healthcheck.sls b/salt/kibana/healthcheck.sls
index f209c3964..bfeb633c5 100644
--- a/salt/kibana/healthcheck.sls
+++ b/salt/kibana/healthcheck.sls
@@ -9,7 +9,7 @@
 
 wait_for_so-kibana:
   http.wait_for_successful_query:
-    - name: "https://localhost:5601/api/status"
+    - name: "http://localhost:5601/api/status"
     - username: 'so_elastic'
     - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}'
     - ssl: True

From 4741cc92bd8d6d84d8551ac10331f091caef2b82 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Wed, 10 Jun 2026 17:52:08 -0500
Subject: [PATCH 6/6] fleet manager start kibana if it isn't already running
 and wait for healthly status

---
 salt/elasticfleet/manager.sls |  2 +-
 salt/kibana/enabled.sls       | 17 ++++++++++++++---
 salt/kibana/healthcheck.sls   | 27 ---------------------------
 3 files changed, 15 insertions(+), 31 deletions(-)
 delete mode 100644 salt/kibana/healthcheck.sls

diff --git a/salt/elasticfleet/manager.sls b/salt/elasticfleet/manager.sls
index a0aa83460..c9fe91d4d 100644
--- a/salt/elasticfleet/manager.sls
+++ b/salt/elasticfleet/manager.sls
@@ -9,7 +9,7 @@
 
 include:
   - elasticfleet.config
-  - kibana.healthcheck
+  - kibana.enabled
 
 # If enabled, automatically update Fleet Logstash Outputs
 {% if ELASTICFLEETMERGED.config.server.enable_auto_configuration %}
diff --git a/salt/kibana/enabled.sls b/salt/kibana/enabled.sls
index 4bb5fef9c..a2fb6cde9 100644
--- a/salt/kibana/enabled.sls
+++ b/salt/kibana/enabled.sls
@@ -6,11 +6,11 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 
 include:
   - kibana.config
-  - kibana.healthcheck
   - kibana.sostatus
 
 # Start the kibana docker
@@ -60,8 +60,19 @@ so-kibana:
     {% endif %}
     - watch:
       - file: kibanaconfig
-    - require_in:
-      - http: wait_for_so-kibana
+
+wait_for_so-kibana:
+  http.wait_for_successful_query:
+    - name: "http://localhost:5601/api/status"
+    - username: 'so_elastic'
+    - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}'
+    - ssl: True
+    - verify_ssl: False
+    - status: 200
+    - wait_for: 300
+    - request_interval: 15
+    - require:
+      - docker_container: so-kibana
 
 delete_so-kibana_so-status.disabled:
   file.uncomment:
diff --git a/salt/kibana/healthcheck.sls b/salt/kibana/healthcheck.sls
deleted file mode 100644
index bfeb633c5..000000000
--- a/salt/kibana/healthcheck.sls
+++ /dev/null
@@ -1,27 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
-
-wait_for_so-kibana:
-  http.wait_for_successful_query:
-    - name: "http://localhost:5601/api/status"
-    - username: 'so_elastic'
-    - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}'
-    - ssl: True
-    - verify_ssl: False
-    - status: 200
-    - wait_for: 300
-    - request_interval: 15
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}