From bd70dd53fb8e9e3acdd68bd7c023327b57f5e270 Mon Sep 17 00:00:00 2001
From: Mike Reeves <reevesmk@gmail.com>
Date: Wed, 15 Jul 2026 12:00:11 -0400
Subject: [PATCH] soup: add cluster-health and Fleet Server pre-flight checks

Before making any changes, verify the grid is in a good state:
 - check_cluster_health: waits for Elasticsearch to reach at least 'yellow'
   (blocks only on red/unreachable, since yellow is normal), modeled on the
   wait in so-elasticsearch-roles-load.
 - check_fleet_server: confirms the Fleet Server status API returns HTTP 200,
   modeled on the wait_for_so-elastic-fleet state in elasticfleet/enabled.sls.

Both run alongside the existing check_pillar_items (manager pillar render) and
verify_es_version_compatibility, before soup modifies anything, so a failure
exits cleanly with an actionable message and no partial changes. Valid on all
manager roles soup runs on (eval/standalone/manager/managerhype/managersearch/
import), which all run Elasticsearch and the Fleet Server.
---
 salt/manager/tools/sbin/soup | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 1200fc423..37e550db2 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -327,6 +327,31 @@ check_pillar_items() {
   fi
 }
 
+check_cluster_health() {
+  echo "Checking Elasticsearch cluster health."
+  # Block only if the cluster cannot reach at least 'yellow' status (i.e. it is red or
+  # unreachable); 'yellow' is normal for many Security Onion deployments. Modeled on the
+  # wait used in so-elasticsearch-roles-load.
+  if so-elasticsearch-query "_cluster/health?wait_for_status=yellow&timeout=120s" --fail > /dev/null 2>&1; then
+    printf "\nThe Elasticsearch cluster is healthy. We can proceed with SOUP.\n\n"
+  else
+    printf "\nThe Elasticsearch cluster is not healthy (it did not reach at least 'yellow' status). Please resolve the cluster health issue before running SOUP again.\n\n"
+    exit 0
+  fi
+}
+
+check_fleet_server() {
+  echo "Checking that Elastic Fleet Server is responding."
+  # Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls,
+  # which waits for HTTP 200 from the Fleet Server status API.
+  if curl -sk --fail --retry 3 --retry-delay 10 --max-time 30 "https://localhost:8220/api/status" > /dev/null 2>&1; then
+    printf "\nElastic Fleet Server is responding. We can proceed with SOUP.\n\n"
+  else
+    printf "\nElastic Fleet Server is not responding at https://localhost:8220/api/status. Please ensure Elastic Fleet is healthy before running SOUP again.\n\n"
+    exit 0
+  fi
+}
+
 check_saltmaster_status() {
   set +e
   echo "Waiting on the Salt Master service to be ready."
@@ -1854,6 +1879,12 @@ main() {
   echo "Verifying Elasticsearch version compatibility across the grid before upgrading."
   verify_es_version_compatibility
 
+  # Pre-flight health checks: confirm the grid is in a good state before we change
+  # anything. These run before any modifications, so a failure exits cleanly and the
+  # operator can fix the issue and re-run soup.
+  check_cluster_health
+  check_fleet_server
+
   echo "Checking for Salt Master and Minion updates."
   upgrade_check_salt
   set -e