From 8b488f92267fcb319f8ee7e2f34c3245270d110e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 14 Jul 2026 17:30:58 -0400 Subject: [PATCH 1/5] soup: make failed upgrades and hotfixes resumable A failed highstate mid-upgrade left /etc/soversion already advanced to the target version (the highstate stamps it from the pillar via the soversionfile state), so a re-run of soup saw INSTALLEDVERSION == NEWVERSION and reported "already running the latest version", stranding the box with post-upgrade steps never run. Introduce /etc/sopostversion, a soup-owned marker (no salt state manages it) that records post-upgrade walk progress. It is seeded from the pre-upgrade version before the highstate, advanced after each post_to_* step, and removed on successful completion. upgrade_check treats a leftover marker as "upgrade not finished" and resumes the remaining post steps instead of bailing. Also fix the hotfix path: /etc/sohotfix was written before the hotfix highstate, so a failed hotfix highstate looked already-applied on re-run. Since no salt state manages /etc/sohotfix, defer its write (update_version) until after the highstate succeeds so it is an honest completion marker. --- salt/manager/tools/sbin/soup | 46 ++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index a87dfecfc..18cf6a732 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -12,7 +12,17 @@ UPDATE_DIR=/tmp/sogh/securityonion DEFAULT_SALT_DIR=/opt/so/saltstack/default INSTALLEDVERSION=$(cat /etc/soversion) -POSTVERSION=$INSTALLEDVERSION +# /etc/sopostversion is a soup-owned marker (no salt state manages it) tracking how +# far the post-upgrade walk has progressed. Its presence means a prior upgrade did +# not finish its post-upgrade steps; its contents are the resume point. It is read +# here before preupgrade_changes mutates INSTALLEDVERSION and before any highstate +# stamps /etc/soversion from the pillar. +POSTVERSION_FILE=/etc/sopostversion +if [ -f "$POSTVERSION_FILE" ]; then + POSTVERSION=$(cat "$POSTVERSION_FILE") +else + POSTVERSION=$INSTALLEDVERSION +fi INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}') BATCHSIZE=5 SOUP_LOG=/root/soup.log @@ -414,6 +424,13 @@ preupgrade_changes() { true } +set_postversion() { + # Persist post-upgrade walk progress so an interrupted upgrade can resume the + # remaining steps on the next soup run (see /etc/sopostversion handling). + POSTVERSION="$1" + echo "$POSTVERSION" > "$POSTVERSION_FILE" +} + postupgrade_changes() { # This function is to add any new pillar items if needed. echo "Running post upgrade processes." @@ -421,6 +438,8 @@ postupgrade_changes() { [[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0 [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0 [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0 + # All applicable post-upgrade steps completed; clear the resume marker. + rm -f "$POSTVERSION_FILE" true } @@ -513,7 +532,7 @@ post_to_3.0.0() { # convert yes/no in suricata pillars to true/false convert_suricata_yes_no - POSTVERSION=3.0.0 + set_postversion 3.0.0 } ### 3.0.0 End ### @@ -776,7 +795,7 @@ post_to_3.1.0() { # Check for unhealthy / unauthorized integration transform jobs and attempt reauthorizations check_transform_health_and_reauthorize || true - POSTVERSION=3.1.0 + set_postversion 3.1.0 } ### 3.1.0 End ### @@ -911,7 +930,7 @@ post_to_3.2.0() { update_kafka_metadata "4.3" - POSTVERSION=3.2.0 + set_postversion 3.2.0 } ### 3.2.0 End ### @@ -1063,6 +1082,14 @@ upgrade_check() { fi [[ -f /etc/sohotfix ]] && CURRENTHOTFIX=$(cat /etc/sohotfix) if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then + # A leftover post-version marker means a previous upgrade to this version + # advanced /etc/soversion (the highstate stamps it from the pillar) but did not + # finish its post-upgrade steps. Resume the upgrade instead of reporting "latest". + if [ -f "$POSTVERSION_FILE" ] && [ "$(cat "$POSTVERSION_FILE")" != "$NEWVERSION" ]; then + echo "A previous upgrade to $NEWVERSION did not complete its post-upgrade steps; resuming." + is_hotfix=false + return 0 + fi echo "Checking to see if there are hotfixes needed" if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then echo "You are already running the latest version of Security Onion." @@ -1814,9 +1841,14 @@ main() { create_local_directories "/opt/so/saltstack/default" apply_hotfix echo "Hotfix applied" - update_version enable_highstate highstate + # Record the hotfix only after the highstate succeeds. /etc/sohotfix is written + # solely by soup (no salt state manages it), so deferring the write means a failed + # hotfix highstate leaves the old hotfix value and re-running soup re-applies it, + # rather than reporting "already latest". The soversion/pillar writes in + # update_version are no-ops here since the version is unchanged for a hotfix. + update_version else echo "" echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION." @@ -1873,6 +1905,10 @@ main() { copy_new_files echo "" create_local_directories "/opt/so/saltstack/default" + # Seed the resume marker before the highstate stamps /etc/soversion to the new + # version, so an interrupted upgrade is detectable as "not finished" on re-run. + # POSTVERSION still holds the pre-upgrade (or prior resume) version here. + [ -f "$POSTVERSION_FILE" ] || echo "$POSTVERSION" > "$POSTVERSION_FILE" update_version echo "" From 618712469e55e6e2d55025de095fa5106ef4b6f1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 15 Jul 2026 09:42:59 -0400 Subject: [PATCH 2/5] soup: clearly report incomplete upgrades on trap exit When soup fails via the EXIT trap after it has begun modifying the system, print a prominent UPGRADE INCOMPLETE banner instructing the user to run soup again to resume and complete the update. Gated on a new SOUP_UPGRADE_STARTED flag set at the start of the hotfix and upgrade branches, so pre-flight gate failures (ES compatibility, disk, network) that abort before any changes are made do not show it. --- salt/manager/tools/sbin/soup | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 18cf6a732..4fcdfebf9 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -33,6 +33,10 @@ NOTIFYCUSTOMELASTICCONFIG=false TOPFILE=/opt/so/saltstack/default/salt/top.sls BACKUPTOPFILE=/opt/so/saltstack/default/salt/top.sls.backup SALTUPGRADED=false +# Set true once soup begins modifying the system (past the pre-flight checks), so the +# EXIT trap can tell the user the update did not finish and must be re-run. Only the +# pre-flight gates (ES compatibility, disk, network) fail before this is set. +SOUP_UPGRADE_STARTED=false SALT_CLOUD_INSTALLED=false SALT_CLOUD_CONFIGURED=false # Check if salt-cloud is installed @@ -133,6 +137,28 @@ check_err() { echo "SOUP XTRACE debug log (if enabled) at $SOUP_DEBUG_LOG. Re-run soup with SOUP_DEBUG=1 to create $SOUP_DEBUG_LOG" + # If soup had already started modifying the system, make it unmistakable that the + # update is incomplete and must be re-run. soup is resumable: a version upgrade + # picks up from the /etc/sopostversion marker, and a hotfix re-applies because + # /etc/sohotfix is only advanced after a successful highstate. + if [[ "$SOUP_UPGRADE_STARTED" == "true" ]]; then + echo "" + echo "==============================================================================" + echo " UPGRADE INCOMPLETE" + echo "==============================================================================" + echo " This soup run did NOT finish. Your Security Onion installation may be in a" + echo " partially-updated state and is not yet fully upgraded." + echo "" + echo " Review the error above and $SOUP_LOG, resolve the underlying problem, then" + echo " run soup again to resume and complete the update:" + echo "" + echo " sudo soup" + echo "" + echo " soup is resumable -- re-running it continues from where this run stopped." + echo "==============================================================================" + echo "" + fi + exit $exit_code fi @@ -1831,6 +1857,7 @@ main() { fi if [ "$is_hotfix" == "true" ]; then + SOUP_UPGRADE_STARTED=true echo "Applying $HOTFIXVERSION hotfix" # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars if [[ ! "$MINION_ROLE" == "import" ]]; then @@ -1850,6 +1877,7 @@ main() { # update_version are no-ops here since the version is unchanged for a hotfix. update_version else + SOUP_UPGRADE_STARTED=true echo "" echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION." echo "" From be7d8a2aa77d06ef5773b27ac2fba3fe05addaf9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 15 Jul 2026 11:35:04 -0400 Subject: [PATCH 3/5] soup: make partial-upgrade state clear and avoid re-running completed upgrades After a partial upgrade, /etc/soversion already reads the target version, so soup's startup line "Found that Security Onion X is currently installed" made it look finished even as soup resumed. When a resume marker is present and differs from the installed version, print an explicit NOTE that the grid is only partially upgraded and this run will resume and complete it. Also clear any stale resume marker in the already-latest path so a successfully completed upgrade is never mistaken for a partial one and re-run on a later invocation (the marker is normally removed at the end of postupgrade_changes; this is a belt-and-suspenders guard). --- salt/manager/tools/sbin/soup | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 4fcdfebf9..1200fc423 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1118,6 +1118,10 @@ upgrade_check() { fi echo "Checking to see if there are hotfixes needed" if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then + # Reaching here means we are at the target version and NOT resuming (the resume + # check above returned otherwise). Clear any stale resume marker so a completed + # upgrade is never mistaken for a partial one and re-run on a later invocation. + rm -f "$POSTVERSION_FILE" echo "You are already running the latest version of Security Onion." exit 0 else @@ -1813,6 +1817,15 @@ main() { set_minionid MINION_ROLE=$(lookup_role) echo "Found that Security Onion $INSTALLEDVERSION is currently installed." + # /etc/soversion is stamped to the target version before the upgrade fully + # completes, so a lingering resume marker means this grid is only partially + # upgraded even though the line above shows the target version. Make that explicit + # so it is not mistaken for a finished upgrade. + if [ -f "$POSTVERSION_FILE" ] && [ "$(cat "$POSTVERSION_FILE")" != "$INSTALLEDVERSION" ]; then + echo "" + echo "NOTE: A previous upgrade to $INSTALLEDVERSION did not finish. This grid is" + echo " partially upgraded and this soup run will resume and complete it." + fi echo "" check_minimum_version From bd70dd53fb8e9e3acdd68bd7c023327b57f5e270 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 15 Jul 2026 12:00:11 -0400 Subject: [PATCH 4/5] soup: add cluster-health and Fleet Server pre-flight checks Before making any changes, verify the grid is in a good state: - check_cluster_health: waits for Elasticsearch to reach at least 'yellow' (blocks only on red/unreachable, since yellow is normal), modeled on the wait in so-elasticsearch-roles-load. - check_fleet_server: confirms the Fleet Server status API returns HTTP 200, modeled on the wait_for_so-elastic-fleet state in elasticfleet/enabled.sls. Both run alongside the existing check_pillar_items (manager pillar render) and verify_es_version_compatibility, before soup modifies anything, so a failure exits cleanly with an actionable message and no partial changes. Valid on all manager roles soup runs on (eval/standalone/manager/managerhype/managersearch/ import), which all run Elasticsearch and the Fleet Server. --- salt/manager/tools/sbin/soup | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 1200fc423..37e550db2 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -327,6 +327,31 @@ check_pillar_items() { fi } +check_cluster_health() { + echo "Checking Elasticsearch cluster health." + # Block only if the cluster cannot reach at least 'yellow' status (i.e. it is red or + # unreachable); 'yellow' is normal for many Security Onion deployments. Modeled on the + # wait used in so-elasticsearch-roles-load. + if so-elasticsearch-query "_cluster/health?wait_for_status=yellow&timeout=120s" --fail > /dev/null 2>&1; then + printf "\nThe Elasticsearch cluster is healthy. We can proceed with SOUP.\n\n" + else + printf "\nThe Elasticsearch cluster is not healthy (it did not reach at least 'yellow' status). Please resolve the cluster health issue before running SOUP again.\n\n" + exit 0 + fi +} + +check_fleet_server() { + echo "Checking that Elastic Fleet Server is responding." + # Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls, + # which waits for HTTP 200 from the Fleet Server status API. + if curl -sk --fail --retry 3 --retry-delay 10 --max-time 30 "https://localhost:8220/api/status" > /dev/null 2>&1; then + printf "\nElastic Fleet Server is responding. We can proceed with SOUP.\n\n" + else + printf "\nElastic Fleet Server is not responding at https://localhost:8220/api/status. Please ensure Elastic Fleet is healthy before running SOUP again.\n\n" + exit 0 + fi +} + check_saltmaster_status() { set +e echo "Waiting on the Salt Master service to be ready." @@ -1854,6 +1879,12 @@ main() { echo "Verifying Elasticsearch version compatibility across the grid before upgrading." verify_es_version_compatibility + # Pre-flight health checks: confirm the grid is in a good state before we change + # anything. These run before any modifications, so a failure exits cleanly and the + # operator can fix the issue and re-run soup. + check_cluster_health + check_fleet_server + echo "Checking for Salt Master and Minion updates." upgrade_check_salt set -e From 186bf86e996f17f00d4af0eb7292888881f7cba3 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 15 Jul 2026 12:14:18 -0400 Subject: [PATCH 5/5] soup: require green Elasticsearch cluster before upgrading Change the pre-flight cluster-health gate to wait_for_status=green instead of yellow, so soup only proceeds when the cluster is fully green. --- salt/manager/tools/sbin/soup | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 37e550db2..ec2dcaed5 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -329,13 +329,12 @@ check_pillar_items() { check_cluster_health() { echo "Checking Elasticsearch cluster health." - # Block only if the cluster cannot reach at least 'yellow' status (i.e. it is red or - # unreachable); 'yellow' is normal for many Security Onion deployments. Modeled on the - # wait used in so-elasticsearch-roles-load. - if so-elasticsearch-query "_cluster/health?wait_for_status=yellow&timeout=120s" --fail > /dev/null 2>&1; then - printf "\nThe Elasticsearch cluster is healthy. We can proceed with SOUP.\n\n" + # Require a 'green' cluster before upgrading; anything less (yellow, red, or + # unreachable) blocks. Modeled on the wait used in so-elasticsearch-roles-load. + if so-elasticsearch-query "_cluster/health?wait_for_status=green&timeout=120s" --fail > /dev/null 2>&1; then + printf "\nThe Elasticsearch cluster is healthy (green). We can proceed with SOUP.\n\n" else - printf "\nThe Elasticsearch cluster is not healthy (it did not reach at least 'yellow' status). Please resolve the cluster health issue before running SOUP again.\n\n" + printf "\nThe Elasticsearch cluster is not green. Please resolve the cluster health issue so the cluster is green before running SOUP again.\n\n" exit 0 fi }