From 5af6c56996ac5287e892ac9780dd972ee9558355 Mon Sep 17 00:00:00 2001
From: Josh Patterson <josh.patterson@securityonionsolutions.com>
Date: Fri, 10 Jul 2026 14:54:20 -0400
Subject: [PATCH] so-salt-minion-check: force highstate if none has completed
 since boot

Add a second, independent trigger to the every-5-minute health check: if the
host has been up >= 15 minutes (HIGHSTATE_UPTIME_REQ) and no highstate has
completed since this boot (lasthighstate mtime older than boot time), run
salt-call state.highstate. This recovers a host whose boot highstate
(so-boot-highstate.service) failed or was skipped, even while the minion is
otherwise healthy and touching state-apply-test.

The new path deliberately does not enable highstate, so a soup-disabled
highstate is respected and never forced mid-upgrade. A saltutil.running guard
plus queue=True prevents stacking across successive cron runs, and a RESTARTED
flag suppresses the new block when the existing minion-restart path already
queued a highstate.
---
 .../tools/sbin_jinja/so-salt-minion-check      | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/salt/common/tools/sbin_jinja/so-salt-minion-check b/salt/common/tools/sbin_jinja/so-salt-minion-check
index 281efbcb7..936ecda9a 100755
--- a/salt/common/tools/sbin_jinja/so-salt-minion-check
+++ b/salt/common/tools/sbin_jinja/so-salt-minion-check
@@ -18,6 +18,7 @@
 
 QUIET=false
 UPTIME_REQ=1800 #in seconds, how long the box has to be up before considering restarting salt-minion due to /opt/so/log/salt/state-apply-test not being touched
+HIGHSTATE_UPTIME_REQ=900 #in seconds; if the box has been up this long and no highstate has completed since boot, force one
 CURRENT_TIME=$(date +%s)
 SYSTEM_START_TIME=$(date -d "$(</proc/uptime awk '{print $1}') seconds ago" +%s)
 LAST_HIGHSTATE_END=$([ -e "/opt/so/log/salt/lasthighstate" ] && date -r /opt/so/log/salt/lasthighstate +%s || echo 0)
@@ -76,6 +77,8 @@ done
 
 log "running so-salt-minion-check"
 
+RESTARTED=false
+
 if [ $CURRENT_TIME -ge $((SYSTEM_START_TIME+$UPTIME_REQ))  ]; then
   if [ $THRESHOLD_DATE -le $CURRENT_TIME ]; then
     log "salt-minion is unable to apply states" E
@@ -93,9 +96,24 @@ if [ $CURRENT_TIME -ge $((SYSTEM_START_TIME+$UPTIME_REQ))  ]; then
     logCmd "systemctl start salt-minion" I
     log "waiting for salt-minion to become ready, then applying highstate in the background (queued)" I
     nohup bash -c '/usr/sbin/so-salt-minion-wait; salt-call state.highstate queue=True' >> "/opt/so/log/salt/so-salt-minion-check" 2>&1 &
+    RESTARTED=true
   else
     log "/opt/so/log/salt/healthcheck-state-apply last touched: `date -d @$LAST_HEALTHCHECK_STATE_APPLY` must be touched by `date -d @$THRESHOLD_DATE` to avoid salt-minion restart" I
   fi
 else
   log "system uptime only $((CURRENT_TIME-SYSTEM_START_TIME)) seconds does not meet $UPTIME_REQ second requirement." I
 fi
+
+# If the host has been up long enough but no highstate has completed since this boot,
+# force one. This recovers a host whose boot highstate (so-boot-highstate.service) failed
+# or was skipped, even while the minion is otherwise healthy (touching state-apply-test).
+# We deliberately do NOT enable highstate here: if soup has disabled it during an upgrade,
+# Salt will refuse the highstate and we avoid forcing one mid-upgrade.
+if ! $RESTARTED && [ $CURRENT_TIME -ge $((SYSTEM_START_TIME+HIGHSTATE_UPTIME_REQ)) ] && [ $LAST_HIGHSTATE_END -lt $SYSTEM_START_TIME ]; then
+  if salt-call --local saltutil.running 2>/dev/null | grep -q 'state.highstate'; then
+    log "no highstate has completed since boot, but one is already running; skipping" I
+  else
+    log "no highstate has completed since boot after $((CURRENT_TIME-SYSTEM_START_TIME))s uptime; applying highstate" E
+    nohup bash -c 'salt-call state.highstate -l info queue=True' >> "/opt/so/log/salt/so-salt-minion-check" 2>&1 &
+  fi
+fi