CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2026-07-13 04:22:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

so-salt-minion-check: force highstate if none has completed since boot

Browse Source 
Add a second, independent trigger to the every-5-minute health check: if the
host has been up >= 15 minutes (HIGHSTATE_UPTIME_REQ) and no highstate has
completed since this boot (lasthighstate mtime older than boot time), run
salt-call state.highstate. This recovers a host whose boot highstate
(so-boot-highstate.service) failed or was skipped, even while the minion is
otherwise healthy and touching state-apply-test.

The new path deliberately does not enable highstate, so a soup-disabled
highstate is respected and never forced mid-upgrade. A saltutil.running guard
plus queue=True prevents stacking across successive cron runs, and a RESTARTED
flag suppresses the new block when the existing minion-restart path already
queued a highstate.
This commit is contained in:
Josh Patterson
2026-07-10 14:54:20 -04:00
parent 89e6a746c8
commit 5af6c56996
1 changed files with 18 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
salt/common/tools/sbin_jinja/so-salt-minion-check
+18
View File
@@ -18,6 +18,7 @@
QUIET=false
UPTIME_REQ=1800 #in seconds, how long the box has to be up before considering restarting salt-minion due to /opt/so/log/salt/state-apply-test not being touched
HIGHSTATE_UPTIME_REQ=900 #in seconds; if the box has been up this long and no highstate has completed since boot, force one
CURRENT_TIME=$(date +%s)
SYSTEM_START_TIME=$(date -d "$(</proc/uptime awk '{print $1}') seconds ago" +%s)
LAST_HIGHSTATE_END=$([ -e "/opt/so/log/salt/lasthighstate" ] && date -r /opt/so/log/salt/lasthighstate +%s || echo 0)
@@ -76,6 +77,8 @@ done
log "running so-salt-minion-check"
RESTARTED=false
if [ $CURRENT_TIME -ge $((SYSTEM_START_TIME+$UPTIME_REQ)) ]; then
if [ $THRESHOLD_DATE -le $CURRENT_TIME ]; then
log "salt-minion is unable to apply states" E
@@ -93,9 +96,24 @@ if [ $CURRENT_TIME -ge $((SYSTEM_START_TIME+$UPTIME_REQ)) ]; then
logCmd "systemctl start salt-minion" I
log "waiting for salt-minion to become ready, then applying highstate in the background (queued)" I
nohup bash -c '/usr/sbin/so-salt-minion-wait; salt-call state.highstate queue=True' >> "/opt/so/log/salt/so-salt-minion-check" 2>&1 &
RESTARTED=true
else
log "/opt/so/log/salt/healthcheck-state-apply last touched: `date -d @$LAST_HEALTHCHECK_STATE_APPLY` must be touched by `date -d @$THRESHOLD_DATE` to avoid salt-minion restart" I
fi
else
log "system uptime only $((CURRENT_TIME-SYSTEM_START_TIME)) seconds does not meet $UPTIME_REQ second requirement." I
fi
# If the host has been up long enough but no highstate has completed since this boot,
# force one. This recovers a host whose boot highstate (so-boot-highstate.service) failed
# or was skipped, even while the minion is otherwise healthy (touching state-apply-test).
# We deliberately do NOT enable highstate here: if soup has disabled it during an upgrade,
# Salt will refuse the highstate and we avoid forcing one mid-upgrade.
if ! $RESTARTED && [ $CURRENT_TIME -ge $((SYSTEM_START_TIME+HIGHSTATE_UPTIME_REQ)) ] && [ $LAST_HIGHSTATE_END -lt $SYSTEM_START_TIME ]; then
if salt-call --local saltutil.running 2>/dev/null | grep -q 'state.highstate'; then
log "no highstate has completed since boot, but one is already running; skipping" I
else
log "no highstate has completed since boot after $((CURRENT_TIME-SYSTEM_START_TIME))s uptime; applying highstate" E
nohup bash -c 'salt-call state.highstate -l info queue=True' >> "/opt/so/log/salt/so-salt-minion-check" 2>&1 &
fi
fi