CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2026-07-13 04:22:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

so-salt-minion-check: tag and clarify log lines per check

Browse Source 
With two independent checks now writing to the same log, messages like
"system uptime only N seconds does not meet 1800 second requirement" were
ambiguous about which check they came from. Prefix every line with a
[minion-restart-check] or [boot-highstate-check] tag and reword the uptime,
threshold, and healthy messages to say what was evaluated and why it was
skipped.

Restructure the boot-highstate check from a nested if into an if/elif chain so
each outcome (restart already queued, uptime too low, healthy, already running,
forcing) logs its own reason instead of silently doing nothing.
This commit is contained in:
Josh Patterson
2026-07-10 15:53:49 -04:00
parent 5af6c56996
commit ed533efb7b
1 changed files with 30 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
salt/common/tools/sbin_jinja/so-salt-minion-check
+30 -23
View File
@@ -79,41 +79,48 @@ log "running so-salt-minion-check"
RESTARTED=false
# Check 1 (minion-restart-check): if the minion has stopped applying states (the
# state-apply-test healthcheck file has gone stale), restart the salt-minion service.
if [ $CURRENT_TIME -ge $((SYSTEM_START_TIME+$UPTIME_REQ)) ]; then
if [ $THRESHOLD_DATE -le $CURRENT_TIME ]; then
log "salt-minion is unable to apply states" E
log "/opt/so/log/salt/healthcheck-state-apply not touched by required date: `date -d @$THRESHOLD_DATE`, last touched: `date -d @$LAST_HEALTHCHECK_STATE_APPLY`" I
log "last highstate completed at `date -d @$LAST_HIGHSTATE_END`" I
log "checking if any jobs are running" I
log "[minion-restart-check] salt-minion is unable to apply states; restarting salt-minion" E
log "[minion-restart-check] state-apply-test not touched by required date `date -d @$THRESHOLD_DATE`, last touched `date -d @$LAST_HEALTHCHECK_STATE_APPLY`" I
log "[minion-restart-check] last highstate completed at `date -d @$LAST_HIGHSTATE_END`" I
log "[minion-restart-check] checking if any jobs are running" I
logCmd "salt-call --local saltutil.running" I
log "ensure salt.minion-state-apply-test is enabled" I
log "[minion-restart-check] ensure salt.minion-state-apply-test is enabled" I
logCmd "salt-call state.enable salt.minion-state-apply-test" I
log "ensure highstate is enabled" I
log "[minion-restart-check] ensure highstate is enabled" I
logCmd "salt-call state.enable highstate" I
log "killing all salt-minion processes" I
log "[minion-restart-check] killing all salt-minion processes" I
logCmd "pkill -9 -ef /usr/bin/salt-minion" I
log "starting salt-minion service" I
log "[minion-restart-check] starting salt-minion service" I
logCmd "systemctl start salt-minion" I
log "waiting for salt-minion to become ready, then applying highstate in the background (queued)" I
log "[minion-restart-check] waiting for salt-minion to become ready, then applying highstate in the background (queued)" I
nohup bash -c '/usr/sbin/so-salt-minion-wait; salt-call state.highstate queue=True' >> "/opt/so/log/salt/so-salt-minion-check" 2>&1 &
RESTARTED=true
else
log "/opt/so/log/salt/healthcheck-state-apply last touched: `date -d @$LAST_HEALTHCHECK_STATE_APPLY` must be touched by `date -d @$THRESHOLD_DATE` to avoid salt-minion restart" I
log "[minion-restart-check] healthy: state-apply-test last touched `date -d @$LAST_HEALTHCHECK_STATE_APPLY`, must go stale past `date -d @$THRESHOLD_DATE` to trigger a salt-minion restart" I
fi
else
log "system uptime only $((CURRENT_TIME-SYSTEM_START_TIME)) seconds does not meet $UPTIME_REQ second requirement." I
log "[minion-restart-check] skipped: system uptime $((CURRENT_TIME-SYSTEM_START_TIME))s is below the ${UPTIME_REQ}s minimum required before a salt-minion restart" I
fi
# If the host has been up long enough but no highstate has completed since this boot,
# force one. This recovers a host whose boot highstate (so-boot-highstate.service) failed
# or was skipped, even while the minion is otherwise healthy (touching state-apply-test).
# We deliberately do NOT enable highstate here: if soup has disabled it during an upgrade,
# Salt will refuse the highstate and we avoid forcing one mid-upgrade.
if ! $RESTARTED && [ $CURRENT_TIME -ge $((SYSTEM_START_TIME+HIGHSTATE_UPTIME_REQ)) ] && [ $LAST_HIGHSTATE_END -lt $SYSTEM_START_TIME ]; then
if salt-call --local saltutil.running 2>/dev/null | grep -q 'state.highstate'; then
log "no highstate has completed since boot, but one is already running; skipping" I
else
log "no highstate has completed since boot after $((CURRENT_TIME-SYSTEM_START_TIME))s uptime; applying highstate" E
nohup bash -c 'salt-call state.highstate -l info queue=True' >> "/opt/so/log/salt/so-salt-minion-check" 2>&1 &
fi
# Check 2 (boot-highstate-check): if the host has been up long enough but no highstate
# has completed since this boot, force one. This recovers a host whose boot highstate
# (so-boot-highstate.service) failed or was skipped, even while the minion is otherwise
# healthy (touching state-apply-test). We deliberately do NOT enable highstate here: if
# soup has disabled it during an upgrade, Salt will refuse the highstate and we avoid
# forcing one mid-upgrade.
if $RESTARTED; then
log "[boot-highstate-check] skipped: minion-restart-check already queued a highstate this run" I
elif [ $CURRENT_TIME -lt $((SYSTEM_START_TIME+HIGHSTATE_UPTIME_REQ)) ]; then
log "[boot-highstate-check] skipped: system uptime $((CURRENT_TIME-SYSTEM_START_TIME))s is below the ${HIGHSTATE_UPTIME_REQ}s minimum required before forcing a highstate" I
elif [ $LAST_HIGHSTATE_END -ge $SYSTEM_START_TIME ]; then
log "[boot-highstate-check] healthy: a highstate completed at `date -d @$LAST_HIGHSTATE_END`, after this boot at `date -d @$SYSTEM_START_TIME`" I
elif salt-call --local saltutil.running 2>/dev/null | grep -q 'state.highstate'; then
log "[boot-highstate-check] no highstate has completed since boot, but one is already running; skipping" I
else
log "[boot-highstate-check] no highstate has completed since boot after $((CURRENT_TIME-SYSTEM_START_TIME))s uptime; applying highstate" E
nohup bash -c 'salt-call state.highstate -l info queue=True' >> "/opt/so/log/salt/so-salt-minion-check" 2>&1 &
fi