From ed533efb7b52bd3b6fadf38277124f3c811896eb Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 10 Jul 2026 15:53:49 -0400 Subject: [PATCH] so-salt-minion-check: tag and clarify log lines per check With two independent checks now writing to the same log, messages like "system uptime only N seconds does not meet 1800 second requirement" were ambiguous about which check they came from. Prefix every line with a [minion-restart-check] or [boot-highstate-check] tag and reword the uptime, threshold, and healthy messages to say what was evaluated and why it was skipped. Restructure the boot-highstate check from a nested if into an if/elif chain so each outcome (restart already queued, uptime too low, healthy, already running, forcing) logs its own reason instead of silently doing nothing. --- .../tools/sbin_jinja/so-salt-minion-check | 53 +++++++++++-------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/salt/common/tools/sbin_jinja/so-salt-minion-check b/salt/common/tools/sbin_jinja/so-salt-minion-check index 936ecda9a..c78846a5f 100755 --- a/salt/common/tools/sbin_jinja/so-salt-minion-check +++ b/salt/common/tools/sbin_jinja/so-salt-minion-check @@ -79,41 +79,48 @@ log "running so-salt-minion-check" RESTARTED=false +# Check 1 (minion-restart-check): if the minion has stopped applying states (the +# state-apply-test healthcheck file has gone stale), restart the salt-minion service. if [ $CURRENT_TIME -ge $((SYSTEM_START_TIME+$UPTIME_REQ)) ]; then if [ $THRESHOLD_DATE -le $CURRENT_TIME ]; then - log "salt-minion is unable to apply states" E - log "/opt/so/log/salt/healthcheck-state-apply not touched by required date: `date -d @$THRESHOLD_DATE`, last touched: `date -d @$LAST_HEALTHCHECK_STATE_APPLY`" I - log "last highstate completed at `date -d @$LAST_HIGHSTATE_END`" I - log "checking if any jobs are running" I + log "[minion-restart-check] salt-minion is unable to apply states; restarting salt-minion" E + log "[minion-restart-check] state-apply-test not touched by required date `date -d @$THRESHOLD_DATE`, last touched `date -d @$LAST_HEALTHCHECK_STATE_APPLY`" I + log "[minion-restart-check] last highstate completed at `date -d @$LAST_HIGHSTATE_END`" I + log "[minion-restart-check] checking if any jobs are running" I logCmd "salt-call --local saltutil.running" I - log "ensure salt.minion-state-apply-test is enabled" I + log "[minion-restart-check] ensure salt.minion-state-apply-test is enabled" I logCmd "salt-call state.enable salt.minion-state-apply-test" I - log "ensure highstate is enabled" I + log "[minion-restart-check] ensure highstate is enabled" I logCmd "salt-call state.enable highstate" I - log "killing all salt-minion processes" I + log "[minion-restart-check] killing all salt-minion processes" I logCmd "pkill -9 -ef /usr/bin/salt-minion" I - log "starting salt-minion service" I + log "[minion-restart-check] starting salt-minion service" I logCmd "systemctl start salt-minion" I - log "waiting for salt-minion to become ready, then applying highstate in the background (queued)" I + log "[minion-restart-check] waiting for salt-minion to become ready, then applying highstate in the background (queued)" I nohup bash -c '/usr/sbin/so-salt-minion-wait; salt-call state.highstate queue=True' >> "/opt/so/log/salt/so-salt-minion-check" 2>&1 & RESTARTED=true else - log "/opt/so/log/salt/healthcheck-state-apply last touched: `date -d @$LAST_HEALTHCHECK_STATE_APPLY` must be touched by `date -d @$THRESHOLD_DATE` to avoid salt-minion restart" I + log "[minion-restart-check] healthy: state-apply-test last touched `date -d @$LAST_HEALTHCHECK_STATE_APPLY`, must go stale past `date -d @$THRESHOLD_DATE` to trigger a salt-minion restart" I fi else - log "system uptime only $((CURRENT_TIME-SYSTEM_START_TIME)) seconds does not meet $UPTIME_REQ second requirement." I + log "[minion-restart-check] skipped: system uptime $((CURRENT_TIME-SYSTEM_START_TIME))s is below the ${UPTIME_REQ}s minimum required before a salt-minion restart" I fi -# If the host has been up long enough but no highstate has completed since this boot, -# force one. This recovers a host whose boot highstate (so-boot-highstate.service) failed -# or was skipped, even while the minion is otherwise healthy (touching state-apply-test). -# We deliberately do NOT enable highstate here: if soup has disabled it during an upgrade, -# Salt will refuse the highstate and we avoid forcing one mid-upgrade. -if ! $RESTARTED && [ $CURRENT_TIME -ge $((SYSTEM_START_TIME+HIGHSTATE_UPTIME_REQ)) ] && [ $LAST_HIGHSTATE_END -lt $SYSTEM_START_TIME ]; then - if salt-call --local saltutil.running 2>/dev/null | grep -q 'state.highstate'; then - log "no highstate has completed since boot, but one is already running; skipping" I - else - log "no highstate has completed since boot after $((CURRENT_TIME-SYSTEM_START_TIME))s uptime; applying highstate" E - nohup bash -c 'salt-call state.highstate -l info queue=True' >> "/opt/so/log/salt/so-salt-minion-check" 2>&1 & - fi +# Check 2 (boot-highstate-check): if the host has been up long enough but no highstate +# has completed since this boot, force one. This recovers a host whose boot highstate +# (so-boot-highstate.service) failed or was skipped, even while the minion is otherwise +# healthy (touching state-apply-test). We deliberately do NOT enable highstate here: if +# soup has disabled it during an upgrade, Salt will refuse the highstate and we avoid +# forcing one mid-upgrade. +if $RESTARTED; then + log "[boot-highstate-check] skipped: minion-restart-check already queued a highstate this run" I +elif [ $CURRENT_TIME -lt $((SYSTEM_START_TIME+HIGHSTATE_UPTIME_REQ)) ]; then + log "[boot-highstate-check] skipped: system uptime $((CURRENT_TIME-SYSTEM_START_TIME))s is below the ${HIGHSTATE_UPTIME_REQ}s minimum required before forcing a highstate" I +elif [ $LAST_HIGHSTATE_END -ge $SYSTEM_START_TIME ]; then + log "[boot-highstate-check] healthy: a highstate completed at `date -d @$LAST_HIGHSTATE_END`, after this boot at `date -d @$SYSTEM_START_TIME`" I +elif salt-call --local saltutil.running 2>/dev/null | grep -q 'state.highstate'; then + log "[boot-highstate-check] no highstate has completed since boot, but one is already running; skipping" I +else + log "[boot-highstate-check] no highstate has completed since boot after $((CURRENT_TIME-SYSTEM_START_TIME))s uptime; applying highstate" E + nohup bash -c 'salt-call state.highstate -l info queue=True' >> "/opt/so/log/salt/so-salt-minion-check" 2>&1 & fi