diff --git a/salt/common/tools/sbin_jinja/so-salt-minion-check b/salt/common/tools/sbin_jinja/so-salt-minion-check index c78846a5f..353448251 100755 --- a/salt/common/tools/sbin_jinja/so-salt-minion-check +++ b/salt/common/tools/sbin_jinja/so-salt-minion-check @@ -25,7 +25,7 @@ LAST_HIGHSTATE_END=$([ -e "/opt/so/log/salt/lasthighstate" ] && date -r /opt/so/ LAST_HEALTHCHECK_STATE_APPLY=$([ -e "/opt/so/log/salt/state-apply-test" ] && date -r /opt/so/log/salt/state-apply-test +%s || echo 0) # SETTING THRESHOLD TO ANYTHING UNDER 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default # THRESHOLD is derived from the salt schedule highstate interval + 1 hour, so the minion-check grace period tracks the schedule automatically. -THRESHOLD=$(( ({{ SCHEDULEMERGED.highstate_interval_hours }} + 1) * 3600 )) #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted +THRESHOLD=$(( ({{ SCHEDULEMERGED.highstate_interval_minutes }} + 60) * 60 )) #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted THRESHOLD_DATE=$((LAST_HEALTHCHECK_STATE_APPLY+THRESHOLD)) logCmd() { diff --git a/salt/salt/defaults.yaml b/salt/salt/defaults.yaml index 12d0373d5..140c61fcb 100644 --- a/salt/salt/defaults.yaml +++ b/salt/salt/defaults.yaml @@ -6,4 +6,4 @@ salt: batch: '25%' batch_wait: 15 schedule: - highstate_interval_hours: 2 + highstate_interval_minutes: 120 diff --git a/salt/salt/highstate_schedule.sls b/salt/salt/highstate_schedule.sls index ca2797387..5f5ff2fa9 100644 --- a/salt/salt/highstate_schedule.sls +++ b/salt/salt/highstate_schedule.sls @@ -1,11 +1,15 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'salt/schedule.map.jinja' import SCHEDULEMERGED %} +{# splay a quarter of the interval, clamped to [5 min, 30 min], so short intervals + don't get jitter larger than the interval itself #} +{% set SPLAY = [[(SCHEDULEMERGED.highstate_interval_minutes * 60 // 4) | int, 300] | max, 1800] | min %} + highstate_schedule: schedule.present: - function: state.highstate - - hours: {{ SCHEDULEMERGED.highstate_interval_hours }} + - minutes: {{ SCHEDULEMERGED.highstate_interval_minutes }} - maxrunning: 1 {% if not GLOBALS.is_manager %} - - splay: 1800 + - splay: {{ SPLAY }} {% endif %} diff --git a/salt/salt/soc_salt.yaml b/salt/salt/soc_salt.yaml index 0aa0aaf89..f3f2ff93a 100644 --- a/salt/salt/soc_salt.yaml +++ b/salt/salt/soc_salt.yaml @@ -31,9 +31,11 @@ salt: global: True advanced: True schedule: - highstate_interval_hours: - description: How often every minion in the grid runs a scheduled state.highstate, in hours. Lower values keep minions closer in sync at the cost of more load; higher values reduce load but increase worst-case latency for non-pushed changes. The salt-minion health check restarts a minion if its last highstate is older than this value plus one hour. + highstate_interval_minutes: + description: How often every minion in the grid runs a scheduled state.highstate, in minutes. Minimum 15 minutes. Lower values keep minions closer in sync at the cost of more load; higher values reduce load but increase worst-case latency for non-pushed changes. If Auto Apply is disabled, set this to the 15-minute minimum so changes are still picked up promptly. The salt-minion health check restarts a minion if its last state apply is older than this value plus one hour. forcedType: int helpLink: push global: True advanced: True + regex: '^(1[5-9]|[2-9][0-9]|[1-9][0-9]{2,4})$' + regexFailureMessage: The value must be an integer of at least 15 minutes (maximum 99999).