diff --git a/salt/docker_clean/init.sls b/salt/docker_clean/init.sls index ee60f5591..86dc19a8f 100644 --- a/salt/docker_clean/init.sls +++ b/salt/docker_clean/init.sls @@ -9,7 +9,7 @@ prune_images: cmd.run: - name: so-docker-prune - - order: last + - order: 9000 {% else %} diff --git a/salt/salt/lasthighstate.sls b/salt/salt/lasthighstate.sls index 606bd1082..88d060483 100644 --- a/salt/salt/lasthighstate.sls +++ b/salt/salt/lasthighstate.sls @@ -1,4 +1,4 @@ lasthighstate: file.touch: - name: /opt/so/log/salt/lasthighstate - - order: last \ No newline at end of file + - order: 9001 diff --git a/salt/salt/master.sls b/salt/salt/master.sls index b33d3631d..3833232fd 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -71,7 +71,6 @@ reactor_pushstate_config: - source: salt://salt/files/reactor_pushstate.conf - watch_in: - service: salt_master_service - - order: last {% else %} reactor_pushstate_config: file.absent: @@ -95,7 +94,7 @@ salt_master_service: - file: checkmine_engine - file: pillarWatch_engine - file: engines_config - - order: last + - order: 9002 {% else %} diff --git a/salt/salt/minion/init.sls b/salt/salt/minion/init.sls index eb7018aed..42f98de2a 100644 --- a/salt/salt/minion/init.sls +++ b/salt/salt/minion/init.sls @@ -88,13 +88,17 @@ enable_startup_states: {% endif %} -# this has to be outside the if statement above since there are _in calls to this state +# this has to be outside the if statement above since there are _in calls to this state. +# uses watch (not listen) so the restart fires in-state and its result lands on this state's +# running entry; that is what lets wait_for_salt_minion_ready below detect any restart +# uniformly via onchanges, regardless of whether the trigger came from these files or from +# external watch_in's (e.g. beacons, master/pyinotify). salt_minion_service: service.running: - name: salt-minion - enable: True - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" - - listen: + - watch: - file: mine_functions {% if INSTALLEDSALTVERSION|string == SALTVERSION|string %} - file: set_log_levels @@ -103,3 +107,32 @@ salt_minion_service: - file: signing_policy {% endif %} - order: last + +# block until the just-restarted salt-minion is back and can execute modules locally, so +# follow-on jobs and the next highstate iteration do not race the restart. onchanges + +# require on salt_minion_service catches every restart trigger uniformly because watch +# mod_watch results replace the service state's running entry. initial sleep gives the +# systemctl restart (--no-block by default for salt-minion on >=3006.15) time to begin +# tearing down the old process before we probe for readiness. +wait_for_salt_minion_ready: + cmd.run: + - name: | + sleep 3 + timeout=120 + elapsed=3 + while [ $elapsed -lt $timeout ]; do + if systemctl is-active --quiet salt-minion \ + && salt-call --local --timeout=5 --out=quiet test.ping >/dev/null 2>&1; then + echo "salt-minion ready after ${elapsed}s" + exit 0 + fi + sleep 1 + elapsed=$((elapsed+1)) + done + echo "salt-minion did not become ready within ${timeout}s" >&2 + exit 1 + - onchanges: + - service: salt_minion_service + - require: + - service: salt_minion_service + - order: last