wait for salt-minion service to be ready before finishing state run

salt/docker_clean/init.sls

@@ -9,7 +9,7 @@
 prune_images:
   cmd.run:
     - name: so-docker-prune
-    - order: last
+    - order: 9000
 
 {% else %}
 

salt/salt/lasthighstate.sls

@@ -1,4 +1,4 @@
 lasthighstate:
   file.touch:
     - name: /opt/so/log/salt/lasthighstate
-    - order: last
+    - order: 9001

salt/salt/master.sls

@@ -71,7 +71,6 @@ reactor_pushstate_config:
     - source: salt://salt/files/reactor_pushstate.conf
     - watch_in:
       - service: salt_master_service
-    - order: last
 {% else %}
 reactor_pushstate_config:
   file.absent:
@@ -95,7 +94,7 @@ salt_master_service:
       - file: checkmine_engine
       - file: pillarWatch_engine
       - file: engines_config
-    - order: last
+    - order: 9002
 
 {% else %}
 

salt/salt/minion/init.sls

@@ -88,13 +88,17 @@ enable_startup_states:
 
 {% endif %}
 
-# this has to be outside the if statement above since there are <requisite>_in calls to this state
+# this has to be outside the if statement above since there are <requisite>_in calls to this state.
+# uses watch (not listen) so the restart fires in-state and its result lands on this state's
+# running entry; that is what lets wait_for_salt_minion_ready below detect any restart
+# uniformly via onchanges, regardless of whether the trigger came from these files or from
+# external watch_in's (e.g. beacons, master/pyinotify).
 salt_minion_service:
   service.running:
     - name: salt-minion
     - enable: True
     - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}"
-    - listen:
+    - watch:
       - file: mine_functions
 {% if INSTALLEDSALTVERSION|string == SALTVERSION|string %}
       - file: set_log_levels
@@ -103,3 +107,32 @@ salt_minion_service:
       - file: signing_policy
 {% endif %}
     - order: last
+
+# block until the just-restarted salt-minion is back and can execute modules locally, so
+# follow-on jobs and the next highstate iteration do not race the restart. onchanges +
+# require on salt_minion_service catches every restart trigger uniformly because watch
+# mod_watch results replace the service state's running entry. initial sleep gives the
+# systemctl restart (--no-block by default for salt-minion on >=3006.15) time to begin
+# tearing down the old process before we probe for readiness.
+wait_for_salt_minion_ready:
+  cmd.run:
+    - name: |
+        sleep 3
+        timeout=120
+        elapsed=3
+        while [ $elapsed -lt $timeout ]; do
+          if systemctl is-active --quiet salt-minion \
+             && salt-call --local --timeout=5 --out=quiet test.ping >/dev/null 2>&1; then
+            echo "salt-minion ready after ${elapsed}s"
+            exit 0
+          fi
+          sleep 1
+          elapsed=$((elapsed+1))
+        done
+        echo "salt-minion did not become ready within ${timeout}s" >&2
+        exit 1
+    - onchanges:
+      - service: salt_minion_service
+    - require:
+      - service: salt_minion_service
+    - order: last