From 65816ae7171698d9e639d6f00d9cd6877848b096 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 18 Nov 2026 10:27:34 -0500 Subject: [PATCH] FIX: hold push dispatch while salt-master has only just started A push dispatched seconds after a salt-master restart was lost: minions rejoin the new publisher on a randomized ZeroMQ reconnect delay (up to ~22s with the default recon_default/recon_max), and a job published before then reaches nobody. The orchestration's refresh_pillar step timed out with "No minions responded", the drainer logged "push failed" as an ERROR, and so-log-check failed the run. Seen on a standalone 3.5.0 install: a highstate restarted salt-master (new reactor/engine config), the drainer dispatched a soc push 8s later, and the job cache shows the refresh_pillar job matched the minion but got no return. Measured on the same host over five restarts, jobs published in the first 4-9s were lost every time. The drainer now leaves ready intents in place for a later pass while /var/run/salt-master.pid is less than 45s old. On a multi-node grid this also avoids a batch state run silently skipping the minions that have not reconnected yet, since batch only sends the job to minions that answer its ping. --- salt/manager/tools/sbin/so-push-drainer | 19 ++++++++++++++++ .../tools/sbin/so-push-drainer_test.py | 22 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/salt/manager/tools/sbin/so-push-drainer b/salt/manager/tools/sbin/so-push-drainer index 569f47ef2..aad62b5c7 100644 --- a/salt/manager/tools/sbin/so-push-drainer +++ b/salt/manager/tools/sbin/so-push-drainer @@ -17,6 +17,7 @@ is older than debounce_seconds, this script: * concatenates the actions lists from every ready intent * dedupes by (state or __highstate__, tgt, tgt_type) * drops actions whose target matches no accepted minion (e.g. a role with no nodes yet) + * holds everything for a later pass while salt-master has only just started * dispatches a single `salt-run state.orchestrate orch.push_batch --async` with the deduped actions list passed as pillar kwargs * deletes the contributed intent files on successful dispatch @@ -46,6 +47,12 @@ PENDING_DIR = '/opt/so/state/push_pending' LOCK_FILE = os.path.join(PENDING_DIR, '.lock') LOG_FILE = '/opt/so/log/salt/so-push-drainer.log' MASTER_CONFIG = '/etc/salt/master' +MASTER_PIDFILE = '/var/run/salt-master.pid' + +# Minions rejoin a restarted master's publisher up to ~22s after it returns (zmq reconnect interval +# plus jitter, each up to recon_default + recon_max) and a job published before then reaches nobody. +# Doubled to allow for a slow master start. +MASTER_SETTLE_SECONDS = 45 DISPATCHED_DIR = '/opt/so/state/push_dispatched' @@ -140,6 +147,14 @@ def _targeted(actions, log): return targeted +def _master_settling(now): + try: + age = now - os.path.getmtime(MASTER_PIDFILE) + except OSError: + return False + return 0 <= age < MASTER_SETTLE_SECONDS + + def _dispatch(actions, log): pillar_arg = json.dumps({'actions': actions}) cmd = [ @@ -410,6 +425,10 @@ def main(): log.debug('no ready intents (%d still in debounce window)', skipped) return 0 + if _master_settling(now): + log.info('salt-master just started; holding %d intent(s) until minions reconnect', len(ready)) + return 0 + combined_actions = [] oldest_first_touch = now all_paths = [] diff --git a/salt/manager/tools/sbin/so-push-drainer_test.py b/salt/manager/tools/sbin/so-push-drainer_test.py index a6deab5f2..3f26d1822 100644 --- a/salt/manager/tools/sbin/so-push-drainer_test.py +++ b/salt/manager/tools/sbin/so-push-drainer_test.py @@ -199,6 +199,17 @@ class TestHelpers(DrainerTestCase): drainer._unlink(self.tmpdir, self.log) self.log.exception.assert_called_once() + def test_master_settling(self): + pidfile = os.path.join(self.tmpdir, 'salt-master.pid') + now = time.time() + with patch.object(drainer, 'MASTER_PIDFILE', pidfile): + self.assertFalse(drainer._master_settling(now)) + self.write_json(self.tmpdir, 'salt-master.pid', '1100') + for age, settling in ((0, True), (drainer.MASTER_SETTLE_SECONDS - 1, True), + (drainer.MASTER_SETTLE_SECONDS, False), (-60, False)): + os.utime(pidfile, (now - age, now - age)) + self.assertIs(drainer._master_settling(now), settling, age) + class TestDispatch(DrainerTestCase): @@ -475,6 +486,7 @@ class TestMain(DrainerTestCase): ('_load_push_cfg', {'side_effect': lambda: self.cfg}), ('_check_dispatched', {}), ('_targeted', {'side_effect': lambda actions, log: actions}), + ('_master_settling', {'return_value': False}), ): patcher = patch.object(drainer, name, **kwargs) setattr(self, name, patcher.start()) @@ -528,6 +540,16 @@ class TestMain(DrainerTestCase): with patch.object(drainer.os, 'unlink', side_effect=OSError('busy')): self.assertEqual(drainer.main(), 0) + def test_holds_while_master_settling(self): + path = self.intent('pillar_soc.json') + self._master_settling.return_value = True + with patch.object(drainer, '_dispatch') as dispatch: + self.assertEqual(drainer.main(), 0) + dispatch.assert_not_called() + self._check_dispatched.assert_called_once() + self.assertTrue(os.path.exists(path)) + self.assertIn('holding 1 intent(s)', self.logged('info')) + def test_dispatch_failure_keeps_intents(self): path = self.intent('pillar_soc.json') with patch.object(drainer, '_dispatch', return_value=None):