FIX: hold push dispatch while salt-master has only just started

A push dispatched seconds after a salt-master restart was lost: minions
rejoin the new publisher on a randomized ZeroMQ reconnect delay (up to
~22s with the default recon_default/recon_max), and a job published
before then reaches nobody. The orchestration's refresh_pillar step timed
out with "No minions responded", the drainer logged "push failed" as an
ERROR, and so-log-check failed the run.

Seen on a standalone 3.5.0 install: a highstate restarted salt-master
(new reactor/engine config), the drainer dispatched a soc push 8s later,
and the job cache shows the refresh_pillar job matched the minion but got
no return. Measured on the same host over five restarts, jobs published
in the first 4-9s were lost every time.

The drainer now leaves ready intents in place for a later pass while
/var/run/salt-master.pid is less than 45s old. On a multi-node grid this
also avoids a batch state run silently skipping the minions that have
not reconnected yet, since batch only sends the job to minions that
answer its ping.
This commit is contained in:
Josh Patterson committed 2026-11-18 10:27:34 -05:00
1 parent 4dbea0e28d
commit 65816ae717
2 files changed
+41

No files matched your search

+19
View File
@@ -17,6 +17,7 @@ is older than debounce_seconds, this script:
* concatenates the actions lists from every ready intent
* dedupes by (state or __highstate__, tgt, tgt_type)
* drops actions whose target matches no accepted minion (e.g. a role with no nodes yet)
* holds everything for a later pass while salt-master has only just started
* dispatches a single `salt-run state.orchestrate orch.push_batch --async`
with the deduped actions list passed as pillar kwargs
* deletes the contributed intent files on successful dispatch
@@ -46,6 +47,12 @@ PENDING_DIR = '/opt/so/state/push_pending'
LOCK_FILE = os.path.join(PENDING_DIR, '.lock')
LOG_FILE = '/opt/so/log/salt/so-push-drainer.log'
MASTER_CONFIG = '/etc/salt/master'
MASTER_PIDFILE = '/var/run/salt-master.pid'
# Minions rejoin a restarted master's publisher up to ~22s after it returns (zmq reconnect interval
# plus jitter, each up to recon_default + recon_max) and a job published before then reaches nobody.
# Doubled to allow for a slow master start.
MASTER_SETTLE_SECONDS = 45
DISPATCHED_DIR = '/opt/so/state/push_dispatched'
@@ -140,6 +147,14 @@ def _targeted(actions, log):
return targeted
def _master_settling(now):
try:
age = now - os.path.getmtime(MASTER_PIDFILE)
except OSError:
return False
return 0 <= age < MASTER_SETTLE_SECONDS
def _dispatch(actions, log):
pillar_arg = json.dumps({'actions': actions})
cmd = [
@@ -410,6 +425,10 @@ def main():
log.debug('no ready intents (%d still in debounce window)', skipped)
return 0
if _master_settling(now):
log.info('salt-master just started; holding %d intent(s) until minions reconnect', len(ready))
return 0
combined_actions = []
oldest_first_touch = now
all_paths = []
@@ -199,6 +199,17 @@ class TestHelpers(DrainerTestCase):
drainer._unlink(self.tmpdir, self.log)
self.log.exception.assert_called_once()
def test_master_settling(self):
pidfile = os.path.join(self.tmpdir, 'salt-master.pid')
now = time.time()
with patch.object(drainer, 'MASTER_PIDFILE', pidfile):
self.assertFalse(drainer._master_settling(now))
self.write_json(self.tmpdir, 'salt-master.pid', '1100')
for age, settling in ((0, True), (drainer.MASTER_SETTLE_SECONDS - 1, True),
(drainer.MASTER_SETTLE_SECONDS, False), (-60, False)):
os.utime(pidfile, (now - age, now - age))
self.assertIs(drainer._master_settling(now), settling, age)
class TestDispatch(DrainerTestCase):
@@ -475,6 +486,7 @@ class TestMain(DrainerTestCase):
('_load_push_cfg', {'side_effect': lambda: self.cfg}),
('_check_dispatched', {}),
('_targeted', {'side_effect': lambda actions, log: actions}),
('_master_settling', {'return_value': False}),
):
patcher = patch.object(drainer, name, **kwargs)
setattr(self, name, patcher.start())
@@ -528,6 +540,16 @@ class TestMain(DrainerTestCase):
with patch.object(drainer.os, 'unlink', side_effect=OSError('busy')):
self.assertEqual(drainer.main(), 0)
def test_holds_while_master_settling(self):
path = self.intent('pillar_soc.json')
self._master_settling.return_value = True
with patch.object(drainer, '_dispatch') as dispatch:
self.assertEqual(drainer.main(), 0)
dispatch.assert_not_called()
self._check_dispatched.assert_called_once()
self.assertTrue(os.path.exists(path))
self.assertIn('holding 1 intent(s)', self.logged('info'))
def test_dispatch_failure_keeps_intents(self):
path = self.intent('pillar_soc.json')
with patch.object(drainer, '_dispatch', return_value=None):