diff --git a/salt/manager/tools/sbin/so-push-drainer b/salt/manager/tools/sbin/so-push-drainer index 569f47ef2..aad62b5c7 100644 --- a/salt/manager/tools/sbin/so-push-drainer +++ b/salt/manager/tools/sbin/so-push-drainer @@ -17,6 +17,7 @@ is older than debounce_seconds, this script: * concatenates the actions lists from every ready intent * dedupes by (state or __highstate__, tgt, tgt_type) * drops actions whose target matches no accepted minion (e.g. a role with no nodes yet) + * holds everything for a later pass while salt-master has only just started * dispatches a single `salt-run state.orchestrate orch.push_batch --async` with the deduped actions list passed as pillar kwargs * deletes the contributed intent files on successful dispatch @@ -46,6 +47,12 @@ PENDING_DIR = '/opt/so/state/push_pending' LOCK_FILE = os.path.join(PENDING_DIR, '.lock') LOG_FILE = '/opt/so/log/salt/so-push-drainer.log' MASTER_CONFIG = '/etc/salt/master' +MASTER_PIDFILE = '/var/run/salt-master.pid' + +# Minions rejoin a restarted master's publisher up to ~22s after it returns (zmq reconnect interval +# plus jitter, each up to recon_default + recon_max) and a job published before then reaches nobody. +# Doubled to allow for a slow master start. +MASTER_SETTLE_SECONDS = 45 DISPATCHED_DIR = '/opt/so/state/push_dispatched' @@ -140,6 +147,14 @@ def _targeted(actions, log): return targeted +def _master_settling(now): + try: + age = now - os.path.getmtime(MASTER_PIDFILE) + except OSError: + return False + return 0 <= age < MASTER_SETTLE_SECONDS + + def _dispatch(actions, log): pillar_arg = json.dumps({'actions': actions}) cmd = [ @@ -410,6 +425,10 @@ def main(): log.debug('no ready intents (%d still in debounce window)', skipped) return 0 + if _master_settling(now): + log.info('salt-master just started; holding %d intent(s) until minions reconnect', len(ready)) + return 0 + combined_actions = [] oldest_first_touch = now all_paths = [] diff --git a/salt/manager/tools/sbin/so-push-drainer_test.py b/salt/manager/tools/sbin/so-push-drainer_test.py index a6deab5f2..3f26d1822 100644 --- a/salt/manager/tools/sbin/so-push-drainer_test.py +++ b/salt/manager/tools/sbin/so-push-drainer_test.py @@ -199,6 +199,17 @@ class TestHelpers(DrainerTestCase): drainer._unlink(self.tmpdir, self.log) self.log.exception.assert_called_once() + def test_master_settling(self): + pidfile = os.path.join(self.tmpdir, 'salt-master.pid') + now = time.time() + with patch.object(drainer, 'MASTER_PIDFILE', pidfile): + self.assertFalse(drainer._master_settling(now)) + self.write_json(self.tmpdir, 'salt-master.pid', '1100') + for age, settling in ((0, True), (drainer.MASTER_SETTLE_SECONDS - 1, True), + (drainer.MASTER_SETTLE_SECONDS, False), (-60, False)): + os.utime(pidfile, (now - age, now - age)) + self.assertIs(drainer._master_settling(now), settling, age) + class TestDispatch(DrainerTestCase): @@ -475,6 +486,7 @@ class TestMain(DrainerTestCase): ('_load_push_cfg', {'side_effect': lambda: self.cfg}), ('_check_dispatched', {}), ('_targeted', {'side_effect': lambda actions, log: actions}), + ('_master_settling', {'return_value': False}), ): patcher = patch.object(drainer, name, **kwargs) setattr(self, name, patcher.start()) @@ -528,6 +540,16 @@ class TestMain(DrainerTestCase): with patch.object(drainer.os, 'unlink', side_effect=OSError('busy')): self.assertEqual(drainer.main(), 0) + def test_holds_while_master_settling(self): + path = self.intent('pillar_soc.json') + self._master_settling.return_value = True + with patch.object(drainer, '_dispatch') as dispatch: + self.assertEqual(drainer.main(), 0) + dispatch.assert_not_called() + self._check_dispatched.assert_called_once() + self.assertTrue(os.path.exists(path)) + self.assertIn('holding 1 intent(s)', self.logged('info')) + def test_dispatch_failure_keeps_intents(self): path = self.intent('pillar_soc.json') with patch.object(drainer, '_dispatch', return_value=None):