From cad18f5bfce03ae543143d01859f37e98313af6e Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 5 Oct 2026 17:48:47 -0400 Subject: [PATCH 1/6] FIX: report push result as unknown when the orchestration raises A queued push is tracked by an orchestration that waits on the minion's state run. If salt-master restarts while it waits, the orchestration step raises AuthenticationError and never collects the minion's return, but the minion still runs the queued state. so-push-drainer logged these as "push failed ... change will be applied at the next scheduled highstate", which was wrong on both counts. Seen on a fresh 3.4.0 standalone: hydra.enabled and telegraf.output were saved just after SOC started a grid highstate. Both pushes queued behind it. That highstate was the first with a license granting the vrt feature, so salt.master wrote vrt_engine.conf and reactor_hypervisor.conf and restarted salt-master as its last step. Both orchestrations raised, the minion then ran both state runs with 0 failed states, and the drainer logged two ERRORs. When every failed step of the orchestration raised (salt's "An exception occurred in this state:" comment, with no minion returns), log a WARNING that the result is unknown and the state run may still have completed. Conflicts, failed states, render errors, and results mixing a raised step with a real failure still log ERROR. --- salt/manager/tools/sbin/so-push-drainer | 27 ++++++++++++++++++- .../tools/sbin/so-push-drainer_test.py | 26 +++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/so-push-drainer b/salt/manager/tools/sbin/so-push-drainer index a09d1cb43..698e65e35 100644 --- a/salt/manager/tools/sbin/so-push-drainer +++ b/salt/manager/tools/sbin/so-push-drainer @@ -53,6 +53,9 @@ RESULT_MAX_AGE = 7200 RESULT_CHECKS_PER_PASS = 5 TEXT_LIMIT = 500 +# Lead-in salt puts on the comment of an orchestration step that raised. +STEP_RAISED = 'An exception occurred in this state:' + # salt-run --async reports the jid only in a log line (stderr by default). JID_RE = re.compile(r'salt/run/(\d{20})') @@ -242,6 +245,24 @@ def _orch_failures(ret): return failures +def _failed_steps(ret): + steps = [] + for job in ret.values() if isinstance(ret, dict) else []: + job_ret = job.get('return') if isinstance(job, dict) else None + data = job_ret.get('data') if isinstance(job_ret, dict) else None + for group in data.values() if isinstance(data, dict) else []: + if isinstance(group, dict): + steps.extend(step for step in group.values() if isinstance(step, dict) and step.get('result') is False) + return steps + + +def _result_unknown(ret): + # A step that raised (e.g. salt-master restarted while it waited on a queued state run) + # never collected the minion's return, so the state run may still have completed. + steps = _failed_steps(ret) + return bool(steps) and all(str(step.get('comment', '')).startswith(STEP_RAISED) for step in steps) + + def _recheck_delay(age): return min(RESULT_RECHECK_MAX, max(RESULT_CHECK_DELAY, age / 4)) @@ -283,7 +304,11 @@ def _report_result(record, age, log): return True return False failures = _orch_failures(ret) - if failures: + if failures and _result_unknown(ret): + log.warning('push result unknown jid=%s paths=%s; the orchestration lost track of the state run, which may ' + 'still have completed (if not, the change will be applied at the next scheduled highstate): %s', + jid, paths, ' | '.join(failures)) + elif failures: log.error('push failed jid=%s paths=%s; change will be applied at the next scheduled highstate: %s', jid, paths, ' | '.join(failures)) else: diff --git a/salt/manager/tools/sbin/so-push-drainer_test.py b/salt/manager/tools/sbin/so-push-drainer_test.py index 002011810..012d329f4 100644 --- a/salt/manager/tools/sbin/so-push-drainer_test.py +++ b/salt/manager/tools/sbin/so-push-drainer_test.py @@ -76,6 +76,17 @@ STATE_FAIL_RET = _orch_ret({ }, }, success=False) +RAISED = ('An exception occurred in this state: Traceback (most recent call last):\n' + ' File "salt/client/__init__.py", line 1934, in pub\n' + ' raise AuthenticationError(err_msg)\n' + 'salt.exceptions.AuthenticationError: Authentication error occurred.\n') + +RAISED_RET = _orch_ret(dict(REFRESH_STEP, **{ + 'salt_|-apply_hydra_1_|-apply_hydra_1_|-state': { + '__id__': 'apply_hydra_1', 'result': False, 'changes': {}, 'comment': RAISED, + }, +}), success=False) + SUCCESS_RET = _orch_ret(dict(REFRESH_STEP, **{ 'salt_|-apply_telegraf_1_|-apply_telegraf_1_|-state': { '__id__': 'apply_telegraf_1', 'result': True, @@ -294,6 +305,15 @@ class TestResults(DrainerTestCase): ret = {MASTER: {'return': 'Exception occurred', 'success': False}} self.assertEqual(drainer._orch_failures(ret), ['orchestration reported failure: Exception occurred']) + def test_result_unknown(self): + self.assertTrue(drainer._result_unknown(RAISED_RET)) + for ret in (CONFLICT_RET, STATE_FAIL_RET, SUCCESS_RET, ['No minions matched'], {MASTER: 'odd'}, + {MASTER: {'return': {'data': {MASTER: ['Rendering SLS failed']}}, 'success': False}}): + self.assertFalse(drainer._result_unknown(ret), ret) + mixed = _orch_ret(dict(RAISED_RET[MASTER]['return']['data'][MASTER], + **CONFLICT_RET[MASTER]['return']['data'][MASTER]), success=False) + self.assertFalse(drainer._result_unknown(mixed)) + def record(self, jid, age, now): return self.write_json(self.dispatched, jid + '.json', { 'jid': jid, 'dispatched_at': now - age, 'actions': [], 'paths': ['audit:' + jid], @@ -306,6 +326,7 @@ class TestResults(DrainerTestCase): '2_ok': SUCCESS_RET, '3_pending': {}, '4_expired': None, + '6_unknown': RAISED_RET, } young = self.record('0_young', 5, now) paths = {jid: self.record(jid, 60, now) for jid in results} @@ -317,13 +338,16 @@ class TestResults(DrainerTestCase): self.assertTrue(os.path.exists(young)) with open(paths['3_pending']) as f: self.assertEqual(json.load(f)['checked_at'], now) - for jid in ('1_failed', '2_ok', '4_expired'): + for jid in ('1_failed', '2_ok', '4_expired', '6_unknown'): self.assertFalse(os.path.exists(paths[jid]), jid) self.assertFalse(os.path.exists(bad)) self.assertIn('push failed jid=1_failed', self.logged('error')) self.assertIn('is running as PID 372218', self.logged('error')) self.assertIn('push succeeded jid=2_ok', self.logged('info')) self.assertIn('no result for jid=4_expired', self.logged('warning')) + self.assertIn('push result unknown jid=6_unknown', self.logged('warning')) + self.assertIn('AuthenticationError: Authentication error occurred.', self.logged('warning')) + self.assertNotIn('6_unknown', self.logged('error')) def test_check_dispatched_survives_bad_result(self): now = time.time() From 22a4acbb5af82acf35b50c0cd9ca8d9bbd7c382c Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 10 Nov 2026 15:21:12 -0500 Subject: [PATCH 2/6] FIX: skip push actions whose target matches no minions so-push-drainer dispatched every queued action, even when its target matched no accepted minion. The orchestration's saltutil.refresh_pillar step then failed with "No minions responded", and the drainer logged a push failure that so-log-check reports as an ERROR. Seen on a fresh distributed manager: the first drain after setup sent strelka and suricata rule pushes targeting sensor roles before any sensor had joined the grid. Nothing had failed; no node held the role. Before dispatch, resolve each action's target with the master's CkMinions (accepted keys and cached grains) and drop the ones that match nothing, logging each at INFO. If none remain, clear the intents without dispatching. Uncached minions still count as a match, and a lookup error keeps the action, so pushes are never dropped when the answer is uncertain. --- salt/manager/tools/sbin/so-push-drainer | 43 +++++++++-- .../tools/sbin/so-push-drainer_test.py | 72 ++++++++++++++++++- 2 files changed, 107 insertions(+), 8 deletions(-) diff --git a/salt/manager/tools/sbin/so-push-drainer b/salt/manager/tools/sbin/so-push-drainer index 698e65e35..99461a029 100644 --- a/salt/manager/tools/sbin/so-push-drainer +++ b/salt/manager/tools/sbin/so-push-drainer @@ -16,6 +16,7 @@ For each intent file under /opt/so/state/push_pending/*.json whose last_touch is older than debounce_seconds, this script: * concatenates the actions lists from every ready intent * dedupes by (state or __highstate__, tgt, tgt_type) + * drops actions whose target matches no accepted minion (e.g. a role with no nodes yet) * dispatches a single `salt-run state.orchestrate orch.push_batch --async` with the deduped actions list passed as pillar kwargs * deletes the contributed intent files on successful dispatch @@ -38,10 +39,13 @@ import sys import time import salt.client +import salt.config +import salt.utils.minions PENDING_DIR = '/opt/so/state/push_pending' LOCK_FILE = os.path.join(PENDING_DIR, '.lock') LOG_FILE = '/opt/so/log/salt/so-push-drainer.log' +MASTER_CONFIG = '/etc/salt/master' DISPATCHED_DIR = '/opt/so/state/push_dispatched' @@ -113,6 +117,29 @@ def _dedupe_actions(actions): return deduped +def _targeted(actions, log): + """Drops actions whose target matches no accepted minion (e.g. a role with no nodes yet), which + the orchestration would otherwise report as failed with 'No minions responded'.""" + try: + ckminions = salt.utils.minions.CkMinions(salt.config.master_config(MASTER_CONFIG)) + except Exception: + log.exception('cannot load master config; dispatching without checking targets') + return actions + targeted = [] + for action in actions: + try: + matched = ckminions.check_minions(action['tgt'], action.get('tgt_type', 'compound')).get('minions') + except Exception: + log.exception('cannot resolve tgt=%s; dispatching it anyway', action.get('tgt')) + matched = True + if matched: + targeted.append(action) + else: + log.info('skipping %s: no minions match tgt=%s', + 'highstate' if action.get('highstate') else action.get('state'), action.get('tgt')) + return targeted + + def _dispatch(actions, log): pillar_arg = json.dumps({'actions': actions}) cmd = [ @@ -407,12 +434,16 @@ def main(): log.info('action: %s tgt=%s', 'highstate' if action.get('highstate') else action.get('state'), action.get('tgt')) - jid = _dispatch(deduped, log) - if jid is None: - log.warning('dispatch failed; leaving intent files in place for retry') - return 1 - if jid: - _record_dispatch(jid, deduped, all_paths[:20], log) + targeted = _targeted(deduped, log) + if targeted: + jid = _dispatch(targeted, log) + if jid is None: + log.warning('dispatch failed; leaving intent files in place for retry') + return 1 + if jid: + _record_dispatch(jid, targeted, all_paths[:20], log) + else: + log.info('no action matches a minion; nothing to dispatch') for path, _ in ready: try: diff --git a/salt/manager/tools/sbin/so-push-drainer_test.py b/salt/manager/tools/sbin/so-push-drainer_test.py index 012d329f4..10b825041 100644 --- a/salt/manager/tools/sbin/so-push-drainer_test.py +++ b/salt/manager/tools/sbin/so-push-drainer_test.py @@ -22,10 +22,11 @@ _loader = SourceFileLoader('so_push_drainer', SCRIPT) _spec = importlib.util.spec_from_loader('so_push_drainer', _loader) drainer = importlib.util.module_from_spec(_spec) -# salt is not installed where these tests run; the drainer only needs salt.client.Caller. +# salt is not installed where these tests run; the drainer only needs salt.client.Caller and CkMinions. # Mocked only while the drainer loads: run from the repo root, 'salt' is this repo's salt/ directory. _salt = MagicMock() -with patch.dict(sys.modules, {'salt': _salt, 'salt.client': _salt.client}): +with patch.dict(sys.modules, {'salt': _salt, 'salt.client': _salt.client, 'salt.config': _salt.config, + 'salt.utils': _salt.utils, 'salt.utils.minions': _salt.utils.minions}): _loader.exec_module(drainer) MASTER = 'manager.localdomain_master' @@ -405,6 +406,49 @@ class TestResults(DrainerTestCase): self.assertIn('push succeeded jid=2_done', self.logged('info')) +class TestTargeted(DrainerTestCase): + + def setUp(self): + super().setUp() + self.ckminions = MagicMock() + patcher = patch.object(drainer.salt.utils.minions, 'CkMinions', return_value=self.ckminions) + self.CkMinions = patcher.start() + self.addCleanup(patcher.stop) + + def test_drops_unmatched(self): + matches = {'G@role:so-sensor': ['sensor_sensor'], 'manager_manager': ['manager_manager']} + self.ckminions.check_minions.side_effect = lambda tgt, tgt_type: {'minions': matches.get(tgt, [])} + actions = [ + {'state': 'strelka', 'tgt': 'G@role:so-sensor'}, + {'state': 'suricata', 'tgt': 'G@role:so-heavynode'}, + {'highstate': True, 'tgt': 'G@role:so-idh'}, + {'state': 'soc', 'tgt': 'manager_manager', 'tgt_type': 'list'}, + ] + self.assertEqual(drainer._targeted(actions, self.log), [actions[0], actions[3]]) + self.CkMinions.assert_called_once() + self.assertEqual([c.args for c in self.ckminions.check_minions.call_args_list], [ + ('G@role:so-sensor', 'compound'), ('G@role:so-heavynode', 'compound'), + ('G@role:so-idh', 'compound'), ('manager_manager', 'list'), + ]) + info = self.logged('info') + self.assertIn('skipping suricata: no minions match tgt=G@role:so-heavynode', info) + self.assertIn('skipping highstate: no minions match tgt=G@role:so-idh', info) + self.assertNotIn('strelka', info) + + def test_master_config_error_keeps_actions(self): + actions = [{'state': 'soc', 'tgt': '*'}] + with patch.object(drainer.salt.config, 'master_config', side_effect=OSError('no config')): + self.assertEqual(drainer._targeted(actions, self.log), actions) + self.log.exception.assert_called_once() + self.CkMinions.assert_not_called() + + def test_check_error_keeps_action(self): + actions = [{'state': 'soc', 'tgt': 'G@role:'}] + self.ckminions.check_minions.side_effect = RuntimeError('bad target') + self.assertEqual(drainer._targeted(actions, self.log), actions) + self.log.exception.assert_called_once() + + class TestMain(DrainerTestCase): def setUp(self): @@ -414,6 +458,7 @@ class TestMain(DrainerTestCase): ('_make_logger', {'return_value': self.log}), ('_load_push_cfg', {'side_effect': lambda: self.cfg}), ('_check_dispatched', {}), + ('_targeted', {'side_effect': lambda actions, log: actions}), ): patcher = patch.object(drainer, name, **kwargs) setattr(self, name, patcher.start()) @@ -496,6 +541,29 @@ class TestMain(DrainerTestCase): record.assert_not_called() self.log.exception.assert_called_once() + def test_untargeted_not_dispatched(self): + path = self.intent('strelka.json', actions=[{'state': 'strelka', 'tgt': 'G@role:so-sensor'}]) + self._targeted.side_effect = None + self._targeted.return_value = [] + with patch.object(drainer, '_dispatch') as dispatch, \ + patch.object(drainer, '_record_dispatch') as record: + self.assertEqual(drainer.main(), 0) + dispatch.assert_not_called() + record.assert_not_called() + self.assertFalse(os.path.exists(path)) + self.assertIn('no action matches a minion; nothing to dispatch', self.logged('info')) + + def test_dispatches_only_targeted(self): + kept = {'state': 'soc', 'tgt': '*'} + self.intent('mixed.json', actions=[kept, {'state': 'strelka', 'tgt': 'G@role:so-sensor'}]) + self._targeted.side_effect = None + self._targeted.return_value = [kept] + with patch.object(drainer, '_dispatch', return_value=JID) as dispatch, \ + patch.object(drainer, '_record_dispatch') as record: + self.assertEqual(drainer.main(), 0) + self.assertEqual(dispatch.call_args[0][0], [kept]) + self.assertEqual(record.call_args[0][1], [kept]) + if __name__ == '__main__': unittest.main() From cc970892ed88538ca5345dc4f7c536c0c27d14f1 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 11 Nov 2026 10:07:08 -0500 Subject: [PATCH 3/6] FIX: keep the app's target when pushing a per-node setting For a setting saved for a single node, push_pillar replaced the app's target from pillar_push_map.yaml with the node id. The app's state was then applied to that node even if the node does not run the app. Seen in the lab: the SOC config test saves elasticsearch.config.cluster.routing.allocation.disk.threshold_enabled for the first node SOC lists, and SOC lists nodes in random order. When that node was a sensor, the elasticsearch state was applied to it and failed to render (so_roles has no so-sensor entry), and so-push-drainer logged "push failed". Narrow the app's target to the node instead of replacing it: "L@ and ( )". A node outside the app's target then matches nothing, and so-push-drainer skips the action. --- salt/reactor/push_pillar.sls | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/salt/reactor/push_pillar.sls b/salt/reactor/push_pillar.sls index 30417d6a2..2063a5744 100644 --- a/salt/reactor/push_pillar.sls +++ b/salt/reactor/push_pillar.sls @@ -7,7 +7,7 @@ # Two branches, keyed on node_id: # A) node_id populated -> the change is scoped to that one minion. Look up the # app in pillar_push_map.yaml and write an intent that runs the app's mapped -# state(s) targeted to just that node. +# state(s) on just that node, and only if the node is in the app's target. # B) node_id empty -> grid-wide app change. Look up the app in # pillar_push_map.yaml and write an intent with the entry's actions as-is. # @@ -116,15 +116,17 @@ def _app_from_setting(setting_id): def _node_actions(entry, node_id): - # Copy the app's mapped actions but retarget each one to the single node. + # Copy the app's mapped actions but narrow each one to the single node. # Preserves the state/highstate selection and any batch/batch_wait overrides. + # The app's own target is kept so the state is not applied to a node that does not + # run the app (e.g. an elasticsearch setting saved for a sensor fails to render there). actions = [] for action in entry: if not isinstance(action, dict): continue node_action = dict(action) - node_action['tgt'] = node_id - node_action['tgt_type'] = 'glob' + node_action['tgt'] = 'L@{} and ( {} )'.format(node_id, action.get('tgt', '*')) + node_action['tgt_type'] = 'compound' actions.append(node_action) return actions From 70053e362cdfae2415dbcb5d50e2ffa545f55c77 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 16 Nov 2026 10:20:05 -0500 Subject: [PATCH 4/6] FIX: exclude the drainer's push-result-unknown warning from so-log-check so-push-drainer logs "push result unknown" as a WARNING when salt-master restarts while a pushed state run is queued: the orchestration's next publish fails with AuthenticationError, but the minion still runs the queued state. so-log-check matches any line containing "error", so the quoted exception still failed the check. Reproduced on a distributed 3.5.0 grid: a sensoroni push queued behind a 150s state run on the manager, then salt-master was restarted. The orchestration (which runs under salt-minion.service, so it survives the restart) raised AuthenticationError 9s later, the drainer logged the WARNING, and the queued state run completed with 0 failed states. so-log-check --unknown flagged the line. The pattern requires AuthenticationError so an unknown result from any other raised exception, e.g. a bug in a state module, is still reported. --- salt/common/tools/sbin/so-log-check | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 73e7feedc..9f3025fdc 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -244,6 +244,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown" # so-dockerregistry logs a tag lookup miss during image copy; not tied to one docker version EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Could not index event to Elasticsearch.*\"version\" => \"9.0.8\"" # Expected during Elastic upgrade temporarily, as policies referencing older pipelines are updated + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|push result unknown.*AuthenticationError" # so-push-drainer WARNING: salt-master restarted while a push was queued; the queued state run still completes fi RESULT=0 From cc9dd0a101c4182cf0dfdcb13302e8c0105075fe Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 16 Nov 2026 14:19:51 -0500 Subject: [PATCH 5/6] FIX: drop malformed push_dispatched records instead of failing the pass so-push-drainer computed a record's age before the per-record try/except. A record whose dispatched_at or checked_at was not a number raised TypeError there, so every pass exited before the drain and auto-apply stopped. The traceback went to the stderr of a schedule with return_job: False, so nothing reached so-push-drainer.log. Only the drainer writes these records, so this takes a hand-edited file, but result tracking is not supposed to be able to block the drain. Log a warning and remove the record, as is already done for unreadable JSON and a missing jid. Also reword the so-log-check exclusion comment, which said the queued state run "still completes" where the drainer only claims it may have, and note at the drainer's warning that so-log-check matches its text. --- salt/common/tools/sbin/so-log-check | 2 +- salt/manager/tools/sbin/so-push-drainer | 13 ++++++++++--- salt/manager/tools/sbin/so-push-drainer_test.py | 16 ++++++++++++++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 9f3025fdc..a1434f704 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -244,7 +244,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown" # so-dockerregistry logs a tag lookup miss during image copy; not tied to one docker version EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Could not index event to Elasticsearch.*\"version\" => \"9.0.8\"" # Expected during Elastic upgrade temporarily, as policies referencing older pipelines are updated - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|push result unknown.*AuthenticationError" # so-push-drainer WARNING: salt-master restarted while a push was queued; the queued state run still completes + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|push result unknown.*AuthenticationError" # so-push-drainer WARNING: salt-master restarted while a push was queued; the queued state run may still complete fi RESULT=0 diff --git a/salt/manager/tools/sbin/so-push-drainer b/salt/manager/tools/sbin/so-push-drainer index 99461a029..569f47ef2 100644 --- a/salt/manager/tools/sbin/so-push-drainer +++ b/salt/manager/tools/sbin/so-push-drainer @@ -301,9 +301,15 @@ def _check_dispatched(log, now): if not isinstance(record, dict) or not record.get('jid'): _unlink(path, log) continue - age = now - record.get('dispatched_at', 0) - last_check = record.get('checked_at', record.get('dispatched_at', 0)) - if now - last_check >= _recheck_delay(age): + try: + age = now - record.get('dispatched_at', 0) + last_check = record.get('checked_at', record.get('dispatched_at', 0)) + is_due = now - last_check >= _recheck_delay(age) + except TypeError: + log.warning('malformed record for jid=%s; no longer tracking', record.get('jid')) + _unlink(path, log) + continue + if is_due: due.append((last_check, path, record, age)) # Least recently checked first, so pushes that are still running can't starve finished ones. for _, path, record, age in sorted(due, key=lambda item: item[:2])[:RESULT_CHECKS_PER_PASS]: @@ -332,6 +338,7 @@ def _report_result(record, age, log): return False failures = _orch_failures(ret) if failures and _result_unknown(ret): + # so-log-check excludes this message by its text. log.warning('push result unknown jid=%s paths=%s; the orchestration lost track of the state run, which may ' 'still have completed (if not, the change will be applied at the next scheduled highstate): %s', jid, paths, ' | '.join(failures)) diff --git a/salt/manager/tools/sbin/so-push-drainer_test.py b/salt/manager/tools/sbin/so-push-drainer_test.py index 10b825041..a6deab5f2 100644 --- a/salt/manager/tools/sbin/so-push-drainer_test.py +++ b/salt/manager/tools/sbin/so-push-drainer_test.py @@ -369,6 +369,22 @@ class TestResults(DrainerTestCase): self.assertIn('jid=1_bad', self.log.exception.call_args[0][0] % self.log.exception.call_args[0][1:]) self.assertIn('push succeeded jid=2_ok', self.logged('info')) + def test_check_dispatched_drops_malformed_record(self): + now = time.time() + bad_dispatched = self.write_json(self.dispatched, '1_bad.json', {'jid': '1_bad', 'dispatched_at': 'soon'}) + bad_checked = self.write_json(self.dispatched, '2_bad.json', { + 'jid': '2_bad', 'dispatched_at': now - 60, 'checked_at': None, + }) + good = self.record('3_ok', 60, now) + with patch.object(drainer, '_lookup_jid', return_value=SUCCESS_RET) as lookup: + drainer._check_dispatched(self.log, now) + for path in (bad_dispatched, bad_checked, good): + self.assertFalse(os.path.exists(path), path) + self.assertEqual([c.args[0] for c in lookup.call_args_list], ['3_ok']) + self.assertIn('malformed record for jid=1_bad', self.logged('warning')) + self.assertIn('malformed record for jid=2_bad', self.logged('warning')) + self.assertIn('push succeeded jid=3_ok', self.logged('info')) + def test_recheck_delay(self): self.assertEqual(drainer._recheck_delay(10), drainer.RESULT_CHECK_DELAY) self.assertEqual(drainer._recheck_delay(400), 100) From 9045aff88d82ad8d0015b33272e22a00526051c8 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 16 Nov 2026 14:19:51 -0500 Subject: [PATCH 6/6] FIX: correct the batch default in the pillar_push_map.yaml comment The comment gave the orch default batch as 25%. salt/salt/defaults.yaml sets salt:auto_apply:batch to 10%. --- salt/reactor/pillar_push_map.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/reactor/pillar_push_map.yaml b/salt/reactor/pillar_push_map.yaml index 8c78d2bb7..b49397c30 100644 --- a/salt/reactor/pillar_push_map.yaml +++ b/salt/reactor/pillar_push_map.yaml @@ -3,7 +3,7 @@ # # One pillar directory can map to multiple (state, tgt) actions. # tgt is a raw salt compound expression. tgt_type is always "compound". -# Per-action `batch` / `batch_wait` override the orch defaults (25% / 15s). +# Per-action `batch` / `batch_wait` override the orch defaults (10% / 15s). # An action with `highstate: True` triggers state.highstate instead of # state.apply -- see salt/orch/push_batch.sls. #