From 70053e362cdfae2415dbcb5d50e2ffa545f55c77 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 16 Nov 2026 10:20:05 -0500 Subject: [PATCH 1/3] FIX: exclude the drainer's push-result-unknown warning from so-log-check so-push-drainer logs "push result unknown" as a WARNING when salt-master restarts while a pushed state run is queued: the orchestration's next publish fails with AuthenticationError, but the minion still runs the queued state. so-log-check matches any line containing "error", so the quoted exception still failed the check. Reproduced on a distributed 3.5.0 grid: a sensoroni push queued behind a 150s state run on the manager, then salt-master was restarted. The orchestration (which runs under salt-minion.service, so it survives the restart) raised AuthenticationError 9s later, the drainer logged the WARNING, and the queued state run completed with 0 failed states. so-log-check --unknown flagged the line. The pattern requires AuthenticationError so an unknown result from any other raised exception, e.g. a bug in a state module, is still reported. --- salt/common/tools/sbin/so-log-check | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 73e7feedc..9f3025fdc 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -244,6 +244,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown" # so-dockerregistry logs a tag lookup miss during image copy; not tied to one docker version EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Could not index event to Elasticsearch.*\"version\" => \"9.0.8\"" # Expected during Elastic upgrade temporarily, as policies referencing older pipelines are updated + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|push result unknown.*AuthenticationError" # so-push-drainer WARNING: salt-master restarted while a push was queued; the queued state run still completes fi RESULT=0 From cc9dd0a101c4182cf0dfdcb13302e8c0105075fe Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 16 Nov 2026 14:19:51 -0500 Subject: [PATCH 2/3] FIX: drop malformed push_dispatched records instead of failing the pass so-push-drainer computed a record's age before the per-record try/except. A record whose dispatched_at or checked_at was not a number raised TypeError there, so every pass exited before the drain and auto-apply stopped. The traceback went to the stderr of a schedule with return_job: False, so nothing reached so-push-drainer.log. Only the drainer writes these records, so this takes a hand-edited file, but result tracking is not supposed to be able to block the drain. Log a warning and remove the record, as is already done for unreadable JSON and a missing jid. Also reword the so-log-check exclusion comment, which said the queued state run "still completes" where the drainer only claims it may have, and note at the drainer's warning that so-log-check matches its text. --- salt/common/tools/sbin/so-log-check | 2 +- salt/manager/tools/sbin/so-push-drainer | 13 ++++++++++--- salt/manager/tools/sbin/so-push-drainer_test.py | 16 ++++++++++++++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 9f3025fdc..a1434f704 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -244,7 +244,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown" # so-dockerregistry logs a tag lookup miss during image copy; not tied to one docker version EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Could not index event to Elasticsearch.*\"version\" => \"9.0.8\"" # Expected during Elastic upgrade temporarily, as policies referencing older pipelines are updated - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|push result unknown.*AuthenticationError" # so-push-drainer WARNING: salt-master restarted while a push was queued; the queued state run still completes + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|push result unknown.*AuthenticationError" # so-push-drainer WARNING: salt-master restarted while a push was queued; the queued state run may still complete fi RESULT=0 diff --git a/salt/manager/tools/sbin/so-push-drainer b/salt/manager/tools/sbin/so-push-drainer index 99461a029..569f47ef2 100644 --- a/salt/manager/tools/sbin/so-push-drainer +++ b/salt/manager/tools/sbin/so-push-drainer @@ -301,9 +301,15 @@ def _check_dispatched(log, now): if not isinstance(record, dict) or not record.get('jid'): _unlink(path, log) continue - age = now - record.get('dispatched_at', 0) - last_check = record.get('checked_at', record.get('dispatched_at', 0)) - if now - last_check >= _recheck_delay(age): + try: + age = now - record.get('dispatched_at', 0) + last_check = record.get('checked_at', record.get('dispatched_at', 0)) + is_due = now - last_check >= _recheck_delay(age) + except TypeError: + log.warning('malformed record for jid=%s; no longer tracking', record.get('jid')) + _unlink(path, log) + continue + if is_due: due.append((last_check, path, record, age)) # Least recently checked first, so pushes that are still running can't starve finished ones. for _, path, record, age in sorted(due, key=lambda item: item[:2])[:RESULT_CHECKS_PER_PASS]: @@ -332,6 +338,7 @@ def _report_result(record, age, log): return False failures = _orch_failures(ret) if failures and _result_unknown(ret): + # so-log-check excludes this message by its text. log.warning('push result unknown jid=%s paths=%s; the orchestration lost track of the state run, which may ' 'still have completed (if not, the change will be applied at the next scheduled highstate): %s', jid, paths, ' | '.join(failures)) diff --git a/salt/manager/tools/sbin/so-push-drainer_test.py b/salt/manager/tools/sbin/so-push-drainer_test.py index 10b825041..a6deab5f2 100644 --- a/salt/manager/tools/sbin/so-push-drainer_test.py +++ b/salt/manager/tools/sbin/so-push-drainer_test.py @@ -369,6 +369,22 @@ class TestResults(DrainerTestCase): self.assertIn('jid=1_bad', self.log.exception.call_args[0][0] % self.log.exception.call_args[0][1:]) self.assertIn('push succeeded jid=2_ok', self.logged('info')) + def test_check_dispatched_drops_malformed_record(self): + now = time.time() + bad_dispatched = self.write_json(self.dispatched, '1_bad.json', {'jid': '1_bad', 'dispatched_at': 'soon'}) + bad_checked = self.write_json(self.dispatched, '2_bad.json', { + 'jid': '2_bad', 'dispatched_at': now - 60, 'checked_at': None, + }) + good = self.record('3_ok', 60, now) + with patch.object(drainer, '_lookup_jid', return_value=SUCCESS_RET) as lookup: + drainer._check_dispatched(self.log, now) + for path in (bad_dispatched, bad_checked, good): + self.assertFalse(os.path.exists(path), path) + self.assertEqual([c.args[0] for c in lookup.call_args_list], ['3_ok']) + self.assertIn('malformed record for jid=1_bad', self.logged('warning')) + self.assertIn('malformed record for jid=2_bad', self.logged('warning')) + self.assertIn('push succeeded jid=3_ok', self.logged('info')) + def test_recheck_delay(self): self.assertEqual(drainer._recheck_delay(10), drainer.RESULT_CHECK_DELAY) self.assertEqual(drainer._recheck_delay(400), 100) From 9045aff88d82ad8d0015b33272e22a00526051c8 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 16 Nov 2026 14:19:51 -0500 Subject: [PATCH 3/3] FIX: correct the batch default in the pillar_push_map.yaml comment The comment gave the orch default batch as 25%. salt/salt/defaults.yaml sets salt:auto_apply:batch to 10%. --- salt/reactor/pillar_push_map.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/reactor/pillar_push_map.yaml b/salt/reactor/pillar_push_map.yaml index 8c78d2bb7..b49397c30 100644 --- a/salt/reactor/pillar_push_map.yaml +++ b/salt/reactor/pillar_push_map.yaml @@ -3,7 +3,7 @@ # # One pillar directory can map to multiple (state, tgt) actions. # tgt is a raw salt compound expression. tgt_type is always "compound". -# Per-action `batch` / `batch_wait` override the orch defaults (25% / 15s). +# Per-action `batch` / `batch_wait` override the orch defaults (10% / 15s). # An action with `highstate: True` triggers state.highstate instead of # state.apply -- see salt/orch/push_batch.sls. #