From cc9dd0a101c4182cf0dfdcb13302e8c0105075fe Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 16 Nov 2026 14:19:51 -0500 Subject: [PATCH] FIX: drop malformed push_dispatched records instead of failing the pass so-push-drainer computed a record's age before the per-record try/except. A record whose dispatched_at or checked_at was not a number raised TypeError there, so every pass exited before the drain and auto-apply stopped. The traceback went to the stderr of a schedule with return_job: False, so nothing reached so-push-drainer.log. Only the drainer writes these records, so this takes a hand-edited file, but result tracking is not supposed to be able to block the drain. Log a warning and remove the record, as is already done for unreadable JSON and a missing jid. Also reword the so-log-check exclusion comment, which said the queued state run "still completes" where the drainer only claims it may have, and note at the drainer's warning that so-log-check matches its text. --- salt/common/tools/sbin/so-log-check | 2 +- salt/manager/tools/sbin/so-push-drainer | 13 ++++++++++--- salt/manager/tools/sbin/so-push-drainer_test.py | 16 ++++++++++++++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 9f3025fdc..a1434f704 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -244,7 +244,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown" # so-dockerregistry logs a tag lookup miss during image copy; not tied to one docker version EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Could not index event to Elasticsearch.*\"version\" => \"9.0.8\"" # Expected during Elastic upgrade temporarily, as policies referencing older pipelines are updated - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|push result unknown.*AuthenticationError" # so-push-drainer WARNING: salt-master restarted while a push was queued; the queued state run still completes + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|push result unknown.*AuthenticationError" # so-push-drainer WARNING: salt-master restarted while a push was queued; the queued state run may still complete fi RESULT=0 diff --git a/salt/manager/tools/sbin/so-push-drainer b/salt/manager/tools/sbin/so-push-drainer index 99461a029..569f47ef2 100644 --- a/salt/manager/tools/sbin/so-push-drainer +++ b/salt/manager/tools/sbin/so-push-drainer @@ -301,9 +301,15 @@ def _check_dispatched(log, now): if not isinstance(record, dict) or not record.get('jid'): _unlink(path, log) continue - age = now - record.get('dispatched_at', 0) - last_check = record.get('checked_at', record.get('dispatched_at', 0)) - if now - last_check >= _recheck_delay(age): + try: + age = now - record.get('dispatched_at', 0) + last_check = record.get('checked_at', record.get('dispatched_at', 0)) + is_due = now - last_check >= _recheck_delay(age) + except TypeError: + log.warning('malformed record for jid=%s; no longer tracking', record.get('jid')) + _unlink(path, log) + continue + if is_due: due.append((last_check, path, record, age)) # Least recently checked first, so pushes that are still running can't starve finished ones. for _, path, record, age in sorted(due, key=lambda item: item[:2])[:RESULT_CHECKS_PER_PASS]: @@ -332,6 +338,7 @@ def _report_result(record, age, log): return False failures = _orch_failures(ret) if failures and _result_unknown(ret): + # so-log-check excludes this message by its text. log.warning('push result unknown jid=%s paths=%s; the orchestration lost track of the state run, which may ' 'still have completed (if not, the change will be applied at the next scheduled highstate): %s', jid, paths, ' | '.join(failures)) diff --git a/salt/manager/tools/sbin/so-push-drainer_test.py b/salt/manager/tools/sbin/so-push-drainer_test.py index 10b825041..a6deab5f2 100644 --- a/salt/manager/tools/sbin/so-push-drainer_test.py +++ b/salt/manager/tools/sbin/so-push-drainer_test.py @@ -369,6 +369,22 @@ class TestResults(DrainerTestCase): self.assertIn('jid=1_bad', self.log.exception.call_args[0][0] % self.log.exception.call_args[0][1:]) self.assertIn('push succeeded jid=2_ok', self.logged('info')) + def test_check_dispatched_drops_malformed_record(self): + now = time.time() + bad_dispatched = self.write_json(self.dispatched, '1_bad.json', {'jid': '1_bad', 'dispatched_at': 'soon'}) + bad_checked = self.write_json(self.dispatched, '2_bad.json', { + 'jid': '2_bad', 'dispatched_at': now - 60, 'checked_at': None, + }) + good = self.record('3_ok', 60, now) + with patch.object(drainer, '_lookup_jid', return_value=SUCCESS_RET) as lookup: + drainer._check_dispatched(self.log, now) + for path in (bad_dispatched, bad_checked, good): + self.assertFalse(os.path.exists(path), path) + self.assertEqual([c.args[0] for c in lookup.call_args_list], ['3_ok']) + self.assertIn('malformed record for jid=1_bad', self.logged('warning')) + self.assertIn('malformed record for jid=2_bad', self.logged('warning')) + self.assertIn('push succeeded jid=3_ok', self.logged('info')) + def test_recheck_delay(self): self.assertEqual(drainer._recheck_delay(10), drainer.RESULT_CHECK_DELAY) self.assertEqual(drainer._recheck_delay(400), 100)