FIX: drop malformed push_dispatched records instead of failing the pass

so-push-drainer computed a record's age before the per-record
try/except. A record whose dispatched_at or checked_at was not a number
raised TypeError there, so every pass exited before the drain and
auto-apply stopped. The traceback went to the stderr of a schedule with
return_job: False, so nothing reached so-push-drainer.log.

Only the drainer writes these records, so this takes a hand-edited file,
but result tracking is not supposed to be able to block the drain. Log a
warning and remove the record, as is already done for unreadable JSON
and a missing jid.

Also reword the so-log-check exclusion comment, which said the queued
state run "still completes" where the drainer only claims it may have,
and note at the drainer's warning that so-log-check matches its text.
This commit is contained in:
Josh Patterson committed 2026-11-16 14:19:51 -05:00
1 parent 70053e362c
commit cc9dd0a101
3 files changed
+27 -4

No files matched your search

+1 -1
View File
@@ -244,7 +244,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown" # so-dockerregistry logs a tag lookup miss during image copy; not tied to one docker version
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Could not index event to Elasticsearch.*\"version\" => \"9.0.8\"" # Expected during Elastic upgrade temporarily, as policies referencing older pipelines are updated
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|push result unknown.*AuthenticationError" # so-push-drainer WARNING: salt-master restarted while a push was queued; the queued state run still completes
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|push result unknown.*AuthenticationError" # so-push-drainer WARNING: salt-master restarted while a push was queued; the queued state run may still complete
fi
RESULT=0
+10 -3
View File
@@ -301,9 +301,15 @@ def _check_dispatched(log, now):
if not isinstance(record, dict) or not record.get('jid'):
_unlink(path, log)
continue
age = now - record.get('dispatched_at', 0)
last_check = record.get('checked_at', record.get('dispatched_at', 0))
if now - last_check >= _recheck_delay(age):
try:
age = now - record.get('dispatched_at', 0)
last_check = record.get('checked_at', record.get('dispatched_at', 0))
is_due = now - last_check >= _recheck_delay(age)
except TypeError:
log.warning('malformed record for jid=%s; no longer tracking', record.get('jid'))
_unlink(path, log)
continue
if is_due:
due.append((last_check, path, record, age))
# Least recently checked first, so pushes that are still running can't starve finished ones.
for _, path, record, age in sorted(due, key=lambda item: item[:2])[:RESULT_CHECKS_PER_PASS]:
@@ -332,6 +338,7 @@ def _report_result(record, age, log):
return False
failures = _orch_failures(ret)
if failures and _result_unknown(ret):
# so-log-check excludes this message by its text.
log.warning('push result unknown jid=%s paths=%s; the orchestration lost track of the state run, which may '
'still have completed (if not, the change will be applied at the next scheduled highstate): %s',
jid, paths, ' | '.join(failures))
@@ -369,6 +369,22 @@ class TestResults(DrainerTestCase):
self.assertIn('jid=1_bad', self.log.exception.call_args[0][0] % self.log.exception.call_args[0][1:])
self.assertIn('push succeeded jid=2_ok', self.logged('info'))
def test_check_dispatched_drops_malformed_record(self):
now = time.time()
bad_dispatched = self.write_json(self.dispatched, '1_bad.json', {'jid': '1_bad', 'dispatched_at': 'soon'})
bad_checked = self.write_json(self.dispatched, '2_bad.json', {
'jid': '2_bad', 'dispatched_at': now - 60, 'checked_at': None,
})
good = self.record('3_ok', 60, now)
with patch.object(drainer, '_lookup_jid', return_value=SUCCESS_RET) as lookup:
drainer._check_dispatched(self.log, now)
for path in (bad_dispatched, bad_checked, good):
self.assertFalse(os.path.exists(path), path)
self.assertEqual([c.args[0] for c in lookup.call_args_list], ['3_ok'])
self.assertIn('malformed record for jid=1_bad', self.logged('warning'))
self.assertIn('malformed record for jid=2_bad', self.logged('warning'))
self.assertIn('push succeeded jid=3_ok', self.logged('info'))
def test_recheck_delay(self):
self.assertEqual(drainer._recheck_delay(10), drainer.RESULT_CHECK_DELAY)
self.assertEqual(drainer._recheck_delay(400), 100)