Merge pull request #16402 from Security-Onion-Solutions/fix/auto-apply-push-result-unknown

FIX: drop malformed push result records instead of failing the drain
This commit is contained in:
Josh Patterson authored and GitHub committed 2026-11-16 14:25:17 -05:00
commit 96cce0aad8
4 files changed
+28 -4

No files matched your search

+1
View File
@@ -244,6 +244,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown" # so-dockerregistry logs a tag lookup miss during image copy; not tied to one docker version
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Could not index event to Elasticsearch.*\"version\" => \"9.0.8\"" # Expected during Elastic upgrade temporarily, as policies referencing older pipelines are updated
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|push result unknown.*AuthenticationError" # so-push-drainer WARNING: salt-master restarted while a push was queued; the queued state run may still complete
fi
RESULT=0
+10 -3
View File
@@ -301,9 +301,15 @@ def _check_dispatched(log, now):
if not isinstance(record, dict) or not record.get('jid'):
_unlink(path, log)
continue
age = now - record.get('dispatched_at', 0)
last_check = record.get('checked_at', record.get('dispatched_at', 0))
if now - last_check >= _recheck_delay(age):
try:
age = now - record.get('dispatched_at', 0)
last_check = record.get('checked_at', record.get('dispatched_at', 0))
is_due = now - last_check >= _recheck_delay(age)
except TypeError:
log.warning('malformed record for jid=%s; no longer tracking', record.get('jid'))
_unlink(path, log)
continue
if is_due:
due.append((last_check, path, record, age))
# Least recently checked first, so pushes that are still running can't starve finished ones.
for _, path, record, age in sorted(due, key=lambda item: item[:2])[:RESULT_CHECKS_PER_PASS]:
@@ -332,6 +338,7 @@ def _report_result(record, age, log):
return False
failures = _orch_failures(ret)
if failures and _result_unknown(ret):
# so-log-check excludes this message by its text.
log.warning('push result unknown jid=%s paths=%s; the orchestration lost track of the state run, which may '
'still have completed (if not, the change will be applied at the next scheduled highstate): %s',
jid, paths, ' | '.join(failures))
@@ -369,6 +369,22 @@ class TestResults(DrainerTestCase):
self.assertIn('jid=1_bad', self.log.exception.call_args[0][0] % self.log.exception.call_args[0][1:])
self.assertIn('push succeeded jid=2_ok', self.logged('info'))
def test_check_dispatched_drops_malformed_record(self):
now = time.time()
bad_dispatched = self.write_json(self.dispatched, '1_bad.json', {'jid': '1_bad', 'dispatched_at': 'soon'})
bad_checked = self.write_json(self.dispatched, '2_bad.json', {
'jid': '2_bad', 'dispatched_at': now - 60, 'checked_at': None,
})
good = self.record('3_ok', 60, now)
with patch.object(drainer, '_lookup_jid', return_value=SUCCESS_RET) as lookup:
drainer._check_dispatched(self.log, now)
for path in (bad_dispatched, bad_checked, good):
self.assertFalse(os.path.exists(path), path)
self.assertEqual([c.args[0] for c in lookup.call_args_list], ['3_ok'])
self.assertIn('malformed record for jid=1_bad', self.logged('warning'))
self.assertIn('malformed record for jid=2_bad', self.logged('warning'))
self.assertIn('push succeeded jid=3_ok', self.logged('info'))
def test_recheck_delay(self):
self.assertEqual(drainer._recheck_delay(10), drainer.RESULT_CHECK_DELAY)
self.assertEqual(drainer._recheck_delay(400), 100)
+1 -1
View File
@@ -3,7 +3,7 @@
#
# One pillar directory can map to multiple (state, tgt) actions.
# tgt is a raw salt compound expression. tgt_type is always "compound".
# Per-action `batch` / `batch_wait` override the orch defaults (25% / 15s).
# Per-action `batch` / `batch_wait` override the orch defaults (10% / 15s).
# An action with `highstate: True` triggers state.highstate instead of
# state.apply -- see salt/orch/push_batch.sls.
#