From 9732e1c639358785bcbc28e2d04d72f1da45e547 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 30 Sep 2026 15:43:04 -0400 Subject: [PATCH] Trim tracebacks in push failure log lines When an orchestration step raises, salt returns the full traceback as the step comment, and the drainer wrote it verbatim, putting ~70 lines into so-push-drainer.log per failure. Collapse comments to one line and, for tracebacks, keep only the lead-in and the raised exception, e.g. "apply_soc_1: An exception occurred in this state: salt.exceptions.AuthenticationError: Authentication error occurred." Seen on a standalone when a pushed highstate restarted salt-master while two queued pushes were waiting: their orchestrations lost the master connection and failed with AuthenticationError, although the minion completed both state runs. --- salt/manager/tools/sbin/so-push-drainer | 8 ++++++-- salt/manager/tools/sbin/so-push-drainer_test.py | 9 +++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/so-push-drainer b/salt/manager/tools/sbin/so-push-drainer index d80000ccb..369fcc0fe 100644 --- a/salt/manager/tools/sbin/so-push-drainer +++ b/salt/manager/tools/sbin/so-push-drainer @@ -144,7 +144,11 @@ def _dispatch(actions, log): def _trim(value): text = value if isinstance(value, str) else json.dumps(value, default=str) - text = text.strip() + lines = [line.strip() for line in text.splitlines() if line.strip()] + if 'Traceback (most recent call last):' in text: + # Keep the lead-in and the raised exception; the frames are noise in a log line. + lines = [text.split('Traceback (most recent call last):', 1)[0].strip(), lines[-1]] + text = ' '.join(line for line in lines if line) return text if len(text) <= TEXT_LIMIT else text[:TEXT_LIMIT] + '...' @@ -207,7 +211,7 @@ def _orch_failures(ret): for step in steps.values(): if not isinstance(step, dict) or step.get('result') is not False: continue - failures.append('{}: {}'.format(step.get('__id__', step.get('name')), step.get('comment'))) + failures.append('{}: {}'.format(step.get('__id__', step.get('name')), _trim(step.get('comment', '')))) minion_rets = (step.get('changes') or {}).get('ret') or {} for minion, minion_ret in minion_rets.items(): text = _minion_failure(minion_ret) diff --git a/salt/manager/tools/sbin/so-push-drainer_test.py b/salt/manager/tools/sbin/so-push-drainer_test.py index 65c1c042b..f758b6107 100644 --- a/salt/manager/tools/sbin/so-push-drainer_test.py +++ b/salt/manager/tools/sbin/so-push-drainer_test.py @@ -166,6 +166,15 @@ class TestHelpers(DrainerTestCase): self.assertEqual(drainer._trim(None), 'null') self.assertEqual(drainer._trim('x' * 600), 'x' * drainer.TEXT_LIMIT + '...') + def test_trim_traceback(self): + comment = ('An exception occurred in this state: Traceback (most recent call last):\n' + ' File "salt/client/__init__.py", line 1934, in pub\n' + ' raise AuthenticationError(err_msg)\n' + 'salt.exceptions.AuthenticationError: Authentication error occurred.\n') + self.assertEqual(drainer._trim(comment), 'An exception occurred in this state: ' + 'salt.exceptions.AuthenticationError: Authentication error occurred.') + self.assertEqual(drainer._trim('line one\n line two\n'), 'line one line two') + def test_unlink_missing_logs(self): drainer._unlink(os.path.join(self.tmpdir, 'missing'), self.log) self.log.exception.assert_called_once()