Trim tracebacks in push failure log lines

When an orchestration step raises, salt returns the full traceback as the
step comment, and the drainer wrote it verbatim, putting ~70 lines into
so-push-drainer.log per failure. Collapse comments to one line and, for
tracebacks, keep only the lead-in and the raised exception, e.g.
"apply_soc_1: An exception occurred in this state:
salt.exceptions.AuthenticationError: Authentication error occurred."

Seen on a standalone when a pushed highstate restarted salt-master while
two queued pushes were waiting: their orchestrations lost the master
connection and failed with AuthenticationError, although the minion
completed both state runs.

salt/manager/tools/sbin/so-push-drainer

@@ -144,7 +144,11 @@ def _dispatch(actions, log):
 
 def _trim(value):
     text = value if isinstance(value, str) else json.dumps(value, default=str)
-    text = text.strip()
+    lines = [line.strip() for line in text.splitlines() if line.strip()]
+    if 'Traceback (most recent call last):' in text:
+        # Keep the lead-in and the raised exception; the frames are noise in a log line.
+        lines = [text.split('Traceback (most recent call last):', 1)[0].strip(), lines[-1]]
+    text = ' '.join(line for line in lines if line)
     return text if len(text) <= TEXT_LIMIT else text[:TEXT_LIMIT] + '...'
 
 
@@ -207,7 +211,7 @@ def _orch_failures(ret):
             for step in steps.values():
                 if not isinstance(step, dict) or step.get('result') is not False:
                     continue
-                failures.append('{}: {}'.format(step.get('__id__', step.get('name')), step.get('comment')))
+                failures.append('{}: {}'.format(step.get('__id__', step.get('name')), _trim(step.get('comment', ''))))
                 minion_rets = (step.get('changes') or {}).get('ret') or {}
                 for minion, minion_ret in minion_rets.items():
                     text = _minion_failure(minion_ret)

salt/manager/tools/sbin/so-push-drainer_test.py

@@ -166,6 +166,15 @@ class TestHelpers(DrainerTestCase):
         self.assertEqual(drainer._trim(None), 'null')
         self.assertEqual(drainer._trim('x' * 600), 'x' * drainer.TEXT_LIMIT + '...')
 
+    def test_trim_traceback(self):
+        comment = ('An exception occurred in this state: Traceback (most recent call last):\n'
+                   '  File "salt/client/__init__.py", line 1934, in pub\n'
+                   '    raise AuthenticationError(err_msg)\n'
+                   'salt.exceptions.AuthenticationError: Authentication error occurred.\n')
+        self.assertEqual(drainer._trim(comment), 'An exception occurred in this state: '
+                         'salt.exceptions.AuthenticationError: Authentication error occurred.')
+        self.assertEqual(drainer._trim('line one\n  line two\n'), 'line one line two')
+
     def test_unlink_missing_logs(self):
         drainer._unlink(os.path.join(self.tmpdir, 'missing'), self.log)
         self.log.exception.assert_called_once()