Compare commits

...
7 Commits
Author SHA1 Message Date
Josh Patterson 68ba6c835a Merge pull request #16405 from Security-Onion-Solutions/fix/auto-apply-push-after-master-restart
Fix/auto apply push after master restart
2026-11-18 10:51:42 -05:00
Josh Patterson aba2f5ea43 FIX: retry the push orchestration's refresh_pillar step
Holding dispatch in the drainer cannot help an orchestration that is
already running when salt-master restarts: its first action raises, and
the next action's refresh_pillar is published seconds after the restart,
before minions have rejoined the publisher, so it fails with "No minions
responded".

refresh_pillar now retries (3 attempts, 10s apart; a failed attempt takes
about 15s to time out, so the later attempts land after the reconnect
window). It also runs ahead of highstate actions, which now require it,
so they get the same reachability check. Retrying the highstate step
itself is not an option because it would re-run a fleetwide highstate
whenever any state failed.

Run straight after a salt-master restart on a standalone 3.5.0 install,
the orchestration previously failed with "No minions responded"; it now
succeeds on the second attempt. A target that is genuinely offline takes
about 65s to fail instead of 15s.
2026-11-18 10:27:34 -05:00
Josh Patterson 65816ae717 FIX: hold push dispatch while salt-master has only just started
A push dispatched seconds after a salt-master restart was lost: minions
rejoin the new publisher on a randomized ZeroMQ reconnect delay (up to
~22s with the default recon_default/recon_max), and a job published
before then reaches nobody. The orchestration's refresh_pillar step timed
out with "No minions responded", the drainer logged "push failed" as an
ERROR, and so-log-check failed the run.

Seen on a standalone 3.5.0 install: a highstate restarted salt-master
(new reactor/engine config), the drainer dispatched a soc push 8s later,
and the job cache shows the refresh_pillar job matched the minion but got
no return. Measured on the same host over five restarts, jobs published
in the first 4-9s were lost every time.

The drainer now leaves ready intents in place for a later pass while
/var/run/salt-master.pid is less than 45s old. On a multi-node grid this
also avoids a batch state run silently skipping the minions that have
not reconnected yet, since batch only sends the job to minions that
answer its ping.
2026-11-18 10:27:34 -05:00
Mike Reeves 4dbea0e28d Merge pull request #16403 from Security-Onion-Solutions/TOoSmOotH/568
INTERNAL: Trim Telegraf collection for Postgres
2026-11-17 16:58:20 -05:00
Josh Patterson 96cce0aad8 Merge pull request #16402 from Security-Onion-Solutions/fix/auto-apply-push-result-unknown
FIX: drop malformed push result records instead of failing the drain
2026-11-16 14:25:17 -05:00
Josh Patterson 5609aefbd5 Merge pull request #16393 from Security-Onion-Solutions/fix/auto-apply-push-result-unknown
FIX: keep the app's target when pushing a per-node setting
2026-11-11 10:08:41 -05:00
Josh Patterson 06d2d8ac9a Merge pull request #16392 from Security-Onion-Solutions/fix/auto-apply-push-result-unknown
FIX: skip push actions whose target matches no minions
2026-11-10 15:32:03 -05:00
3 changed files with 54 additions and 6 deletions

No files matched your search

+19
View File
@@ -17,6 +17,7 @@ is older than debounce_seconds, this script:
* concatenates the actions lists from every ready intent
* dedupes by (state or __highstate__, tgt, tgt_type)
* drops actions whose target matches no accepted minion (e.g. a role with no nodes yet)
* holds everything for a later pass while salt-master has only just started
* dispatches a single `salt-run state.orchestrate orch.push_batch --async`
with the deduped actions list passed as pillar kwargs
* deletes the contributed intent files on successful dispatch
@@ -46,6 +47,12 @@ PENDING_DIR = '/opt/so/state/push_pending'
LOCK_FILE = os.path.join(PENDING_DIR, '.lock')
LOG_FILE = '/opt/so/log/salt/so-push-drainer.log'
MASTER_CONFIG = '/etc/salt/master'
MASTER_PIDFILE = '/var/run/salt-master.pid'
# Minions rejoin a restarted master's publisher up to ~22s after it returns (zmq reconnect interval
# plus jitter, each up to recon_default + recon_max) and a job published before then reaches nobody.
# Doubled to allow for a slow master start.
MASTER_SETTLE_SECONDS = 45
DISPATCHED_DIR = '/opt/so/state/push_dispatched'
@@ -140,6 +147,14 @@ def _targeted(actions, log):
return targeted
def _master_settling(now):
try:
age = now - os.path.getmtime(MASTER_PIDFILE)
except OSError:
return False
return 0 <= age < MASTER_SETTLE_SECONDS
def _dispatch(actions, log):
pillar_arg = json.dumps({'actions': actions})
cmd = [
@@ -410,6 +425,10 @@ def main():
log.debug('no ready intents (%d still in debounce window)', skipped)
return 0
if _master_settling(now):
log.info('salt-master just started; holding %d intent(s) until minions reconnect', len(ready))
return 0
combined_actions = []
oldest_first_touch = now
all_paths = []
@@ -199,6 +199,17 @@ class TestHelpers(DrainerTestCase):
drainer._unlink(self.tmpdir, self.log)
self.log.exception.assert_called_once()
def test_master_settling(self):
pidfile = os.path.join(self.tmpdir, 'salt-master.pid')
now = time.time()
with patch.object(drainer, 'MASTER_PIDFILE', pidfile):
self.assertFalse(drainer._master_settling(now))
self.write_json(self.tmpdir, 'salt-master.pid', '1100')
for age, settling in ((0, True), (drainer.MASTER_SETTLE_SECONDS - 1, True),
(drainer.MASTER_SETTLE_SECONDS, False), (-60, False)):
os.utime(pidfile, (now - age, now - age))
self.assertIs(drainer._master_settling(now), settling, age)
class TestDispatch(DrainerTestCase):
@@ -475,6 +486,7 @@ class TestMain(DrainerTestCase):
('_load_push_cfg', {'side_effect': lambda: self.cfg}),
('_check_dispatched', {}),
('_targeted', {'side_effect': lambda actions, log: actions}),
('_master_settling', {'return_value': False}),
):
patcher = patch.object(drainer, name, **kwargs)
setattr(self, name, patcher.start())
@@ -528,6 +540,16 @@ class TestMain(DrainerTestCase):
with patch.object(drainer.os, 'unlink', side_effect=OSError('busy')):
self.assertEqual(drainer.main(), 0)
def test_holds_while_master_settling(self):
path = self.intent('pillar_soc.json')
self._master_settling.return_value = True
with patch.object(drainer, '_dispatch') as dispatch:
self.assertEqual(drainer.main(), 0)
dispatch.assert_not_called()
self._check_dispatched.assert_called_once()
self.assertTrue(os.path.exists(path))
self.assertIn('holding 1 intent(s)', self.logged('info'))
def test_dispatch_failure_keeps_intents(self):
path = self.intent('pillar_soc.json')
with patch.object(drainer, '_dispatch', return_value=None):
+13 -6
View File
@@ -4,8 +4,19 @@
{% set BATCH_WAIT = AUTOAPPLY.batch_wait %}
{# queue must be a top-level salt.state arg (kwarg is ignored); an int is max_queue and still fails on conflict #}
{# refresh_pillar doubles as the reachability check: minions take up to ~22s to rejoin a restarted master's
publisher, and a job published before then reaches nobody. Each failed attempt takes ~15s to time out. #}
{% for action in actions %}
refresh_pillar_{{ loop.index }}:
salt.function:
- name: saltutil.refresh_pillar
- tgt: '{{ action.tgt }}'
- tgt_type: {{ action.get('tgt_type', 'compound') }}
- retry:
attempts: 3
interval: 10
{% if action.get('highstate') %}
apply_highstate_{{ loop.index }}:
salt.state:
@@ -15,13 +26,9 @@ apply_highstate_{{ loop.index }}:
- batch: {{ action.get('batch', BATCH) }}
- batch_wait: {{ action.get('batch_wait', BATCH_WAIT) }}
- queue: True
- require:
- salt: refresh_pillar_{{ loop.index }}
{% else %}
refresh_pillar_{{ loop.index }}:
salt.function:
- name: saltutil.refresh_pillar
- tgt: '{{ action.tgt }}'
- tgt_type: {{ action.get('tgt_type', 'compound') }}
apply_{{ action.state | replace('.', '_') }}_{{ loop.index }}:
salt.state:
- tgt: '{{ action.tgt }}'