mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-11-18 18:16:02 +01:00
Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
68ba6c835a | ||
|
|
aba2f5ea43 | ||
|
|
65816ae717 | ||
|
|
4dbea0e28d | ||
|
|
96cce0aad8 | ||
|
|
5609aefbd5 | ||
|
|
06d2d8ac9a |
No files matched your search
@@ -17,6 +17,7 @@ is older than debounce_seconds, this script:
|
||||
* concatenates the actions lists from every ready intent
|
||||
* dedupes by (state or __highstate__, tgt, tgt_type)
|
||||
* drops actions whose target matches no accepted minion (e.g. a role with no nodes yet)
|
||||
* holds everything for a later pass while salt-master has only just started
|
||||
* dispatches a single `salt-run state.orchestrate orch.push_batch --async`
|
||||
with the deduped actions list passed as pillar kwargs
|
||||
* deletes the contributed intent files on successful dispatch
|
||||
@@ -46,6 +47,12 @@ PENDING_DIR = '/opt/so/state/push_pending'
|
||||
LOCK_FILE = os.path.join(PENDING_DIR, '.lock')
|
||||
LOG_FILE = '/opt/so/log/salt/so-push-drainer.log'
|
||||
MASTER_CONFIG = '/etc/salt/master'
|
||||
MASTER_PIDFILE = '/var/run/salt-master.pid'
|
||||
|
||||
# Minions rejoin a restarted master's publisher up to ~22s after it returns (zmq reconnect interval
|
||||
# plus jitter, each up to recon_default + recon_max) and a job published before then reaches nobody.
|
||||
# Doubled to allow for a slow master start.
|
||||
MASTER_SETTLE_SECONDS = 45
|
||||
|
||||
DISPATCHED_DIR = '/opt/so/state/push_dispatched'
|
||||
|
||||
@@ -140,6 +147,14 @@ def _targeted(actions, log):
|
||||
return targeted
|
||||
|
||||
|
||||
def _master_settling(now):
|
||||
try:
|
||||
age = now - os.path.getmtime(MASTER_PIDFILE)
|
||||
except OSError:
|
||||
return False
|
||||
return 0 <= age < MASTER_SETTLE_SECONDS
|
||||
|
||||
|
||||
def _dispatch(actions, log):
|
||||
pillar_arg = json.dumps({'actions': actions})
|
||||
cmd = [
|
||||
@@ -410,6 +425,10 @@ def main():
|
||||
log.debug('no ready intents (%d still in debounce window)', skipped)
|
||||
return 0
|
||||
|
||||
if _master_settling(now):
|
||||
log.info('salt-master just started; holding %d intent(s) until minions reconnect', len(ready))
|
||||
return 0
|
||||
|
||||
combined_actions = []
|
||||
oldest_first_touch = now
|
||||
all_paths = []
|
||||
|
||||
@@ -199,6 +199,17 @@ class TestHelpers(DrainerTestCase):
|
||||
drainer._unlink(self.tmpdir, self.log)
|
||||
self.log.exception.assert_called_once()
|
||||
|
||||
def test_master_settling(self):
|
||||
pidfile = os.path.join(self.tmpdir, 'salt-master.pid')
|
||||
now = time.time()
|
||||
with patch.object(drainer, 'MASTER_PIDFILE', pidfile):
|
||||
self.assertFalse(drainer._master_settling(now))
|
||||
self.write_json(self.tmpdir, 'salt-master.pid', '1100')
|
||||
for age, settling in ((0, True), (drainer.MASTER_SETTLE_SECONDS - 1, True),
|
||||
(drainer.MASTER_SETTLE_SECONDS, False), (-60, False)):
|
||||
os.utime(pidfile, (now - age, now - age))
|
||||
self.assertIs(drainer._master_settling(now), settling, age)
|
||||
|
||||
|
||||
class TestDispatch(DrainerTestCase):
|
||||
|
||||
@@ -475,6 +486,7 @@ class TestMain(DrainerTestCase):
|
||||
('_load_push_cfg', {'side_effect': lambda: self.cfg}),
|
||||
('_check_dispatched', {}),
|
||||
('_targeted', {'side_effect': lambda actions, log: actions}),
|
||||
('_master_settling', {'return_value': False}),
|
||||
):
|
||||
patcher = patch.object(drainer, name, **kwargs)
|
||||
setattr(self, name, patcher.start())
|
||||
@@ -528,6 +540,16 @@ class TestMain(DrainerTestCase):
|
||||
with patch.object(drainer.os, 'unlink', side_effect=OSError('busy')):
|
||||
self.assertEqual(drainer.main(), 0)
|
||||
|
||||
def test_holds_while_master_settling(self):
|
||||
path = self.intent('pillar_soc.json')
|
||||
self._master_settling.return_value = True
|
||||
with patch.object(drainer, '_dispatch') as dispatch:
|
||||
self.assertEqual(drainer.main(), 0)
|
||||
dispatch.assert_not_called()
|
||||
self._check_dispatched.assert_called_once()
|
||||
self.assertTrue(os.path.exists(path))
|
||||
self.assertIn('holding 1 intent(s)', self.logged('info'))
|
||||
|
||||
def test_dispatch_failure_keeps_intents(self):
|
||||
path = self.intent('pillar_soc.json')
|
||||
with patch.object(drainer, '_dispatch', return_value=None):
|
||||
|
||||
@@ -4,8 +4,19 @@
|
||||
{% set BATCH_WAIT = AUTOAPPLY.batch_wait %}
|
||||
|
||||
{# queue must be a top-level salt.state arg (kwarg is ignored); an int is max_queue and still fails on conflict #}
|
||||
{# refresh_pillar doubles as the reachability check: minions take up to ~22s to rejoin a restarted master's
|
||||
publisher, and a job published before then reaches nobody. Each failed attempt takes ~15s to time out. #}
|
||||
|
||||
{% for action in actions %}
|
||||
refresh_pillar_{{ loop.index }}:
|
||||
salt.function:
|
||||
- name: saltutil.refresh_pillar
|
||||
- tgt: '{{ action.tgt }}'
|
||||
- tgt_type: {{ action.get('tgt_type', 'compound') }}
|
||||
- retry:
|
||||
attempts: 3
|
||||
interval: 10
|
||||
|
||||
{% if action.get('highstate') %}
|
||||
apply_highstate_{{ loop.index }}:
|
||||
salt.state:
|
||||
@@ -15,13 +26,9 @@ apply_highstate_{{ loop.index }}:
|
||||
- batch: {{ action.get('batch', BATCH) }}
|
||||
- batch_wait: {{ action.get('batch_wait', BATCH_WAIT) }}
|
||||
- queue: True
|
||||
- require:
|
||||
- salt: refresh_pillar_{{ loop.index }}
|
||||
{% else %}
|
||||
refresh_pillar_{{ loop.index }}:
|
||||
salt.function:
|
||||
- name: saltutil.refresh_pillar
|
||||
- tgt: '{{ action.tgt }}'
|
||||
- tgt_type: {{ action.get('tgt_type', 'compound') }}
|
||||
|
||||
apply_{{ action.state | replace('.', '_') }}_{{ loop.index }}:
|
||||
salt.state:
|
||||
- tgt: '{{ action.tgt }}'
|
||||
|
||||
Reference in new issue
Block a user