mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-11-18 18:16:02 +01:00
Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
68ba6c835a | ||
|
|
aba2f5ea43 | ||
|
|
65816ae717 | ||
|
|
4dbea0e28d | ||
|
|
96cce0aad8 | ||
|
|
5609aefbd5 | ||
|
|
06d2d8ac9a |
No files matched your search
@@ -17,6 +17,7 @@ is older than debounce_seconds, this script:
|
|||||||
* concatenates the actions lists from every ready intent
|
* concatenates the actions lists from every ready intent
|
||||||
* dedupes by (state or __highstate__, tgt, tgt_type)
|
* dedupes by (state or __highstate__, tgt, tgt_type)
|
||||||
* drops actions whose target matches no accepted minion (e.g. a role with no nodes yet)
|
* drops actions whose target matches no accepted minion (e.g. a role with no nodes yet)
|
||||||
|
* holds everything for a later pass while salt-master has only just started
|
||||||
* dispatches a single `salt-run state.orchestrate orch.push_batch --async`
|
* dispatches a single `salt-run state.orchestrate orch.push_batch --async`
|
||||||
with the deduped actions list passed as pillar kwargs
|
with the deduped actions list passed as pillar kwargs
|
||||||
* deletes the contributed intent files on successful dispatch
|
* deletes the contributed intent files on successful dispatch
|
||||||
@@ -46,6 +47,12 @@ PENDING_DIR = '/opt/so/state/push_pending'
|
|||||||
LOCK_FILE = os.path.join(PENDING_DIR, '.lock')
|
LOCK_FILE = os.path.join(PENDING_DIR, '.lock')
|
||||||
LOG_FILE = '/opt/so/log/salt/so-push-drainer.log'
|
LOG_FILE = '/opt/so/log/salt/so-push-drainer.log'
|
||||||
MASTER_CONFIG = '/etc/salt/master'
|
MASTER_CONFIG = '/etc/salt/master'
|
||||||
|
MASTER_PIDFILE = '/var/run/salt-master.pid'
|
||||||
|
|
||||||
|
# Minions rejoin a restarted master's publisher up to ~22s after it returns (zmq reconnect interval
|
||||||
|
# plus jitter, each up to recon_default + recon_max) and a job published before then reaches nobody.
|
||||||
|
# Doubled to allow for a slow master start.
|
||||||
|
MASTER_SETTLE_SECONDS = 45
|
||||||
|
|
||||||
DISPATCHED_DIR = '/opt/so/state/push_dispatched'
|
DISPATCHED_DIR = '/opt/so/state/push_dispatched'
|
||||||
|
|
||||||
@@ -140,6 +147,14 @@ def _targeted(actions, log):
|
|||||||
return targeted
|
return targeted
|
||||||
|
|
||||||
|
|
||||||
|
def _master_settling(now):
|
||||||
|
try:
|
||||||
|
age = now - os.path.getmtime(MASTER_PIDFILE)
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
return 0 <= age < MASTER_SETTLE_SECONDS
|
||||||
|
|
||||||
|
|
||||||
def _dispatch(actions, log):
|
def _dispatch(actions, log):
|
||||||
pillar_arg = json.dumps({'actions': actions})
|
pillar_arg = json.dumps({'actions': actions})
|
||||||
cmd = [
|
cmd = [
|
||||||
@@ -410,6 +425,10 @@ def main():
|
|||||||
log.debug('no ready intents (%d still in debounce window)', skipped)
|
log.debug('no ready intents (%d still in debounce window)', skipped)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
if _master_settling(now):
|
||||||
|
log.info('salt-master just started; holding %d intent(s) until minions reconnect', len(ready))
|
||||||
|
return 0
|
||||||
|
|
||||||
combined_actions = []
|
combined_actions = []
|
||||||
oldest_first_touch = now
|
oldest_first_touch = now
|
||||||
all_paths = []
|
all_paths = []
|
||||||
|
|||||||
@@ -199,6 +199,17 @@ class TestHelpers(DrainerTestCase):
|
|||||||
drainer._unlink(self.tmpdir, self.log)
|
drainer._unlink(self.tmpdir, self.log)
|
||||||
self.log.exception.assert_called_once()
|
self.log.exception.assert_called_once()
|
||||||
|
|
||||||
|
def test_master_settling(self):
|
||||||
|
pidfile = os.path.join(self.tmpdir, 'salt-master.pid')
|
||||||
|
now = time.time()
|
||||||
|
with patch.object(drainer, 'MASTER_PIDFILE', pidfile):
|
||||||
|
self.assertFalse(drainer._master_settling(now))
|
||||||
|
self.write_json(self.tmpdir, 'salt-master.pid', '1100')
|
||||||
|
for age, settling in ((0, True), (drainer.MASTER_SETTLE_SECONDS - 1, True),
|
||||||
|
(drainer.MASTER_SETTLE_SECONDS, False), (-60, False)):
|
||||||
|
os.utime(pidfile, (now - age, now - age))
|
||||||
|
self.assertIs(drainer._master_settling(now), settling, age)
|
||||||
|
|
||||||
|
|
||||||
class TestDispatch(DrainerTestCase):
|
class TestDispatch(DrainerTestCase):
|
||||||
|
|
||||||
@@ -475,6 +486,7 @@ class TestMain(DrainerTestCase):
|
|||||||
('_load_push_cfg', {'side_effect': lambda: self.cfg}),
|
('_load_push_cfg', {'side_effect': lambda: self.cfg}),
|
||||||
('_check_dispatched', {}),
|
('_check_dispatched', {}),
|
||||||
('_targeted', {'side_effect': lambda actions, log: actions}),
|
('_targeted', {'side_effect': lambda actions, log: actions}),
|
||||||
|
('_master_settling', {'return_value': False}),
|
||||||
):
|
):
|
||||||
patcher = patch.object(drainer, name, **kwargs)
|
patcher = patch.object(drainer, name, **kwargs)
|
||||||
setattr(self, name, patcher.start())
|
setattr(self, name, patcher.start())
|
||||||
@@ -528,6 +540,16 @@ class TestMain(DrainerTestCase):
|
|||||||
with patch.object(drainer.os, 'unlink', side_effect=OSError('busy')):
|
with patch.object(drainer.os, 'unlink', side_effect=OSError('busy')):
|
||||||
self.assertEqual(drainer.main(), 0)
|
self.assertEqual(drainer.main(), 0)
|
||||||
|
|
||||||
|
def test_holds_while_master_settling(self):
|
||||||
|
path = self.intent('pillar_soc.json')
|
||||||
|
self._master_settling.return_value = True
|
||||||
|
with patch.object(drainer, '_dispatch') as dispatch:
|
||||||
|
self.assertEqual(drainer.main(), 0)
|
||||||
|
dispatch.assert_not_called()
|
||||||
|
self._check_dispatched.assert_called_once()
|
||||||
|
self.assertTrue(os.path.exists(path))
|
||||||
|
self.assertIn('holding 1 intent(s)', self.logged('info'))
|
||||||
|
|
||||||
def test_dispatch_failure_keeps_intents(self):
|
def test_dispatch_failure_keeps_intents(self):
|
||||||
path = self.intent('pillar_soc.json')
|
path = self.intent('pillar_soc.json')
|
||||||
with patch.object(drainer, '_dispatch', return_value=None):
|
with patch.object(drainer, '_dispatch', return_value=None):
|
||||||
|
|||||||
@@ -4,8 +4,19 @@
|
|||||||
{% set BATCH_WAIT = AUTOAPPLY.batch_wait %}
|
{% set BATCH_WAIT = AUTOAPPLY.batch_wait %}
|
||||||
|
|
||||||
{# queue must be a top-level salt.state arg (kwarg is ignored); an int is max_queue and still fails on conflict #}
|
{# queue must be a top-level salt.state arg (kwarg is ignored); an int is max_queue and still fails on conflict #}
|
||||||
|
{# refresh_pillar doubles as the reachability check: minions take up to ~22s to rejoin a restarted master's
|
||||||
|
publisher, and a job published before then reaches nobody. Each failed attempt takes ~15s to time out. #}
|
||||||
|
|
||||||
{% for action in actions %}
|
{% for action in actions %}
|
||||||
|
refresh_pillar_{{ loop.index }}:
|
||||||
|
salt.function:
|
||||||
|
- name: saltutil.refresh_pillar
|
||||||
|
- tgt: '{{ action.tgt }}'
|
||||||
|
- tgt_type: {{ action.get('tgt_type', 'compound') }}
|
||||||
|
- retry:
|
||||||
|
attempts: 3
|
||||||
|
interval: 10
|
||||||
|
|
||||||
{% if action.get('highstate') %}
|
{% if action.get('highstate') %}
|
||||||
apply_highstate_{{ loop.index }}:
|
apply_highstate_{{ loop.index }}:
|
||||||
salt.state:
|
salt.state:
|
||||||
@@ -15,13 +26,9 @@ apply_highstate_{{ loop.index }}:
|
|||||||
- batch: {{ action.get('batch', BATCH) }}
|
- batch: {{ action.get('batch', BATCH) }}
|
||||||
- batch_wait: {{ action.get('batch_wait', BATCH_WAIT) }}
|
- batch_wait: {{ action.get('batch_wait', BATCH_WAIT) }}
|
||||||
- queue: True
|
- queue: True
|
||||||
|
- require:
|
||||||
|
- salt: refresh_pillar_{{ loop.index }}
|
||||||
{% else %}
|
{% else %}
|
||||||
refresh_pillar_{{ loop.index }}:
|
|
||||||
salt.function:
|
|
||||||
- name: saltutil.refresh_pillar
|
|
||||||
- tgt: '{{ action.tgt }}'
|
|
||||||
- tgt_type: {{ action.get('tgt_type', 'compound') }}
|
|
||||||
|
|
||||||
apply_{{ action.state | replace('.', '_') }}_{{ loop.index }}:
|
apply_{{ action.state | replace('.', '_') }}_{{ loop.index }}:
|
||||||
salt.state:
|
salt.state:
|
||||||
- tgt: '{{ action.tgt }}'
|
- tgt: '{{ action.tgt }}'
|
||||||
|
|||||||
Reference in new issue
Block a user