diff --git a/pillar/top.sls b/pillar/top.sls index 712629dbf..aba3ef4c5 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -3,6 +3,8 @@ base: - ca - global.soc_global - global.adv_global + - salt.soc_salt + - salt.adv_salt - docker.soc_docker - docker.adv_docker - influxdb.token diff --git a/salt/global/defaults.yaml b/salt/global/defaults.yaml index 23cfb1454..60f7885ac 100644 --- a/salt/global/defaults.yaml +++ b/salt/global/defaults.yaml @@ -1,10 +1,3 @@ global: pcapengine: SURICATA pipeline: REDIS - push: - enabled: true - highstate_interval_hours: 2 - debounce_seconds: 30 - drain_interval: 15 - batch: '25%' - batch_wait: 15 diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index c46102fe7..c15f3eb98 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -59,41 +59,4 @@ global: description: Allows use of Endgame with Security Onion. This feature requires a license from Endgame. global: True advanced: True - push: - enabled: - description: Master kill-switch for the active push feature. When disabled, rule and pillar changes are picked up at the next scheduled highstate instead of being pushed immediately. - forcedType: bool - helpLink: push - global: True - highstate_interval_hours: - description: How often every minion in the grid runs a scheduled state.highstate, in hours. Lower values keep minions closer in sync at the cost of more load; higher values reduce load but increase worst-case latency for non-pushed changes. The salt-minion health check restarts a minion if its last highstate is older than this value plus one hour. - forcedType: int - helpLink: push - global: True - advanced: True - debounce_seconds: - description: Trailing-edge debounce window in seconds. A push intent must be quiet for this long before the drainer dispatches. Rapid bursts of edits within this window coalesce into one dispatch. - forcedType: int - helpLink: push - global: True - advanced: True - drain_interval: - description: How often the push drainer checks for ready intents, in seconds. Small values lower dispatch latency at the cost of more background work on the manager. - forcedType: int - helpLink: push - global: True - advanced: True - batch: - description: "Host batch size for push orchestrations. A number (e.g. '10') or a percentage (e.g. '25%'). Limits how many minions run the push state at once so large fleets don't thundering-herd." - helpLink: push - global: True - advanced: True - regex: '^([0-9]+%?)$' - regexFailureMessage: Enter a whole number or a whole-number percentage (e.g. 10 or 25%). - batch_wait: - description: Seconds to wait between host batches in a push orchestration. Gives the fleet time to breathe between waves. - forcedType: int - helpLink: push - global: True - advanced: True diff --git a/salt/manager/beacons.sls b/salt/manager/beacons.sls index 11574f9e2..b04017140 100644 --- a/salt/manager/beacons.sls +++ b/salt/manager/beacons.sls @@ -1,10 +1,10 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} -{% from 'global/map.jinja' import GLOBALMERGED %} +{% from 'salt/auto_apply.map.jinja' import AUTOAPPLY %} include: - salt.minion -{% if GLOBALS.is_manager and GLOBALMERGED.push.enabled %} +{% if GLOBALS.is_manager and AUTOAPPLY.enabled %} salt_beacons_pushstate: file.managed: - name: /etc/salt/minion.d/beacons_pushstate.conf diff --git a/salt/manager/files/beacons_pushstate.conf.jinja b/salt/manager/files/beacons_pushstate.conf.jinja index ce9259359..9f45d5d85 100644 --- a/salt/manager/files/beacons_pushstate.conf.jinja +++ b/salt/manager/files/beacons_pushstate.conf.jinja @@ -1,7 +1,7 @@ -{% from 'global/map.jinja' import GLOBALMERGED %} +{% from 'salt/auto_apply.map.jinja' import AUTOAPPLY %} beacons: pillar_db: - - interval: {{ GLOBALMERGED.push.drain_interval }} + - interval: {{ AUTOAPPLY.drain_interval }} - disable_during_state_run: True inotify: - disable_during_state_run: True diff --git a/salt/manager/tools/sbin/so-push-drainer b/salt/manager/tools/sbin/so-push-drainer index 4ce198fd0..fe25d31fc 100644 --- a/salt/manager/tools/sbin/so-push-drainer +++ b/salt/manager/tools/sbin/so-push-drainer @@ -60,9 +60,9 @@ def _make_logger(): def _load_push_cfg(): - """Read the global:push pillar subtree via salt-call. Returns a dict.""" + """Read the salt:auto_apply pillar subtree via salt-call. Returns a dict.""" caller = salt.client.Caller() - cfg = caller.cmd('pillar.get', 'global:push', {}) + cfg = caller.cmd('pillar.get', 'salt:auto_apply', {}) return cfg if isinstance(cfg, dict) else {} @@ -135,7 +135,7 @@ def main(): try: push = _load_push_cfg() except Exception: - log.exception('failed to read global:push pillar; aborting drain pass') + log.exception('failed to read salt:auto_apply pillar; aborting drain pass') return 1 if not push.get('enabled', True): diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 2b8680191..e667e6448 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -690,6 +690,21 @@ ensure_postgres_local_pillar() { chown -R socore:socore "$dir" } +ensure_salt_local_pillar() { + # The salt.auto_apply settings (moved from global.push) are a new SOC settings + # module, so the new pillar/top.sls references salt.soc_salt / salt.adv_salt + # unconditionally. Managers upgrading from before this change have no + # /opt/so/saltstack/local/pillar/salt/ (make_some_dirs only runs at install + # time), so the stubs must be created here before salt-master restarts against + # the new top.sls. + echo "Ensuring salt local pillar stubs exist." + local dir=/opt/so/saltstack/local/pillar/salt + mkdir -p "$dir" + [[ -f "$dir/soc_salt.sls" ]] || touch "$dir/soc_salt.sls" + [[ -f "$dir/adv_salt.sls" ]] || touch "$dir/adv_salt.sls" + chown -R socore:socore "$dir" +} + ensure_postgres_secret() { # On a fresh install, generate_passwords + secrets_pillar seed # secrets:postgres_pass in /opt/so/saltstack/local/pillar/secrets.sls. That @@ -851,6 +866,8 @@ kibana_backport_streams_index_template() { } up_to_3.2.0() { + ensure_salt_local_pillar + fix_logstash_0013_lumberjack_pipeline_name pin_elasticsearch_data_retention_method diff --git a/salt/orch/push_batch.sls b/salt/orch/push_batch.sls index 9eb435cce..ba8676961 100644 --- a/salt/orch/push_batch.sls +++ b/salt/orch/push_batch.sls @@ -1,7 +1,7 @@ -{% from 'global/map.jinja' import GLOBALMERGED %} +{% from 'salt/auto_apply.map.jinja' import AUTOAPPLY %} {% set actions = salt['pillar.get']('actions', []) %} -{% set BATCH = GLOBALMERGED.push.batch %} -{% set BATCH_WAIT = GLOBALMERGED.push.batch_wait %} +{% set BATCH = AUTOAPPLY.batch %} +{% set BATCH_WAIT = AUTOAPPLY.batch_wait %} {% for action in actions %} {% if action.get('highstate') %} diff --git a/salt/reactor/pillar_push_map.yaml b/salt/reactor/pillar_push_map.yaml index 36699d3de..51ad1db81 100644 --- a/salt/reactor/pillar_push_map.yaml +++ b/salt/reactor/pillar_push_map.yaml @@ -185,6 +185,17 @@ registry: - state: registry tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone' +# salt: fanout to a fleetwide highstate. The salt.auto_apply settings tune the +# push pipeline itself (enabled, debounce/drain intervals, batch sizing) and the +# per-minion highstate schedule; they are consumed by the manager's schedule, +# beacons, and master reactor config as well as every minion's highstate +# schedule, so a targeted re-apply isn't meaningful. A salt audit row only fires +# for SOC-driven salt.auto_apply edits -- salt version bumps go through soup, not +# SOC, so they never reach this map. +salt: + - highstate: True + tgt: '*' + # sensoroni: universal. sensoroni: - state: sensoroni diff --git a/salt/reactor/push_pillar.sls b/salt/reactor/push_pillar.sls index 06ee474df..47e586788 100644 --- a/salt/reactor/push_pillar.sls +++ b/salt/reactor/push_pillar.sls @@ -59,9 +59,9 @@ def _load_push_map(): def _push_enabled(): try: caller = Caller() - return bool(caller.cmd('pillar.get', 'global:push:enabled', True)) + return bool(caller.cmd('pillar.get', 'salt:auto_apply:enabled', True)) except Exception: - LOG.exception('push_pillar: pillar.get global:push:enabled failed, assuming enabled') + LOG.exception('push_pillar: pillar.get salt:auto_apply:enabled failed, assuming enabled') return True diff --git a/salt/reactor/push_strelka.sls b/salt/reactor/push_strelka.sls index 1d6a2b044..21727bc91 100644 --- a/salt/reactor/push_strelka.sls +++ b/salt/reactor/push_strelka.sls @@ -35,9 +35,9 @@ def _sensor_compound(): def _push_enabled(): try: caller = Caller() - return bool(caller.cmd('pillar.get', 'global:push:enabled', True)) + return bool(caller.cmd('pillar.get', 'salt:auto_apply:enabled', True)) except Exception: - LOG.exception('push_strelka: pillar.get global:push:enabled failed, assuming enabled') + LOG.exception('push_strelka: pillar.get salt:auto_apply:enabled failed, assuming enabled') return True diff --git a/salt/reactor/push_suricata.sls b/salt/reactor/push_suricata.sls index 468249296..53900e469 100644 --- a/salt/reactor/push_suricata.sls +++ b/salt/reactor/push_suricata.sls @@ -34,9 +34,9 @@ def _sensor_compound_plus_import(): def _push_enabled(): try: caller = Caller() - return bool(caller.cmd('pillar.get', 'global:push:enabled', True)) + return bool(caller.cmd('pillar.get', 'salt:auto_apply:enabled', True)) except Exception: - LOG.exception('push_suricata: pillar.get global:push:enabled failed, assuming enabled') + LOG.exception('push_suricata: pillar.get salt:auto_apply:enabled failed, assuming enabled') return True diff --git a/salt/salt/adv_salt.yaml b/salt/salt/adv_salt.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/salt/salt/auto_apply.map.jinja b/salt/salt/auto_apply.map.jinja new file mode 100644 index 000000000..6a353f4f8 --- /dev/null +++ b/salt/salt/auto_apply.map.jinja @@ -0,0 +1,2 @@ +{% import_yaml 'salt/defaults.yaml' as SALT_DEFAULTS %} +{% set AUTOAPPLY = salt['pillar.get']('salt:auto_apply', SALT_DEFAULTS.salt.auto_apply, merge=True) %} diff --git a/salt/salt/defaults.yaml b/salt/salt/defaults.yaml new file mode 100644 index 000000000..7dcbf0ff8 --- /dev/null +++ b/salt/salt/defaults.yaml @@ -0,0 +1,8 @@ +salt: + auto_apply: + enabled: true + highstate_interval_hours: 2 + debounce_seconds: 30 + drain_interval: 15 + batch: '25%' + batch_wait: 15 diff --git a/salt/salt/master.sls b/salt/salt/master.sls index 273e97e84..bb39f1395 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -10,7 +10,7 @@ # software that is protected by the license key." {% from 'allowed_states.map.jinja' import allowed_states %} -{% from 'global/map.jinja' import GLOBALMERGED %} +{% from 'salt/auto_apply.map.jinja' import AUTOAPPLY %} {% if sls in allowed_states %} include: @@ -65,7 +65,7 @@ engines_config: - name: /etc/salt/master.d/engines.conf - source: salt://salt/files/engines.conf -{% if GLOBALMERGED.push.enabled %} +{% if AUTOAPPLY.enabled %} reactor_pushstate_config: file.managed: - name: /etc/salt/master.d/reactor_pushstate.conf diff --git a/salt/salt/soc_salt.yaml b/salt/salt/soc_salt.yaml new file mode 100644 index 000000000..530c5ab79 --- /dev/null +++ b/salt/salt/soc_salt.yaml @@ -0,0 +1,38 @@ +salt: + auto_apply: + enabled: + description: Master kill-switch for the active push feature. When disabled, rule and pillar changes are picked up at the next scheduled highstate instead of being pushed immediately. + forcedType: bool + helpLink: push + global: True + highstate_interval_hours: + description: How often every minion in the grid runs a scheduled state.highstate, in hours. Lower values keep minions closer in sync at the cost of more load; higher values reduce load but increase worst-case latency for non-pushed changes. The salt-minion health check restarts a minion if its last highstate is older than this value plus one hour. + forcedType: int + helpLink: push + global: True + advanced: True + debounce_seconds: + description: Trailing-edge debounce window in seconds. A push intent must be quiet for this long before the drainer dispatches. Rapid bursts of edits within this window coalesce into one dispatch. + forcedType: int + helpLink: push + global: True + advanced: True + drain_interval: + description: How often the push drainer checks for ready intents, in seconds. Small values lower dispatch latency at the cost of more background work on the manager. + forcedType: int + helpLink: push + global: True + advanced: True + batch: + description: "Host batch size for push orchestrations. A number (e.g. '10') or a percentage (e.g. '25%'). Limits how many minions run the push state at once so large fleets don't thundering-herd." + helpLink: push + global: True + advanced: True + regex: '^([0-9]+%?)$' + regexFailureMessage: Enter a whole number or a whole-number percentage (e.g. 10 or 25%). + batch_wait: + description: Seconds to wait between host batches in a push orchestration. Gives the fleet time to breathe between waves. + forcedType: int + helpLink: push + global: True + advanced: True diff --git a/salt/schedule.sls b/salt/schedule.sls index 014d24460..0b71dc65f 100644 --- a/salt/schedule.sls +++ b/salt/schedule.sls @@ -1,22 +1,22 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} -{% from 'global/map.jinja' import GLOBALMERGED %} +{% from 'salt/auto_apply.map.jinja' import AUTOAPPLY %} highstate_schedule: schedule.present: - function: state.highstate - - hours: {{ GLOBALMERGED.push.highstate_interval_hours }} + - hours: {{ AUTOAPPLY.highstate_interval_hours }} - maxrunning: 1 {% if not GLOBALS.is_manager %} - splay: 1800 {% endif %} -{% if GLOBALS.is_manager and GLOBALMERGED.push.enabled %} +{% if GLOBALS.is_manager and AUTOAPPLY.enabled %} push_drain_schedule: schedule.present: - function: cmd.run - job_args: - /usr/sbin/so-push-drainer - - seconds: {{ GLOBALMERGED.push.drain_interval }} + - seconds: {{ AUTOAPPLY.drain_interval }} - maxrunning: 1 - return_job: False {% elif GLOBALS.is_manager %} diff --git a/setup/so-functions b/setup/so-functions index 2d5181dc1..8859aef97 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1432,7 +1432,7 @@ make_some_dirs() { mkdir -p $local_salt_dir/salt/firewall/portgroups mkdir -p $local_salt_dir/salt/firewall/ports - for THEDIR in bpf elasticsearch ntp firewall redis backup influxdb postgres strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos hydra idh elastalert stig global kafka versionlock hypervisor vm; do + for THEDIR in bpf elasticsearch ntp firewall redis backup influxdb postgres strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos hydra idh elastalert stig global salt kafka versionlock hypervisor vm; do mkdir -p $local_salt_dir/pillar/$THEDIR touch $local_salt_dir/pillar/$THEDIR/adv_$THEDIR.sls touch $local_salt_dir/pillar/$THEDIR/soc_$THEDIR.sls