mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-05-10 21:30:30 +02:00
make so-yaml PG-canonical and add pillar-change reactor stack
Two coupled changes that together let so_pillar.* be the canonical config store, with config edits driving service reloads automatically: so-yaml PG-canonical mode - Adds /opt/so/conf/so-yaml/mode (and SO_YAML_BACKEND env override) with three values: dual (legacy), postgres (PG-only for managed paths), disk (emergency rollback). Bootstrap files (secrets.sls, ca/init.sls, *.nodes.sls, top.sls, ...) stay disk-only regardless via the existing SkipPath allowlist in so_yaml_postgres.locate. - loadYaml/writeYaml/purgeFile now route to so_pillar.* in postgres mode: replace/add/get all read+write the database with no disk file ever appearing. PG failure is fatal in postgres mode (no silent fallback); dual mode preserves the prior best-effort mirror. - so_yaml_postgres gains read_yaml(path), is_pg_managed(path), and is_enabled() so so-yaml can answer "is this path PG-managed and is PG up" without reaching into private helpers. - schema_pillar.sls writes /opt/so/conf/so-yaml/mode = postgres after the importer succeeds, so flipping postgres:so_pillar:enabled flips so-yaml's behavior in lockstep with the schema being live. pg_notify-driven change fan-out - 008_change_notify.sql adds so_pillar.change_queue + an AFTER trigger on pillar_entry that enqueues the locator and pg_notifies 'so_pillar_change'. Queue is drained at-least-once so engine restarts don't lose events; pg_notify is just the wakeup signal. - New salt-master engine pg_notify_pillar.py LISTENs on the channel, drains the queue with FOR UPDATE SKIP LOCKED, debounces bursts, and fires 'so/pillar/changed' events grouped by (scope, role, minion). - Reactor so_pillar_changed.sls catches the tag and dispatches to orch.so_pillar_reload, which carries a DISPATCH map of pillar-path prefix -> (state sls, role grain set) so adding a new service to the auto-reload list is a one-line edit instead of a new reactor. - Engine + reactor wiring is gated on the same postgres:so_pillar:enabled flag as the schema and ext_pillar config so the whole stack flips on/off together. Tests: 21 new cases (112 total, all passing) covering mode resolution, PG-managed detection, and PG-canonical read/write/purge routing with the PG client stubbed.
This commit is contained in:
@@ -0,0 +1,112 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
# Driven by the so_pillar_changed reactor. Translates a so_pillar.pillar_entry
|
||||
# change into (cache.clear_pillar -> saltutil.refresh_pillar -> state.apply)
|
||||
# on the appropriate target.
|
||||
#
|
||||
# Routing rules live in the DISPATCH map below — one entry per
|
||||
# (pillar_path prefix) -> (state sls, role grain). Add new services here
|
||||
# rather than wiring more reactors.
|
||||
#
|
||||
# Idempotent: state.apply is idempotent; if the pillar value didn't actually
|
||||
# change anything observable, the affected state runs a no-op. Bulk imports
|
||||
# and replays are safe.
|
||||
|
||||
{% set change = salt['pillar.get']('so_pillar_change', {}) %}
|
||||
{% set scope = change.get('scope') %}
|
||||
{% set role = change.get('role_name') %}
|
||||
{% set minion = change.get('minion_id') %}
|
||||
{% set changes = change.get('changes', []) %}
|
||||
|
||||
{# (pillar_path prefix) -> {sls: <state to apply>, role: <role grain that runs it>}
|
||||
role is a grain value (e.g. 'so-sensor'), used to compute compound targets
|
||||
when the change is global or role-scoped. #}
|
||||
{% set DISPATCH = {
|
||||
'suricata.': {'sls': 'suricata.config', 'roles': ['so-sensor', 'so-heavynode', 'so-standalone']},
|
||||
'sensor.': {'sls': 'suricata.config', 'roles': ['so-sensor', 'so-heavynode', 'so-standalone']},
|
||||
'zeek.': {'sls': 'zeek.config', 'roles': ['so-sensor', 'so-heavynode', 'so-standalone']},
|
||||
'stenographer.': {'sls': 'stenographer.config', 'roles': ['so-sensor', 'so-heavynode', 'so-standalone']},
|
||||
'pcap.': {'sls': 'pcap.config', 'roles': ['so-sensor', 'so-heavynode', 'so-standalone']},
|
||||
'logstash.': {'sls': 'logstash.config', 'roles': ['so-manager', 'so-managersearch', 'so-managerhype', 'so-receiver']},
|
||||
'redis.': {'sls': 'redis.config', 'roles': ['so-manager', 'so-managersearch', 'so-managerhype', 'so-standalone']},
|
||||
'kafka.': {'sls': 'kafka.config', 'roles': ['so-manager', 'so-managersearch', 'so-managerhype', 'so-receiver', 'so-searchnode']},
|
||||
'elasticsearch.': {'sls': 'elasticsearch.config','roles': ['so-manager', 'so-managersearch', 'so-managerhype', 'so-searchnode', 'so-heavynode', 'so-standalone']},
|
||||
'kibana.': {'sls': 'kibana.config', 'roles': ['so-manager', 'so-managersearch', 'so-managerhype', 'so-standalone']},
|
||||
'soc.': {'sls': 'soc.config', 'roles': ['so-manager', 'so-managersearch', 'so-managerhype', 'so-standalone']},
|
||||
'telegraf.': {'sls': 'telegraf.config', 'roles': ['*']},
|
||||
'fleet.': {'sls': 'fleet.config', 'roles': ['so-fleet']},
|
||||
'strelka.': {'sls': 'strelka.config', 'roles': ['so-sensor', 'so-heavynode', 'so-standalone']},
|
||||
} %}
|
||||
|
||||
{# Collect a deduplicated set of (sls, target_kind) actions. target_kind is
|
||||
either 'minion:<id>' (scope=minion) or 'roles:so-x,so-y' (scope=role/global). #}
|
||||
{% set actions = {} %}
|
||||
|
||||
{% for c in changes %}
|
||||
{% set path = c.get('pillar_path', '') %}
|
||||
{% for prefix, action in DISPATCH.items() %}
|
||||
{% if path.startswith(prefix) %}
|
||||
{% set sls = action['sls'] %}
|
||||
{% if scope == 'minion' and minion %}
|
||||
{% set key = sls ~ '|minion|' ~ minion %}
|
||||
{% set _ = actions.update({key: {'sls': sls, 'tgt': minion, 'tgt_type': 'glob'}}) %}
|
||||
{% else %}
|
||||
{% set role_targets = action['roles'] %}
|
||||
{% if '*' in role_targets %}
|
||||
{% set tgt = '*' %}
|
||||
{% set tgt_type = 'glob' %}
|
||||
{% else %}
|
||||
{% set tgt = ('I@role:' ~ role_targets|join(' or I@role:')) %}
|
||||
{% set tgt_type = 'compound' %}
|
||||
{% endif %}
|
||||
{% set key = sls ~ '|' ~ tgt %}
|
||||
{% set _ = actions.update({key: {'sls': sls, 'tgt': tgt, 'tgt_type': tgt_type}}) %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
{% if actions %}
|
||||
|
||||
{% for key, action in actions.items() %}
|
||||
{% set safe_id = loop.index0 | string %}
|
||||
|
||||
so_pillar_reload_clear_cache_{{ safe_id }}:
|
||||
salt.runner:
|
||||
- name: cache.clear_pillar
|
||||
- tgt: '{{ action.tgt }}'
|
||||
- tgt_type: '{{ action.tgt_type }}'
|
||||
|
||||
so_pillar_reload_refresh_pillar_{{ safe_id }}:
|
||||
salt.function:
|
||||
- name: saltutil.refresh_pillar
|
||||
- tgt: '{{ action.tgt }}'
|
||||
- tgt_type: '{{ action.tgt_type }}'
|
||||
- kwarg:
|
||||
wait: True
|
||||
- require:
|
||||
- salt: so_pillar_reload_clear_cache_{{ safe_id }}
|
||||
|
||||
so_pillar_reload_apply_state_{{ safe_id }}:
|
||||
salt.state:
|
||||
- tgt: '{{ action.tgt }}'
|
||||
- tgt_type: '{{ action.tgt_type }}'
|
||||
- sls:
|
||||
- {{ action.sls }}
|
||||
- queue: True
|
||||
- require:
|
||||
- salt: so_pillar_reload_refresh_pillar_{{ safe_id }}
|
||||
{% endfor %}
|
||||
|
||||
{% else %}
|
||||
|
||||
{# No DISPATCH entry matched. Pillar still gets refreshed so any other states
|
||||
read fresh values, but no service-specific reload is invoked. #}
|
||||
so_pillar_reload_unmapped_path_noop:
|
||||
test.nop
|
||||
{% do salt.log.info('orch.so_pillar_reload: no dispatch match for %s' % changes) %}
|
||||
|
||||
{% endif %}
|
||||
Reference in New Issue
Block a user