From 613d31c8a64a90ee45635a7ceb7a4022d58c9174 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 5 Mar 2026 11:52:09 -0500 Subject: [PATCH 001/219] merge --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 4a36342fc..03e153fda 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.0.0 +3.0.0-kilo From a0cf0489d6b1720ecb5b7c9eb0c5e1ab517bfb9e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 10 Apr 2026 15:43:16 -0400 Subject: [PATCH 002/219] reduce highstate frequency with active push for rules and pillars - schedule highstate every 2 hours (was 15 minutes); interval lives in global:push:highstate_interval_hours so the SOC admin UI can tune it and so-salt-minion-check derives its threshold as (interval + 1) * 3600 - add inotify beacon on the manager + master reactor + orch.push_batch that writes per-app intent files, with a so-push-drainer schedule on the manager that debounces, dedupes, and dispatches a single orchestration - pillar_push_map.yaml allowlists the apps whose pillar changes trigger an immediate targeted state.apply (targets verified against salt/top.sls); edits under pillar/minions/ trigger a state.highstate on that one minion - host-batch every push orchestration (batch: 25%, batch_wait: 15) so rule changes don't thundering-herd large fleets - new global:push:enabled kill-switch tears down the beacon, reactor config, and drainer schedule on the next highstate for operators who want to keep highstate-only behavior - set restart_policy: unless-stopped on 23 container states so docker recovers crashes without waiting for the next highstate; leave registry (always), strelka/backend (on-failure), kratos, and hydra alone with inline comments explaining why --- .../tools/sbin_jinja/so-salt-minion-check | 5 +- salt/elastalert/enabled.sls | 1 + .../enabled.sls | 1 + salt/elasticagent/enabled.sls | 1 + salt/elasticfleet/enabled.sls | 1 + salt/elasticsearch/enabled.sls | 1 + salt/global/defaults.yaml | 9 +- salt/global/soc_global.yaml | 37 +++ salt/hydra/enabled.sls | 1 + salt/idh/enabled.sls | 1 + salt/influxdb/enabled.sls | 1 + salt/kafka/enabled.sls | 1 + salt/kibana/enabled.sls | 1 + salt/kratos/enabled.sls | 1 + salt/logstash/enabled.sls | 1 + salt/manager/tools/sbin/so-push-drainer | 233 ++++++++++++++++++ salt/nginx/enabled.sls | 1 + salt/orch/push_batch.sls | 37 +++ salt/reactor/pillar_push_map.yaml | 128 ++++++++++ salt/reactor/push_pillar.sls | 170 +++++++++++++ salt/reactor/push_strelka.sls | 96 ++++++++ salt/reactor/push_suricata.sls | 95 +++++++ salt/redis/enabled.sls | 1 + salt/registry/enabled.sls | 3 + salt/salt/beacons.sls | 22 +- salt/salt/files/beacons_pushstate.conf.jinja | 26 ++ salt/salt/files/reactor_pushstate.conf | 7 + salt/salt/master.sls | 17 ++ salt/salt/minion.defaults.yaml | 1 - salt/schedule.sls | 22 +- salt/sensoroni/enabled.sls | 1 + salt/soc/enabled.sls | 1 + salt/strelka/backend/enabled.sls | 4 + salt/strelka/coordinator/enabled.sls | 1 + salt/strelka/filestream/enabled.sls | 1 + salt/strelka/frontend/enabled.sls | 1 + salt/strelka/gatekeeper/enabled.sls | 1 + salt/strelka/manager/enabled.sls | 1 + salt/suricata/enabled.sls | 1 + salt/tcpreplay/init.sls | 1 + salt/telegraf/enabled.sls | 1 + salt/zeek/enabled.sls | 1 + 42 files changed, 927 insertions(+), 10 deletions(-) create mode 100644 salt/manager/tools/sbin/so-push-drainer create mode 100644 salt/orch/push_batch.sls create mode 100644 salt/reactor/pillar_push_map.yaml create mode 100644 salt/reactor/push_pillar.sls create mode 100644 salt/reactor/push_strelka.sls create mode 100644 salt/reactor/push_suricata.sls create mode 100644 salt/salt/files/beacons_pushstate.conf.jinja create mode 100644 salt/salt/files/reactor_pushstate.conf diff --git a/salt/common/tools/sbin_jinja/so-salt-minion-check b/salt/common/tools/sbin_jinja/so-salt-minion-check index 47d3bb7e1..3b2b32afe 100755 --- a/salt/common/tools/sbin_jinja/so-salt-minion-check +++ b/salt/common/tools/sbin_jinja/so-salt-minion-check @@ -1,5 +1,3 @@ -{% import_yaml 'salt/minion.defaults.yaml' as SALT_MINION_DEFAULTS -%} - #!/bin/bash # # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one @@ -25,7 +23,8 @@ SYSTEM_START_TIME=$(date -d "$('.format(len(actions))) + try: + result = subprocess.run( + cmd, check=True, capture_output=True, text=True, timeout=60, + ) + except subprocess.CalledProcessError as exc: + log.error('dispatch failed (rc=%s): stdout=%s stderr=%s', + exc.returncode, exc.stdout, exc.stderr) + return False + except subprocess.TimeoutExpired: + log.error('dispatch timed out after 60s') + return False + except Exception: + log.exception('dispatch raised') + return False + log.info('dispatch accepted: %s', (result.stdout or '').strip()) + return True + + +def main(): + log = _make_logger() + + if not os.path.isdir(PENDING_DIR): + # Nothing to do; reactors create the dir on first use. + return 0 + + try: + push = _load_push_cfg() + except Exception: + log.exception('failed to read global:push pillar; aborting drain pass') + return 1 + + if not push.get('enabled', True): + log.debug('push disabled; exiting') + return 0 + + debounce_seconds = int(push.get('debounce_seconds', 30)) + + os.makedirs(PENDING_DIR, exist_ok=True) + lock_fd = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR, 0o644) + try: + fcntl.flock(lock_fd, fcntl.LOCK_EX) + + intent_files = [ + p for p in sorted(glob.glob(os.path.join(PENDING_DIR, '*.json'))) + if os.path.basename(p) != '.lock' + ] + if not intent_files: + return 0 + + now = time.time() + ready = [] + skipped = 0 + broken = [] + for path in intent_files: + intent = _read_intent(path, log) + if not isinstance(intent, dict): + broken.append(path) + continue + last_touch = intent.get('last_touch', 0) + if now - last_touch < debounce_seconds: + skipped += 1 + continue + ready.append((path, intent)) + + for path in broken: + try: + os.unlink(path) + except OSError: + pass + + if not ready: + if skipped: + log.debug('no ready intents (%d still in debounce window)', skipped) + return 0 + + combined_actions = [] + oldest_first_touch = now + all_paths = [] + for path, intent in ready: + combined_actions.extend(intent.get('actions', []) or []) + first = intent.get('first_touch', now) + if first < oldest_first_touch: + oldest_first_touch = first + all_paths.extend(intent.get('paths', []) or []) + + deduped = _dedupe_actions(combined_actions) + if not deduped: + log.warning('%d intent(s) had no usable actions; clearing', len(ready)) + for path, _ in ready: + try: + os.unlink(path) + except OSError: + pass + return 0 + + debounce_duration = now - oldest_first_touch + log.info( + 'draining %d intent(s): %d action(s) after dedupe (raw=%d), ' + 'debounce_duration=%.1fs, paths=%s', + len(ready), len(deduped), len(combined_actions), + debounce_duration, all_paths[:20], + ) + + if not _dispatch(deduped, log): + log.warning('dispatch failed; leaving intent files in place for retry') + return 1 + + for path, _ in ready: + try: + os.unlink(path) + except OSError: + log.exception('failed to remove drained intent %s', path) + + return 0 + finally: + try: + fcntl.flock(lock_fd, fcntl.LOCK_UN) + finally: + os.close(lock_fd) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/salt/nginx/enabled.sls b/salt/nginx/enabled.sls index 2e4c9631c..c50ad8f8f 100644 --- a/salt/nginx/enabled.sls +++ b/salt/nginx/enabled.sls @@ -34,6 +34,7 @@ make-rule-dir-nginx: so-nginx: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-nginx:{{ GLOBALS.so_version }} + - restart_policy: unless-stopped - hostname: so-nginx - networks: - sobridge: diff --git a/salt/orch/push_batch.sls b/salt/orch/push_batch.sls new file mode 100644 index 000000000..9eb435cce --- /dev/null +++ b/salt/orch/push_batch.sls @@ -0,0 +1,37 @@ +{% from 'global/map.jinja' import GLOBALMERGED %} +{% set actions = salt['pillar.get']('actions', []) %} +{% set BATCH = GLOBALMERGED.push.batch %} +{% set BATCH_WAIT = GLOBALMERGED.push.batch_wait %} + +{% for action in actions %} +{% if action.get('highstate') %} +apply_highstate_{{ loop.index }}: + salt.state: + - tgt: '{{ action.tgt }}' + - tgt_type: {{ action.get('tgt_type', 'compound') }} + - highstate: True + - batch: {{ action.get('batch', BATCH) }} + - batch_wait: {{ action.get('batch_wait', BATCH_WAIT) }} + - kwarg: + queue: 2 +{% else %} +refresh_pillar_{{ loop.index }}: + salt.function: + - name: saltutil.refresh_pillar + - tgt: '{{ action.tgt }}' + - tgt_type: {{ action.get('tgt_type', 'compound') }} + +apply_{{ action.state | replace('.', '_') }}_{{ loop.index }}: + salt.state: + - tgt: '{{ action.tgt }}' + - tgt_type: {{ action.get('tgt_type', 'compound') }} + - sls: + - {{ action.state }} + - batch: {{ action.get('batch', BATCH) }} + - batch_wait: {{ action.get('batch_wait', BATCH_WAIT) }} + - kwarg: + queue: 2 + - require: + - salt: refresh_pillar_{{ loop.index }} +{% endif %} +{% endfor %} diff --git a/salt/reactor/pillar_push_map.yaml b/salt/reactor/pillar_push_map.yaml new file mode 100644 index 000000000..95fb0544e --- /dev/null +++ b/salt/reactor/pillar_push_map.yaml @@ -0,0 +1,128 @@ +# One pillar directory can map to multiple (state, tgt) actions. +# tgt is a raw salt compound expression. tgt_type is always "compound". +# Per-action `batch` / `batch_wait` override the orch defaults (25% / 15s). +# +# Notes: +# - `bpf` is a pillar-only dir (no state of its own) consumed by both +# zeek and suricata via macros, so a bpf pillar change re-applies both. +# - suricata/strelka/zeek/elasticsearch/redis/kafka/logstash etc. have +# their own pillar dirs AND their own state, so they map 1:1 (or 1:2 +# in strelka's case, because of the split init.sls / manager.sls). +# - `data` and `node_data` pillar dirs are intentionally omitted -- +# they're pillar-only data consumed by many states; trying to handle +# them generically would amount to a highstate. +# +# The role sets here were verified line-by-line against salt/top.sls. If +# salt/top.sls changes how an app is targeted, update the corresponding +# compound here. + +# firewall: the one pillar everyone touches. Applied everywhere intentionally +# because every host's iptables needs to know about every other host in the +# grid. Salt's firewall state is idempotent (file.managed + iptables-restore +# onchanges in salt/firewall/init.sls), so hosts whose rendered firewall is +# unchanged do a file comparison and no-op without touching iptables -- actual +# reload happens only on the hosts whose rules actually changed. Fleetwide +# blast radius is intentional and matches the pre-plan behavior via highstate. +# Adding N sensors in a burst coalesces into one dispatch via the drainer. +firewall: + - state: firewall + tgt: '*' + +# bpf is pillar-only (no state); consumed by both zeek and suricata as macros. +# Both states run on sensor_roles + so-import per salt/top.sls. +bpf: + - state: zeek + tgt: 'G@role:so-eval or G@role:so-heavynode or G@role:so-import or G@role:so-sensor or G@role:so-standalone' + - state: suricata + tgt: 'G@role:so-eval or G@role:so-heavynode or G@role:so-import or G@role:so-sensor or G@role:so-standalone' + +# ca is applied universally. +ca: + - state: ca + tgt: '*' + +# elastalert: eval, standalone, manager, managerhype, managersearch (NOT import). +elastalert: + - state: elastalert + tgt: 'G@role:so-eval or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone' + +# elasticsearch: 8 roles. +elasticsearch: + - state: elasticsearch + tgt: 'G@role:so-eval or G@role:so-heavynode or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-searchnode or G@role:so-standalone' + +# elasticagent: so-heavynode only. +elasticagent: + - state: elasticagent + tgt: 'G@role:so-heavynode' + +# elasticfleet: base state only on pillar change. elasticfleet.install_agent_grid +# is a deploy/enrollment step, not a config reload; leave it to the next highstate. +elasticfleet: + - state: elasticfleet + tgt: 'G@role:so-eval or G@role:so-fleet or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone' + +# healthcheck: eval, sensor, standalone only. +healthcheck: + - state: healthcheck + tgt: 'G@role:so-eval or G@role:so-sensor or G@role:so-standalone' + +# influxdb: manager_roles exactly. +influxdb: + - state: influxdb + tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone' + +# kafka: standalone, manager, managerhype, managersearch, searchnode, receiver. +kafka: + - state: kafka + tgt: 'G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode or G@role:so-standalone' + +# kibana: manager_roles exactly. +kibana: + - state: kibana + tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone' + +# logstash: 8 roles, no eval/import. +logstash: + - state: logstash + tgt: 'G@role:so-fleet or G@role:so-heavynode or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode or G@role:so-standalone' + +# nginx: 10 specific roles. NOT receiver, idh, hypervisor, desktop. +nginx: + - state: nginx + tgt: 'G@role:so-eval or G@role:so-fleet or G@role:so-heavynode or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-searchnode or G@role:so-sensor or G@role:so-standalone' + +# redis: 6 roles. standalone, manager, managerhype, managersearch, heavynode, receiver. +# (NOT eval, NOT import, NOT searchnode.) +redis: + - state: redis + tgt: 'G@role:so-heavynode or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-receiver or G@role:so-standalone' + +# soc: manager_roles exactly. +soc: + - state: soc + tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone' + +# strelka: sensor-side only on pillar change (sensor_roles). strelka.manager is +# intentionally NOT fired on pillar changes -- YARA rule and strelka config +# pillar changes are consumed by the sensor-side strelka backend, and re-running +# strelka.manager on managers is both unnecessary and disruptive. strelka.manager +# is left to the 2-hour highstate. +strelka: + - state: strelka + tgt: 'G@role:so-eval or G@role:so-heavynode or G@role:so-sensor or G@role:so-standalone' + +# suricata: sensor_roles + so-import (5 roles). +suricata: + - state: suricata + tgt: 'G@role:so-eval or G@role:so-heavynode or G@role:so-import or G@role:so-sensor or G@role:so-standalone' + +# telegraf: universal. +telegraf: + - state: telegraf + tgt: '*' + +# zeek: sensor_roles + so-import (5 roles). +zeek: + - state: zeek + tgt: 'G@role:so-eval or G@role:so-heavynode or G@role:so-import or G@role:so-sensor or G@role:so-standalone' diff --git a/salt/reactor/push_pillar.sls b/salt/reactor/push_pillar.sls new file mode 100644 index 000000000..a31fe9be4 --- /dev/null +++ b/salt/reactor/push_pillar.sls @@ -0,0 +1,170 @@ +#!py + +# Reactor invoked by the inotify beacon on pillar file changes under +# /opt/so/saltstack/local/pillar/. +# +# Two branches: +# A) per-minion override under pillar/minions/.sls or adv_.sls +# -> write an intent that runs state.highstate on just that minion. +# B) shared app pillar (pillar//...) -> look up in +# pillar_push_map.yaml and write an intent with the entry's actions. +# +# Reactors never dispatch directly. The so-push-drainer schedule picks up +# ready intents, dedupes across pending files, and dispatches orch.push_batch. +# See plan /home/mreeves/.claude/plans/goofy-marinating-hummingbird.md. + +import fcntl +import json +import logging +import os +import time + +import salt.client +import yaml + +LOG = logging.getLogger(__name__) + +PENDING_DIR = '/opt/so/state/push_pending' +LOCK_FILE = os.path.join(PENDING_DIR, '.lock') +MAX_PATHS = 20 + +PILLAR_ROOT = '/opt/so/saltstack/local/pillar/' +MINIONS_PREFIX = PILLAR_ROOT + 'minions/' + +# The pillar_push_map.yaml is shipped via salt:// but the reactor runs on the +# master, which mounts the default saltstack tree at this path. +PUSH_MAP_PATH = '/opt/so/saltstack/default/salt/reactor/pillar_push_map.yaml' + +_PUSH_MAP_CACHE = {'mtime': 0, 'data': None} + + +def _load_push_map(): + try: + st = os.stat(PUSH_MAP_PATH) + except OSError: + LOG.warning('push_pillar: %s not found', PUSH_MAP_PATH) + return {} + if _PUSH_MAP_CACHE['mtime'] != st.st_mtime: + try: + with open(PUSH_MAP_PATH, 'r') as f: + _PUSH_MAP_CACHE['data'] = yaml.safe_load(f) or {} + except Exception: + LOG.exception('push_pillar: failed to load %s', PUSH_MAP_PATH) + _PUSH_MAP_CACHE['data'] = {} + _PUSH_MAP_CACHE['mtime'] = st.st_mtime + return _PUSH_MAP_CACHE['data'] or {} + + +def _push_enabled(): + try: + caller = salt.client.Caller() + return bool(caller.cmd('pillar.get', 'global:push:enabled', True)) + except Exception: + LOG.exception('push_pillar: pillar.get global:push:enabled failed, assuming enabled') + return True + + +def _write_intent(key, actions, path): + now = time.time() + try: + os.makedirs(PENDING_DIR, exist_ok=True) + except OSError: + LOG.exception('push_pillar: cannot create %s', PENDING_DIR) + return + + intent_path = os.path.join(PENDING_DIR, '{}.json'.format(key)) + lock_fd = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR, 0o644) + try: + fcntl.flock(lock_fd, fcntl.LOCK_EX) + + intent = {} + if os.path.exists(intent_path): + try: + with open(intent_path, 'r') as f: + intent = json.load(f) + except (IOError, ValueError): + intent = {} + + intent.setdefault('first_touch', now) + intent['last_touch'] = now + intent['actions'] = actions + paths = intent.get('paths', []) + if path and path not in paths: + paths.append(path) + paths = paths[-MAX_PATHS:] + intent['paths'] = paths + + tmp_path = intent_path + '.tmp' + with open(tmp_path, 'w') as f: + json.dump(intent, f) + os.rename(tmp_path, intent_path) + except Exception: + LOG.exception('push_pillar: failed to write intent %s', intent_path) + finally: + try: + fcntl.flock(lock_fd, fcntl.LOCK_UN) + finally: + os.close(lock_fd) + + +def _minion_id_from_path(path): + # path is e.g. /opt/so/saltstack/local/pillar/minions/sensor1.sls + # or /opt/so/saltstack/local/pillar/minions/adv_sensor1.sls + filename = os.path.basename(path) + if not filename.endswith('.sls'): + return None + stem = filename[:-4] + if stem.startswith('adv_'): + stem = stem[4:] + return stem or None + + +def _app_from_path(path): + # path is e.g. /opt/so/saltstack/local/pillar/zeek/soc_zeek.sls -> 'zeek' + remainder = path[len(PILLAR_ROOT):] + if '/' not in remainder: + return None + return remainder.split('/', 1)[0] or None + + +def run(): + if not _push_enabled(): + LOG.info('push_pillar: push disabled, skipping') + return {} + + path = data.get('data', {}).get('path', '') # noqa: F821 -- data provided by reactor + if not path or not path.startswith(PILLAR_ROOT): + LOG.debug('push_pillar: ignoring path outside pillar root: %s', path) + return {} + + # Branch A: per-minion override + if path.startswith(MINIONS_PREFIX): + minion_id = _minion_id_from_path(path) + if not minion_id: + LOG.debug('push_pillar: ignoring non-sls path under minions/: %s', path) + return {} + actions = [{'highstate': True, 'tgt': minion_id, 'tgt_type': 'glob'}] + _write_intent('minion_{}'.format(minion_id), actions, path) + LOG.info('push_pillar: per-minion intent updated for %s (path=%s)', minion_id, path) + return {} + + # Branch B: shared app pillar -> allowlist lookup + app = _app_from_path(path) + if not app: + LOG.debug('push_pillar: ignoring path with no app segment: %s', path) + return {} + + push_map = _load_push_map() + entry = push_map.get(app) + if not entry: + LOG.warning( + 'push_pillar: pillar dir "%s" is not in pillar_push_map.yaml; ' + 'change will be picked up at the next scheduled highstate (path=%s)', + app, path, + ) + return {} + + actions = list(entry) # copy to avoid mutating the cache + _write_intent('pillar_{}'.format(app), actions, path) + LOG.info('push_pillar: app intent updated for %s (path=%s)', app, path) + return {} diff --git a/salt/reactor/push_strelka.sls b/salt/reactor/push_strelka.sls new file mode 100644 index 000000000..b3ed30ed7 --- /dev/null +++ b/salt/reactor/push_strelka.sls @@ -0,0 +1,96 @@ +#!py + +# Reactor invoked by the inotify beacon on rule file changes under +# /opt/so/saltstack/local/salt/strelka/rules/compiled/. +# +# Writes (or updates) a push intent at /opt/so/state/push_pending/rules_strelka.json +# and returns {}. The so-push-drainer schedule picks up ready intents, dedupes +# across pending files, and dispatches orch.push_batch. Reactors never dispatch +# directly -- see plan /home/mreeves/.claude/plans/goofy-marinating-hummingbird.md. + +import fcntl +import json +import logging +import os +import time + +import salt.client + +LOG = logging.getLogger(__name__) + +PENDING_DIR = '/opt/so/state/push_pending' +LOCK_FILE = os.path.join(PENDING_DIR, '.lock') +MAX_PATHS = 20 + +# Mirrors GLOBALS.sensor_roles in salt/vars/globals.map.jinja. Sensor-side +# strelka runs on exactly these four roles; so-import gets strelka.manager +# instead, which is not fired on pillar changes. +SENSOR_ROLES = ['so-eval', 'so-heavynode', 'so-sensor', 'so-standalone'] + + +def _sensor_compound(): + return ' or '.join('G@role:{}'.format(r) for r in SENSOR_ROLES) + + +def _push_enabled(): + try: + caller = salt.client.Caller() + return bool(caller.cmd('pillar.get', 'global:push:enabled', True)) + except Exception: + LOG.exception('push_strelka: pillar.get global:push:enabled failed, assuming enabled') + return True + + +def _write_intent(key, actions, path): + now = time.time() + try: + os.makedirs(PENDING_DIR, exist_ok=True) + except OSError: + LOG.exception('push_strelka: cannot create %s', PENDING_DIR) + return + + intent_path = os.path.join(PENDING_DIR, '{}.json'.format(key)) + lock_fd = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR, 0o644) + try: + fcntl.flock(lock_fd, fcntl.LOCK_EX) + + intent = {} + if os.path.exists(intent_path): + try: + with open(intent_path, 'r') as f: + intent = json.load(f) + except (IOError, ValueError): + intent = {} + + intent.setdefault('first_touch', now) + intent['last_touch'] = now + intent['actions'] = actions + paths = intent.get('paths', []) + if path and path not in paths: + paths.append(path) + paths = paths[-MAX_PATHS:] + intent['paths'] = paths + + tmp_path = intent_path + '.tmp' + with open(tmp_path, 'w') as f: + json.dump(intent, f) + os.rename(tmp_path, intent_path) + except Exception: + LOG.exception('push_strelka: failed to write intent %s', intent_path) + finally: + try: + fcntl.flock(lock_fd, fcntl.LOCK_UN) + finally: + os.close(lock_fd) + + +def run(): + if not _push_enabled(): + LOG.info('push_strelka: push disabled, skipping') + return {} + + path = data.get('data', {}).get('path', '') # noqa: F821 -- data provided by reactor + actions = [{'state': 'strelka', 'tgt': _sensor_compound()}] + _write_intent('rules_strelka', actions, path) + LOG.info('push_strelka: intent updated for path=%s', path) + return {} diff --git a/salt/reactor/push_suricata.sls b/salt/reactor/push_suricata.sls new file mode 100644 index 000000000..c9c6eee92 --- /dev/null +++ b/salt/reactor/push_suricata.sls @@ -0,0 +1,95 @@ +#!py + +# Reactor invoked by the inotify beacon on rule file changes under +# /opt/so/saltstack/local/salt/suricata/rules/. +# +# Writes (or updates) a push intent at /opt/so/state/push_pending/rules_suricata.json +# and returns {}. The so-push-drainer schedule picks up ready intents, dedupes +# across pending files, and dispatches orch.push_batch. Reactors never dispatch +# directly -- see plan /home/mreeves/.claude/plans/goofy-marinating-hummingbird.md. + +import fcntl +import json +import logging +import os +import time + +import salt.client + +LOG = logging.getLogger(__name__) + +PENDING_DIR = '/opt/so/state/push_pending' +LOCK_FILE = os.path.join(PENDING_DIR, '.lock') +MAX_PATHS = 20 + +# Mirrors GLOBALS.sensor_roles in salt/vars/globals.map.jinja. Suricata also +# runs on so-import per salt/top.sls, so that role is appended below. +SENSOR_ROLES = ['so-eval', 'so-heavynode', 'so-sensor', 'so-standalone'] + + +def _sensor_compound_plus_import(): + return ' or '.join('G@role:{}'.format(r) for r in SENSOR_ROLES) + ' or G@role:so-import' + + +def _push_enabled(): + try: + caller = salt.client.Caller() + return bool(caller.cmd('pillar.get', 'global:push:enabled', True)) + except Exception: + LOG.exception('push_suricata: pillar.get global:push:enabled failed, assuming enabled') + return True + + +def _write_intent(key, actions, path): + now = time.time() + try: + os.makedirs(PENDING_DIR, exist_ok=True) + except OSError: + LOG.exception('push_suricata: cannot create %s', PENDING_DIR) + return + + intent_path = os.path.join(PENDING_DIR, '{}.json'.format(key)) + lock_fd = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR, 0o644) + try: + fcntl.flock(lock_fd, fcntl.LOCK_EX) + + intent = {} + if os.path.exists(intent_path): + try: + with open(intent_path, 'r') as f: + intent = json.load(f) + except (IOError, ValueError): + intent = {} + + intent.setdefault('first_touch', now) + intent['last_touch'] = now + intent['actions'] = actions + paths = intent.get('paths', []) + if path and path not in paths: + paths.append(path) + paths = paths[-MAX_PATHS:] + intent['paths'] = paths + + tmp_path = intent_path + '.tmp' + with open(tmp_path, 'w') as f: + json.dump(intent, f) + os.rename(tmp_path, intent_path) + except Exception: + LOG.exception('push_suricata: failed to write intent %s', intent_path) + finally: + try: + fcntl.flock(lock_fd, fcntl.LOCK_UN) + finally: + os.close(lock_fd) + + +def run(): + if not _push_enabled(): + LOG.info('push_suricata: push disabled, skipping') + return {} + + path = data.get('data', {}).get('path', '') # noqa: F821 -- data provided by reactor + actions = [{'state': 'suricata', 'tgt': _sensor_compound_plus_import()}] + _write_intent('rules_suricata', actions, path) + LOG.info('push_suricata: intent updated for path=%s', path) + return {} diff --git a/salt/redis/enabled.sls b/salt/redis/enabled.sls index 4cea8d028..1c9d1afa0 100644 --- a/salt/redis/enabled.sls +++ b/salt/redis/enabled.sls @@ -17,6 +17,7 @@ include: so-redis: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-redis:{{ GLOBALS.so_version }} + - restart_policy: unless-stopped - hostname: so-redis - user: socore - networks: diff --git a/salt/registry/enabled.sls b/salt/registry/enabled.sls index fc5021910..ab09d1f99 100644 --- a/salt/registry/enabled.sls +++ b/salt/registry/enabled.sls @@ -21,6 +21,9 @@ so-dockerregistry: - networks: - sobridge: - ipv4_address: {{ DOCKERMERGED.containers['so-dockerregistry'].ip }} + # Intentionally `always` (not unless-stopped) -- registry is critical infra + # and must come back up even if it was manually stopped. Do not homogenize + # to unless-stopped; see the container auto-restart section of the plan. - restart_policy: always - port_bindings: {% for BINDING in DOCKERMERGED.containers['so-dockerregistry'].port_bindings %} diff --git a/salt/salt/beacons.sls b/salt/salt/beacons.sls index df6198d01..f3e519edd 100644 --- a/salt/salt/beacons.sls +++ b/salt/salt/beacons.sls @@ -1,3 +1,5 @@ +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'global/map.jinja' import GLOBALMERGED %} {% set CHECKS = salt['pillar.get']('healthcheck:checks', {}) %} {% set ENABLED = salt['pillar.get']('healthcheck:enabled', False) %} {% set SCHEDULE = salt['pillar.get']('healthcheck:schedule', 30) %} @@ -14,12 +16,28 @@ salt_beacons: - defaults: CHECKS: {{ CHECKS }} SCHEDULE: {{ SCHEDULE }} - - watch_in: + - watch_in: - service: salt_minion_service {% else %} salt_beacons: file.absent: - name: /etc/salt/minion.d/beacons.conf - - watch_in: + - watch_in: + - service: salt_minion_service +{% endif %} + +{% if GLOBALS.is_manager and GLOBALMERGED.push.enabled %} +salt_beacons_pushstate: + file.managed: + - name: /etc/salt/minion.d/beacons_pushstate.conf + - source: salt://salt/files/beacons_pushstate.conf.jinja + - template: jinja + - watch_in: + - service: salt_minion_service +{% else %} +salt_beacons_pushstate: + file.absent: + - name: /etc/salt/minion.d/beacons_pushstate.conf + - watch_in: - service: salt_minion_service {% endif %} diff --git a/salt/salt/files/beacons_pushstate.conf.jinja b/salt/salt/files/beacons_pushstate.conf.jinja new file mode 100644 index 000000000..8d3f05864 --- /dev/null +++ b/salt/salt/files/beacons_pushstate.conf.jinja @@ -0,0 +1,26 @@ +beacons: + inotify: + - disable_during_state_run: True + - coalesce: True + - files: + /opt/so/saltstack/local/salt/suricata/rules/: + mask: + - close_write + - moved_to + - delete + recurse: True + auto_add: True + /opt/so/saltstack/local/salt/strelka/rules/compiled/: + mask: + - close_write + - moved_to + - delete + recurse: True + auto_add: True + /opt/so/saltstack/local/pillar/: + mask: + - close_write + - moved_to + - delete + recurse: True + auto_add: True diff --git a/salt/salt/files/reactor_pushstate.conf b/salt/salt/files/reactor_pushstate.conf new file mode 100644 index 000000000..7d3a5a0d7 --- /dev/null +++ b/salt/salt/files/reactor_pushstate.conf @@ -0,0 +1,7 @@ +reactor: + - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/suricata/rules/': + - salt://reactor/push_suricata.sls + - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/strelka/rules/compiled/': + - salt://reactor/push_strelka.sls + - 'salt/beacon/*/inotify//opt/so/saltstack/local/pillar/': + - salt://reactor/push_pillar.sls diff --git a/salt/salt/master.sls b/salt/salt/master.sls index 895150cd7..23af779dd 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -10,6 +10,7 @@ # software that is protected by the license key." {% from 'allowed_states.map.jinja' import allowed_states %} +{% from 'global/map.jinja' import GLOBALMERGED %} {% if sls in allowed_states %} include: @@ -62,6 +63,22 @@ engines_config: - name: /etc/salt/master.d/engines.conf - source: salt://salt/files/engines.conf +{% if GLOBALMERGED.push.enabled %} +reactor_pushstate_config: + file.managed: + - name: /etc/salt/master.d/reactor_pushstate.conf + - source: salt://salt/files/reactor_pushstate.conf + - watch_in: + - service: salt_master_service + - order: last +{% else %} +reactor_pushstate_config: + file.absent: + - name: /etc/salt/master.d/reactor_pushstate.conf + - watch_in: + - service: salt_master_service +{% endif %} + # update the bootstrap script when used for salt-cloud salt_bootstrap_cloud: file.managed: diff --git a/salt/salt/minion.defaults.yaml b/salt/salt/minion.defaults.yaml index 11f3dab41..b258e1c1e 100644 --- a/salt/salt/minion.defaults.yaml +++ b/salt/salt/minion.defaults.yaml @@ -2,4 +2,3 @@ salt: minion: version: '3006.19' - check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default diff --git a/salt/schedule.sls b/salt/schedule.sls index c3b5d85ae..014d24460 100644 --- a/salt/schedule.sls +++ b/salt/schedule.sls @@ -1,10 +1,26 @@ -{% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'global/map.jinja' import GLOBALMERGED %} highstate_schedule: schedule.present: - function: state.highstate - - minutes: 15 + - hours: {{ GLOBALMERGED.push.highstate_interval_hours }} - maxrunning: 1 {% if not GLOBALS.is_manager %} - - splay: 120 + - splay: 1800 +{% endif %} + +{% if GLOBALS.is_manager and GLOBALMERGED.push.enabled %} +push_drain_schedule: + schedule.present: + - function: cmd.run + - job_args: + - /usr/sbin/so-push-drainer + - seconds: {{ GLOBALMERGED.push.drain_interval }} + - maxrunning: 1 + - return_job: False +{% elif GLOBALS.is_manager %} +push_drain_schedule: + schedule.absent: + - name: push_drain_schedule {% endif %} diff --git a/salt/sensoroni/enabled.sls b/salt/sensoroni/enabled.sls index 7790574f6..db4b91dd0 100644 --- a/salt/sensoroni/enabled.sls +++ b/salt/sensoroni/enabled.sls @@ -14,6 +14,7 @@ include: so-sensoroni: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-soc:{{ GLOBALS.so_version }} + - restart_policy: unless-stopped - network_mode: host - binds: - /nsm/import:/nsm/import:rw diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 1805bacaf..1c736eddd 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -18,6 +18,7 @@ include: so-soc: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-soc:{{ GLOBALS.so_version }} + - restart_policy: unless-stopped - hostname: soc - name: so-soc - networks: diff --git a/salt/strelka/backend/enabled.sls b/salt/strelka/backend/enabled.sls index ca3f0e6dc..8c71bdf68 100644 --- a/salt/strelka/backend/enabled.sls +++ b/salt/strelka/backend/enabled.sls @@ -47,6 +47,10 @@ strelka_backend: - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }} {% endfor %} {% endif %} + # Intentionally `on-failure` (not unless-stopped) -- strelka backend shuts + # down cleanly during rule reloads and we do not want those clean exits to + # trigger an auto-restart. Do not homogenize; see the container + # auto-restart section of the plan. - restart_policy: on-failure - watch: - file: strelkasensorcompiledrules diff --git a/salt/strelka/coordinator/enabled.sls b/salt/strelka/coordinator/enabled.sls index 6756a324c..8393d7e68 100644 --- a/salt/strelka/coordinator/enabled.sls +++ b/salt/strelka/coordinator/enabled.sls @@ -15,6 +15,7 @@ include: strelka_coordinator: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-redis:{{ GLOBALS.so_version }} + - restart_policy: unless-stopped - name: so-strelka-coordinator - networks: - sobridge: diff --git a/salt/strelka/filestream/enabled.sls b/salt/strelka/filestream/enabled.sls index b03faf4b1..c01171565 100644 --- a/salt/strelka/filestream/enabled.sls +++ b/salt/strelka/filestream/enabled.sls @@ -15,6 +15,7 @@ include: strelka_filestream: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-strelka-manager:{{ GLOBALS.so_version }} + - restart_policy: unless-stopped - binds: - /opt/so/conf/strelka/filestream/:/etc/strelka/:ro - /nsm/strelka:/nsm/strelka diff --git a/salt/strelka/frontend/enabled.sls b/salt/strelka/frontend/enabled.sls index 58e703898..f2d0eecd1 100644 --- a/salt/strelka/frontend/enabled.sls +++ b/salt/strelka/frontend/enabled.sls @@ -15,6 +15,7 @@ include: strelka_frontend: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-strelka-manager:{{ GLOBALS.so_version }} + - restart_policy: unless-stopped - binds: - /opt/so/conf/strelka/frontend/:/etc/strelka/:ro - /nsm/strelka/log/:/var/log/strelka/:rw diff --git a/salt/strelka/gatekeeper/enabled.sls b/salt/strelka/gatekeeper/enabled.sls index 45b6e467e..19d74a6d8 100644 --- a/salt/strelka/gatekeeper/enabled.sls +++ b/salt/strelka/gatekeeper/enabled.sls @@ -15,6 +15,7 @@ include: strelka_gatekeeper: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-redis:{{ GLOBALS.so_version }} + - restart_policy: unless-stopped - name: so-strelka-gatekeeper - networks: - sobridge: diff --git a/salt/strelka/manager/enabled.sls b/salt/strelka/manager/enabled.sls index 7c73452d8..272767928 100644 --- a/salt/strelka/manager/enabled.sls +++ b/salt/strelka/manager/enabled.sls @@ -15,6 +15,7 @@ include: strelka_manager: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-strelka-manager:{{ GLOBALS.so_version }} + - restart_policy: unless-stopped - binds: - /opt/so/conf/strelka/manager/:/etc/strelka/:ro {% if DOCKERMERGED.containers['so-strelka-manager'].custom_bind_mounts %} diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index d9d7f32ae..10c04e5b9 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -18,6 +18,7 @@ so-suricata: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-suricata:{{ GLOBALS.so_version }} - privileged: True + - restart_policy: unless-stopped - environment: - INTERFACE={{ GLOBALS.sensor.interface }} {% if DOCKERMERGED.containers['so-suricata'].extra_env %} diff --git a/salt/tcpreplay/init.sls b/salt/tcpreplay/init.sls index 7d739d00c..f5b1b05d9 100644 --- a/salt/tcpreplay/init.sls +++ b/salt/tcpreplay/init.sls @@ -7,6 +7,7 @@ so-tcpreplay: docker_container.running: - network_mode: "host" - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-tcpreplay:{{ GLOBALS.so_version }} + - restart_policy: unless-stopped - name: so-tcpreplay - user: root - interactive: True diff --git a/salt/telegraf/enabled.sls b/salt/telegraf/enabled.sls index fc9946149..6a063e08b 100644 --- a/salt/telegraf/enabled.sls +++ b/salt/telegraf/enabled.sls @@ -18,6 +18,7 @@ include: so-telegraf: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-telegraf:{{ GLOBALS.so_version }} + - restart_policy: unless-stopped - user: 939 - group_add: 939,920 - environment: diff --git a/salt/zeek/enabled.sls b/salt/zeek/enabled.sls index ee78714c8..355e555b3 100644 --- a/salt/zeek/enabled.sls +++ b/salt/zeek/enabled.sls @@ -18,6 +18,7 @@ so-zeek: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-zeek:{{ GLOBALS.so_version }} - start: True - privileged: True + - restart_policy: unless-stopped {% if DOCKERMERGED.containers['so-zeek'].ulimits %} - ulimits: {% for ULIMIT in DOCKERMERGED.containers['so-zeek'].ulimits %} From 0a8f2e01a0435eb03ba31504fab1bf73a3d5e088 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 29 Apr 2026 16:41:56 -0400 Subject: [PATCH 003/219] install pyinotify --- salt/salt/master.sls | 1 + salt/salt/master/pyinotify.sls | 20 ++++++++++++++++++ .../pyinotify/pyinotify-0.9.6.tar.gz | Bin 0 -> 60998 bytes 3 files changed, 21 insertions(+) create mode 100644 salt/salt/master/pyinotify.sls create mode 100644 salt/salt/module_packages/pyinotify/pyinotify-0.9.6.tar.gz diff --git a/salt/salt/master.sls b/salt/salt/master.sls index 23af779dd..b33d3631d 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -15,6 +15,7 @@ include: - salt.minion + - salt.master.pyinotify {% if 'vrt' in salt['pillar.get']('features', []) %} - salt.cloud - salt.cloud.reactor_config_hypervisor diff --git a/salt/salt/master/pyinotify.sls b/salt/salt/master/pyinotify.sls new file mode 100644 index 000000000..8aa2f1d53 --- /dev/null +++ b/salt/salt/master/pyinotify.sls @@ -0,0 +1,20 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +pyinotify_module_package: + file.recurse: + - name: /opt/so/conf/salt/module_packages/pyinotify + - source: salt://salt/module_packages/pyinotify + - clean: True + - makedirs: True + +pyinotify_python_module_install: + cmd.run: + - name: /opt/saltstack/salt/bin/python3.10 -m pip install pyinotify --no-index --find-links=/opt/so/conf/salt/module_packages/pyinotify/ --upgrade + - onchanges: + - file: pyinotify_module_package + - failhard: True + - watch_in: + - service: salt_minion_service diff --git a/salt/salt/module_packages/pyinotify/pyinotify-0.9.6.tar.gz b/salt/salt/module_packages/pyinotify/pyinotify-0.9.6.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..3150ad360fcd785b11c1d67e734fe906e973a782 GIT binary patch literal 60998 zcmV(vKMg;E_NYboKMvJYT;NOD>+h6?#e>?E^@Zo*@+%VvZ!To#p)cV8k-LK%k2RjeG5<6ek|2O~J z{PXs#$hQYcx*ex)1uuWAwJGckAhzTSk)&mukHTRr%1p=tQ1~dDF7o8+x)hyZSM2TV z+~3**EG{nML0FVYoQkJmUM6WGzANIv_tD@VWgJd|EWdix+JySg<9w17MS?99;yTXb z!9rZ+A+#3t#3+wrk&VRgI?S))9yT4O3o(uJ0xD#KG6cx!l?cTU`T^jeY0x*22MwMe9;DDxO5{|_*%c)U+v;lYG#}jdJ{^au2 z;foV-dLf>_IRESE@yW5+IJ|)08$I#r^zz5^mzM%cyf}Pz`I9(*A`YMZB>r;x?6@aR z{`>ihlZy*+{sMrVK7D?6dIC>RpBP!^fv*rw5fBw^p(;t4k6hEGy z9iPC%$0yLc!^dYQ+!ze)=1ja%p`p56QlOB@$=+qxC&( z*D@TI`@+LL1i3XF17;AjBcf)v!o2|Rw67UBlPhrmw0euuARxb;Yhe-h& z7qC@mKnMaEAatxdg&!@|&daozO>rZ~(Q`tne}$t7ptxDL7p86ce8FwO0h$>gf}x1F z45kR2Dp5QV{XR@h+3$A%V~l#LpoDD~#=q~UVIEFYp?z^Q%X2`KUMr?}xtPYLG$O3@ zsupmZ=i)%LC7e42;(2QrJ;0@ZpCy1$MF@y9hJ{sa?gjVS8jL$Zfj2g6v(wMcR!h6tD;yGtz?8BfqVntMw$3dboP4Uo}40D;uY&=f#7COxrGQu1lPmcxKM4D z*LfV`EukJ{`W1`;3hQ5GJ~O{Y!?YZ$A27@`(*O~#8X#ewhYR&%oLya6O~K}Umz1qh z4mZFMka;}hO)OtWumUl^ji6|mOk({?{)S5t{*B^s8S3`of%=~7Z(*@Whne{eXndMy z!x&FOnJ;)|sj1N{9hO-(*4@bCXf}*ne8}WP(k$=m(Ec$L6CF$$A09tg8l;C@Sn z*Wx-XB%n@L3}(p~@BjQEgV5E6k&eF^6*jptiMxfsWcL}5&TTLF!Q zvvJ7>I-5*^rWz#W0!gPBu{x0`m-&pT(h2M|B1L4uF&0Id3<*t*qj!Rb!EPXqGg>=> zfn>_tFi*k(G{4AZIb#)^2&6ZNN=oU>xWN0^57=i(TEK)vLF?%J>GQ+OQ@&sOPtT7} zV3be7aS`i#;FP=ZI7ty1Jj==_*({CLzXj&kF%nX0!Gy$4(6QM#O4}vURU-ts;zF?Q z0Y=dqM+(9yL8=)DZPmtrJG-$yHtw47KBI!K@^A{2pgW{PAn|gzpn1u~9Lo?rKSYFK zaT@JHuRsw1)tS&W5~gzN0xC>_Oyt|wY2o_<>F^PS>q5Qk4RnH@hF|FON&0%Qd5Ax_J>J z4Gttaa7F;<2xyWLh#kp5;qC;cCUQFMAMM8O%T8a&a-XX6dfKnzvQJl-zX5G};RRPl zSKkY2;ee~!HSP76oxt9?9#zc?N)8U1E7aGcZzNssFzMw1kksbFab8y?JEf%A0V=|Ao0mGYl)~I@2vuHYM8<`Zn^;=yUT}XhEX$s>ZtQcWzh3z)fDPV zvbe@{P3OnO1rq!+d<>_&%#b2RUU!TXG4G|f;do|kd+rv#=dG%*Y%|R2u2A#q(*@p; z@lG{>`i&>onIY2^tLm~Xs;L$AH;P~oe%}Ib^Xh9$k}~IX3DIG3q18+fdyK*`)r{m;XQeX4`O+aS-@u=7NvV@^hxR?L35iUEBOO^vZs+}GfNUq(S)oR+XCc*Qw@fvinZee87Ba4VbQOp1rGCnze8 z^S;f?lg=YS&0F%1SQZe9AJf_mc#g=NLE&9?Mg=;0>R53@IWZm{%LG?{vaqh*IfpA?|c&!SKsCejAVZ*Bkgb ziHjn zs~=)}5VT&zv>VMCZyVH^0goT~O#SycBBFg^Uj^T@CQ8!5`z(mcVKJu}ku7A%fVK*H zU}6}ih$IHF$Vo6=*Hx_5&jx>kVd7VmKOw#f{+12|HAQ{0#y^RH?Q$5|0qFCl^I#Awrg9W`0JRK<4`DDw5=N(AlPr73BWhaSzR|Z9F?&@&o?d1UolHSzI#2N||v?F|jDK`_2-i~D37*tB@QY@>&B?dQspGE!JUu@?ee#n9fQ-G83IJ8E0^soS^2O=n zeepCdL)aoA+Q6k()egvFmHJ0#=NBjaS1(R4Pxi$tL@K1*9I5`$dG(<=l^Q@kJD2ZY zrgInU5|Fx51?c(nlV{d60lr8pFt|+39RKy?xQ}K?&OE<`okObk|E|onDc=C>^1K2H z_R&wZfUsJkK`&v0?q@X%T;&! zv$LO$PhaedECrIBW{eOQ-r%HFElIs8O^334Av!>|GzdlD4!DW)G#-n+;6bo^zv~|v z)mU|JoIktlKRG|6VMZD0`^ml&nl2U-pww=t;ie|)|4fr&9^*X;H`)zg;#tn@2tMV; zXy8jA;gpnU?o0QH>`~!wPkRuRA)fsA(HZQFv(smPk!#QQ0^Z$MfM0U51;#sdc%6(R zpz%aF0v;0EUM%Sv+4&6O813+|R|?T!m(>Z-tG8^nYUcOu1AhsfQB;rM;l*G2hsUtd zfOl1fx+rA8S%c{h$4G@NMEnkTJ+e)v4Yy=V`3ru}?~rM-dO9v>v++60hD10=BD@Nb z5WzX|Psv67<~)CPa`EFi9OrX9BXEyLd_lvR8OD_@F3Y-54$sc|WIYSl^5HmvgGvhx zTzwM7q>@If9%)EylOhb*uC4%+wJ3{~(cl^2gk-9Y=d}%xN6FPJCsFoRSZpB))CSUy ztcvC#k}9#F?L5mxiuTV$N=bm&F*I{7dIES8ED9u9M&+sO^=uNRTcbRIxqvAt<98)2 zGnvU0(qem@WjC{_0;nb(20o9+Kyu^oQRl<-+LN=xANogda@b1u0w}&;+UL*v`23>1 zFMiRxqpj(iwtBQLcHT)hhut0<&3N>YQ=5sKy4rhgZABcpZ4+rzSNk5-_9-K$vPH?X z-OArmWu%De2%;Jwl6GlZKz4U}Yy~lQz|??1(xq*Vm;tf3<4yq+KHl^xdZq2wy|?2} zpP_kbAY7ie{no#&g@B}vI{`$aRF`0PhwIl+IZl11BC7TGYU)cmq^3U63eGCtQ}r8( zk6Y*xHXEDQgi71({iRuagmW1PT}rJc?nxbT6T!REu{u-H6t$N;BJKr zYgM=t_v}u@jdXywymW)x%lqw)zq}q1u+Faf)kv3c4p4&8@$^Ggi^IYY1cJmYmfILi zDAo@&=|cN!3MiY6-C8#Bn#aX#T(X@ml*zMcj!*(Ag`z>BT>>|i6jIWKt`LKG7|sea zMz1WA)hN+rLtr4PLvfYB8#2YON;9#```mF)X31|{nWdCUn+d4Oq>D1Ioxn*!Vy?n$ zd}Tv&r*LdiXh;5=Apo5f)KGwSQYBp7xCZ)O??GQ&NM-KWVyXZsy1yQty*#;iJt^Ms zEU9gQ1`(Ela7R67apDEgR&m)zecN0Rc$OAg1RqW!+orbsHVLI6dG%|nzB~Mm$g^>H zHSD3}afnp7UEHVF0=#AlrKtld2-z&|miFsti%C2gpfrrJ8f+q_&k`h6l-dXl+a(Q3 zqP>Xy8AFKz3m~u13c>13&+cefk;F&bu#m>`QiwN$qmh$i=4666Z8FG#8fBbZJ5vFR zLOTv@Jvy|{c$-ACa7=4g*aHfg3THC9jpss3HbpT6!b(xbX`~n40Wesw&{r}7xFW2y zMQuhxiie>)@2>b%iB3^Lq24y^Kq!JGu?B51oV0d|O3ZOvUI8t`f%fdD@;$2TH z_QgB#UI1HRH=!ta78@5R5U-u(*KGs6jXg0x8+PeyQv%D|^^N@;RdofzOj58j(+nL` zQOqBcXjWcz#)S@7xL6c0!))BoX8nwjNomK;-~txc=%s*Yu$T?lQzVk8Z;3h(_jbPl z{U4SkN~g7Ys1P0U8%$C%5R!Gtq#0EX6*Nol_mu$Lfbo&8Eax4wii!uVrKe-vaCP6= zhm%M5HB?wbqvE*xDSZC}%0rT&#QV}{TQ3!xf31BHAO>%@ow2a8M(b1xa-@@roIZ_E z53wh1;>Cg5>9EaT@4WHGo$v%Z_Kf{@jSB*r8-sa(T?Sb?9!G+u@k*&Zf>L9-f$OKd;^^ncMAL;dk4(Q>8$LJ z$lxIWZh0J$`Co^%d@D#hxAqi$X=ZGE*FI?9vNpF(UFRWpyDfJMaJNNm&z-ivThX># zl^w@u&~gJwS2Fm9Py7Al(285M<#h8=yIZyWu@4;jlr(8D@i6f#gD+o%bHa}u#+(&T zuQ{K`9Yg5r8e z^7eyXB19uqmX1-D_|j^w9+ZqiQ6s0Q*fP*;$V&jm@7S2xE6y&PuHPj2_zbUOn+eLu z-HLaAkIOStYe7O^zR6@(x;|ZezzZ_Mak{Z7dzr^5UlApYocxN~Kkt~ECnB-tJZkUb zPIjk7qJG}q-YecI@KWx2?=`jzFNr(Kahj$bY_|KWF30EDcZjq+8_<$0iU+f+j-HtA zN=%}SJ8?d0-Oudl(tWUC)msYUJ??1s>+uo!TXHFlj1;vG?CNMXc2QC|z#u53lphpv zuB2eJTLbZ9HivI{kM}px8ACdGWF$bBi?VUNnuQb!2D#l-xo;xZU8qe>+g9NV;LlSL zk4CV+daT_iXsrB6a$rE>x1q!VNSja*$0|W|@OjMU9C|xocP4R&p=QQtnm3Y0+gd`Z zj1i>dX#$-M32h)WB}lwH?@Hnkl?c;JG9RXk#?EdYi4I!SH{;VS=9FcrrxuUDWtdys zV(=EN_UMFGdHXGvnaP6my%7ZtVl50Z4yIeeZt3Ik=qEs+p8|6(L799S^fnF0^KfBq z;hN#7CM8-$ol)DJe|rj#vJ{P7QkE{jJhrh!n+Qq9Kc;Sx5Bq+7Qk*G9%i4aZ22hL_ z!XjcK@qIo>kYJ|UQ9|h2xtW+U%kV(s2ClAUkMsgx+PwlTNtMl?Yq_Q_OAK>^=X_Tl_pF1l)po{u_#Uai0K&tUcM50LBTgw|POa*yw+s0;ef~Ijk zV$mG1P)SsHkYJ6c37cb!aX7p&Hga_W2A8mK!Y@4M;48Fu)(A3DDK2zCq`-ow=tPF{hOVWu#uqn@a1nN@AbPuuIS{V-E7KUq8`qTsaS4iUk zxP(z0n>eu_b^|pi(keYWe0n0RKpFhEX9_(ze0qBJQ@_64-GD7H=V)Kc5oA<79mK_7 zaGGr?C@gU#VHo z9`SX@@n&QPB93Ik&BfGM9Y(b_u!*Ho5$)lv>RYl`kb=7qDC?J74|nJ~=zNymq}e=` zWP80V5!tJ09QBFNJ&;AEeyC&pGWsNgvKwwqBh^OB@s86(*H=`rt#xgzkNR2ZY`m1i zqJHhXyR-A{yA5$icqcw2m>fByU=PVqsX8&Mu>@EhybexnVwSB$&=F@ig?v@G|#i5%da`f>~9V_gWQLu00 zr9dtU4RT3y(_AbHd*gJRtEa+uOCvLk9cMUI>mY~2Az*gf$et0U+GBv?i)lQ>7@FkI zh9x^>>O%2P5igqPNmA*Nqb1r^PXs}rkrCViYG%=8ubdDzW+skR;o~H>3=jU6TT(TFJ~&fUR7D&!)+p{#dwO-1$5+Y}ASy~5aV?XgPI~NbkimZ#&>^;+ zB8r2bdNou#sPI%@QGO@Cd!;9D`;zNJy*L*9g?~#Xu%p;r;&giyny^dT^w$a`1H!2U zKw$BjfSB-LW4Kpm(`EXzBQ7Ej6VL*{1B=e1-AUR7oPdW?%gR`Y;f>Mw_o7ZyzN^0o zv5WoTEXRO(DjZ_~*No*LN5ZW5@7T_3uc4lX{>Jm%f;uzZL%oZ+PV%V_UYTVfEB%+NKy=lI1ymCwoyy}HS+L9$_nKPE$U#0z{5F!ZzuO4tK{ zXz8ur0T||=oFG_OM7P^xghv7+vO-eqo-|QFYkI7oAzeDM43Egz7~=}m4i^?u&q%P; z42O@ojuhgYdh)kpcX9*Gj@v`3S2hIgqmah%*TnKOFlPF#{#2uwKzC(XJ^f0PDxXWU zx&9~_qxZMfXw4pk`tybnX@C7z)dCLXw6Hj1j5i5ro!oWJ21!!5T@u^dw~#k#S=0cl z$Rlfe+2@E!_|+a$PYt_9Y0*IHVYC8G17do5dgC8>E3q1gyEX^4D`t_w z)`aH?Mrz5JSOG%f1~<{xRBT7P+tJ>3c~{C0qg|o)yQLfOL@0ZFhVKFj(~E}2Rta!f zB!h7rv)sxT2`VM<6q>pKkZ6`UsV<0l$yuF>*Jpx}0hwQ~fk#ID1KLj2=B$P8Q+P17 z6NpQ(w{|Up?s)Yr;+YK)`~r`}PPcZ_Nc1faY*IjJr||~lUK?@9zGaltDnK45Md^dx zU%H=?G>t2G#5oyN>`^mGjmRwOPz&8=mZLWCOr3d2b=7~6&PG7HB$L?~X$@X%5^%9F zaKy3hw_5AjCu4McTvN(j5Ud)uSxoX5=b^41-lq1uqI1Hhk;v*bnx$nY8^8mjAzNC5 z>}cG5w{A}>4T<;l#-r6Xi+G)hASoLu#-!z$bO0~I8{13&qc#fMgiMDVSe7B~nPt;U zU}ARG+3+s;9k8V`U$!09SUm(O>(;T?Dnf48Vi$v+Y*a`vWVJ}DZ6VJ)#XkJE(Fn_s zhzRwLaL+jE%28t5?XJbW8{)5)P&7KNHHpQi;Z)k#5`SVSG^61Xoo|RbXK%AHYm2+} z#MnoXSlAfWwP-2B{zzuW!k1`UTGhP}kO~RbZQ{sYn}YlREkQLu*7MksTdq-0{9^1b z{DyvAwt|=Ew_HDaz4RhRF}gW0PWD&Q#0LJIR#dU8AHAZmmb!a-U5UQ%r`(Da0`wlC z4zi|cUL!|*2kkUorK`lv|m{K1uY+SjSat<5G^+I$F2l9zO*i z>2j1(7@`+zGfyma&yZ@yv3_jAoF=Hl}3^5w<8moE$G845Q_K?6p~JC+5) zl;E1Ovwa~tPfq{)sb!$SVs#mwR{UodEtlpL!wvOLg%_T~B%ja#&Vfu-z~{&=LpQF_ zzl@|rjInYnn}l!bWO{;?=`f)^9Xh99hmjR54!1CGF!Wt#mme~87$*5}Hjy!i%=N&w zIhdM+G2%XZYZjGVPFMq9SeLW86Ll$orb9~LdBsyFLl}^9RpuAl23BVFZhKV7yakqR zhcb^IkZsls&ptUvIrfaCov$SoED4;%Iq1sp8_)Z_WJJnEM^(P|BuoRHe7rc&sk~?v82|Pd8f3QM(?j zQw-b*8^WPi_|)p;LF&pkdNH&0s+W3B*aa^62aKRPk$N`<9w|(euZGl<1zy{$ zy>ICfsysh(ot$uEV#-wX*Ups*-3dAXEG*D zN-LIJna_%HIi$q|Ry{z2A-F^&8YlZGvVpf`L))CwlB_!$;SJ()ju(=``HX!F8%~dj zRZ@$><_1n7XofePAr;2c1eg&gFW6|h;2j+R|7a?@oa3bc~SV4YGR1ykN2>-bFd=uT-8>j+C;-4MHImjLE30?QWfxuo^&nX)5WP?umriQKYoKt9ioK-#eaM2DXt-Ug04 zoahi7Q{N`(Y}3B;d>e^X#fivn*mj@V>YcK3wnyP4886!0(Mp&43S&BREMLN*hsknr ztSgT5@vUmOlZ+}Y)YY$)QE-sTZI^O$AjgWFe%CThl=GlBns-aFY6p=8O?nzRYcRIZ zA$!11WOjPa7~KxHZ^XH}5a19B$6y1eA@%7g?VSZIKT#6?oykxs_a^oGjX74&Q zbTyL2R9e}e%WbTk4&+hq`KLu`v0F>zhjM;8EQqITuIn2ls)kj2e#dCM`_d29I)>yr zJRQeqj>`J!P+%gcG36HUT@*Z1GBoHU+YRU7-o%SJI!gew{AxC#oyXFQ21s&`Yb@iB zP4*peZnJTv-UU;4KG`cvf>!HpMYpn2u&mk_p1X)#LBeqM{xE(r-N373F7(N?TyO+T zim4-ARDg(9M~iG-*frR3%7r5g7~8UZler=hAC4r_O-$+$2fMfR)@8Md4d`#!1@fc; zX!`uxc;3G&NLZLEH)ai}ZuzGaWCkHOOq4nnmfKvI7L%luo6=kcnlx7~I*;Qya#?uQ zJ$-p`$!V;-&e%JVa8~u?%1x%WvD)6wFlec*)xG6DH3_!X@Nz23Y}(f-NTopb?Qy0S zA67@yY@$bxLcZ!&vN94$nKW8lVpi=Ltob-eYn6SfHc*0!QNTVLG)bE{em5N_!vslT z4~PmcmdTdUtSOV(wxnfd&vJlbVX^COz<@OAqfjZzMTcZ`L4A=)r?YBe0uK;oPa9X# zr`AorHP^n+gcNc(HEJC$`rg*BS$(rP(ir7gec9xajG+?SttH-A#`9a-#6I&R1 zT|jFhyiHLv1k|9PCRd*mB_^=b^_U18)oeOwbw{3mR*XVrTS>Punxz3wjkB2{>j^%E z}kK_bO5evp^P9RAS7 z+l3<7dTxw%K0xe20(^24?saZ)pP=Jxq)m>AZgZZin&Xz(h%`~YEH_@KZ3S7igV49T ztK=ffw*IjkO2s`@-!SHED4hdE0K(wACJ?pn`%7C~UFiNrs`i1}BuGIiycXQf%~Kpn z4sb*X=N_(m%5cM^?~8*Pjk#kS{{za>|5Ab!ZIJvCX|@oZCtbM_f!W|O&BwiV=HA^et{(njGhZuDlr z2s5?LnfT&>61?(QJ@F;K#^)b#=rDSCTr7*3hjVQYNDI&`>}jzK|1-~wo^dO;6i;in zv5cE@%Df$oXT^2L2(6BgS~(q|Dao?)hXF*1XVk`2nlR!>Lc}7cUd$?1uNEqbwuM`D z7Q_`N2tBNuj1hUVT}hM8j#&{eEoL`ZGQ~2nE`Hp0TWi9$v|wrXS2@`iu&I}#xg#J0 z&_@aJ5c-0TL)e!&P%<##6kxZ2fvUhz#!s_OAmY#E=bJ@RWcSem5$2K%5%Hf+w_2<$ zICgK%3(TgB?&|2W8{+N?&fr1kqwaV)@vxEKQ?d6eMj?9P<qA=PlC1iJePdfMQ2uJ)h}L$+zDUBW3IGET>6?Pv1ibT@q0 zuO&mtwVm+Y3gUO-!H&0$oLYqE`3a>FEMhA)J=u#s*b(0ySn%#X*jWm(eJ7fvcOnFY zz5%M@HUR2rFn8cr*KG&kx-EVC;NHrnuCX<*A%xU35AM)|F7?1`Qx$o1uvTLcwCA-& z)ttuYNmq8G>u!VB?Gr9{nMey=s|~Xq#%`)_KFKwad1?QyACY$~y-PY6toC44tlIc( zcMV&OmEmT)Zn)X+DBA3!nx;Q4Vr;w3mL}t+(O;w4WU6j`p!k-=Nd;zLT3GjCmF`mj z_VqXE@W=;V(bw1{GQ_%2QZ$c3I?@J#Q$6Ckm;$Gx8Z-`1&W?v-OebPGYb%1L^`0}4j-rOJ znotb}8Xw%dzaBoVh{{6$x>GSg+GaT?ocB|nCKHq0#J|Dt7(lU&vk)i%>=2%AV|hE8 zl!PAcZ0$WL;9tOfcjVzh#y+hsh!W@LjidjM%MZ=bI43 zvJ<|l<9JoQd%gPxzS10ZZvm|!kz5CGD$dh83NO=M#ApiA_E83K8GN=U;k1Jz?yZji zYq5c(lvO~DHA1cF*c-JG*7RE$UF}ncSL5T{ECmV!S`fW@e-26-3gUl{d&r)P&?#8=RMI9-Nf91w6xK0+XL zT{4F!ig6rImt4Pe5A0Yw5W|;iJn@~iE=?Qt^5!+fy~cTPMkgX?lK-o!Hm)4%1XX_Et0L$QbH#?W_=c@;x9gx#`# zTfn@d=1T~pB_^h$sAGrGdycq4%B@)qbd_KbRL3KQVc`@&NrsIRgAXbaGg&HNKyltt zed?)d&w)2MeP(xp<1Qpq4$X#o@F7{K8tV|gVQ12=`;xG696hTKE=G_JL}zQaC$@I& zhAgd|SBK8W-^uR@UOQ8C!FJewNKK5GAT8)otq;>EQ#yGvpGBeXHH`V-#{{fdCl2Ff zccJl`_8RjqQW}r@$mVe>#h0WTr~&P1#_SHCp{tT1guHHA_p~|qUKvAMS$N7XBd^B; zbpWj%9A&^26+@G2aayybhpLcowX#WgMrO1y=J7qg$7b4_QUhS{IDx38sM}mc%p%g# zmial+I#vsZ57{y}w1SXJ>k-S5pi#qgm@ZXp>u@BV;-g;l%m+pZgz9`(aF;AI2&rr- zD=43wRylKWX4F&iDQI%2O$24Xe*lIEj&>9$C>QfgnNOy-_BYk;B#zMpR=m zq7S{@r2o22>#|h_4lXQI2s#`ZFxf+3vw0KS;t6Qat{c!j2D` zE?S-jBq^jBmdY9sRIG~31jDdthMCGlyF)8xhO{Cpr9O)5s));UHgUTrn9j6%EY*It zjBqhLpjmDvGXstTJN)zf~dWYa^HPV=HmZae+Ic{sa(V}X5 zS0&CW!TWTaMs)=RnpHD&>QuG1CbhbZrmV(4e`cnv%DC0V_jVkSu91MmZRM_WZFsj$ zcLCb6V{NDRt<~IXLqs`!hdXzAs~TFulu6N$%AgcZ`hU*2rKnfQ7wj`G*we0!~fG zkh`+~@B+S76*uzQ^=TZPa(F}>Ax@`tSLu{?1atR|_&1k=H^-l7IP3H`=3Wl=k;lMg|U8|&R(IW32#;dCfaqgg8PomJVpTh%t%eNdSpxMpDERuVLFP)O2g zj$`+BsghQ@94qT*aDHd$G#uW*SvfsDZGU)1p8PcqZ6E*nhDY(H+cX3x(-2X@toW<+ z=}ttNFWu|4y2m{c-^j__cYeVa4bAIdZ7{}$DeTtfM(WoSOgwt>!dGYvX`Bv|x7~_0 zwBI3PThScCksX3{al9CGgbzik)!#lIITuDz$AZ*dRY!Au^?h5Ci)pR;ym$vSt5>zF zTI15mFfB{m;NVDjhu3GoB)sI|ESF}>*&YG70vOQ=(?)HDWlnioa#&z8%hqWaOox#F z%vcg^`Tol=BEb?O&Yo8v;i;%F2kz`ov*|b8_W~`J*G1W8gXoMAD-sVRNltx@8qxOEorDlp!pZ#l=z5 zeF_S3oz2m+2@VA8Frcx~1_V<{(mBt+;HA4WrvK4b6%;Z)|2667}U~3g{BabD+{(blJg; zLJ3hSM~*D{wI#8_d={-W@3~cifqp*0m}LxiUXK;!{0kj}>A zp7dIGpT%v=HW3Z-Fs*KUKCBj{<;2p0m1nmuIW|rI{{j^&#+wPt&|#^AsfvHIK;}gu_d@ zM^tJ&3d{`D6#UNWem(CWCAm!4)z%Z@@+kwAT-L<@Qja>;%yp3sZtLlDVdy^iF~qp_ zo@wM8(Z-2ycU!e%__Ndwa4uB|YD`vjMMstL&^=5b4)2Hqf9XT8Pn(6H-oZd~$l5H} zK(7dp?p5GcvMy|K$&=lBiR~{L5Ngqg8nW0$J6VEn#?#0qfh_?QTMg1h5TBezVFnVm|`QjL@Jl&=d@6M)s9j;h0?}x|I`O3h9 z2fYTfQQz{@Yk_L+R$AaQX0-qhKClI&6vhYEL%b6yYciDHSzxNz zhlU`h(kZAX8cmRtsukE1%*iLIG<|=md_iA-y(}t?XLhH>%rp;_Nk>c)BwLLor7y&n zqk=sHgjrEqY0PDUb=x|u(VEGM$0yl!7EBh}qDY42R1u+28K@)66}HUE=)@~(JULhU z;)zV@tDsTqCR3y1TSzfn9Xs60-^sfvWh$d#k|LR%6_vHHS*^=TCSOTpB~L>G5zboAs^Dj|>N-{wys<{G?VjuUOoMaG z7rM)2=6rwFU*t|nTP0enn*K|?7-V4{ohmhQ9U*^#VT)u^2}#iunCGM9(=tQL+s;vz zmU%YbI;s)KS#*PT;GhUVg#|XK;bz<=*c3-d{At0pXV8ZLZK)6aUYbyqficrw82Nc; z$ZyI*q7rT5>Wb1a40xU`I8-~K88&ODK;{@hj}19wGT^KXH2FD-R%Osp#tyTzVp7{cqSwE+Y=a$s*(GN!;b?uK$}+&cEoEbrvl*2sEo7a&K8Nj&xT76MB@*^qDACHwgWb*Rwkdm_~}N+qE^xn$hAr+{_Q z)X2P$K~9n8lne`{=gAA#lOxfg(sbu_jCrgahed)hs`VWe{PBIKdUm)!6S5&Lnw_dz zzL`*sbJahSQ`JRmkNk@tRGs157l)%pha;rW|K~UziNj?8Yn_h%bsUZ?9~=7T_8v!w ze9bLup`DhE&h0DM_3CweU?_g3imX(Lpps3srGbIT)-+weNBSnL>5zIC-0tqO5X~~Q zrguk;ya^~=3Y{RcxRVPJhg~CK5ba+j<*0FuVvi%#4Z*i{eWIk{X2mgTZ91A(MbR`s zN8@o0?59KNv;1FK!bbT#lI02)?EY^_Q4 z%1Nt<0&$D?yKm~mo{hnjaTqYYD7TSwe4Jr)@kt!&kZqE87%W(?uf#QGs*jLYkrXQw zqmH|WK~bexQOjBNCKzZn#+y?`Pkn_#k9Hx$0awu3rf~LuQe5B(fIA3)QQo7J zm_q4tRjPdvzl(=6SC-4t6%%N!3blsSTPkB&L;A6z!lYj)^yp(uJ(6VPAH-KT0ds=6VlW0a z%p!P5L25}Vu(z2?#7Fd;1PI`P^_pk-&F>tjqeuDJ(s&FFOxI#RrJE0p#0UW>Ix}HI zEB~%$Fm2^W+iI{aHw9<>HDTEGGyyIDmU0q=Q|4`kVG%Z6>}&J>rU0xIyI;+28IQ*p5m%+wV$qRG(S?o8r9WnD=j`xY z0Nr#4Zf2S&W)+~$G!;73l$EJShTyeWzLvOUc^XEUizS+H0hbj?Mz?TC zhCcbNk0@97HNVvzvc^$fa&oKi);!8aR0p>la`HLzf=F`^Qs(|kQm<<=u4TYTmQLCS z<>zTmI{WX#PAynBWGyEOY~ptz8pT2}n(4kqAkzhH6w*?0ky)YQx%;w4ix1_l?yHr9 z8}js==}M2uQw3$ifgI!MIl0$YCCn=o(vD!86w@jLfpr#}u|kJiCTlS`bzCR!1`V3( zWmbz#Dr}s;acrroKS~{3IgClxr&Dqu=hl5|O+sJmvJP~ce*n}gcL1jSmb351*Ko$5 z4t0R;Yb)4Fdv*f053aPZoPTRX>pekQO3GO?Ut%_bM-9zOIRb%8B7G_Z>yyg0E52MI zQ&&Yzflb{d*)q)C(idrxZKL{boT#h};-m2RXTaj#`qkmeVq1=FIkJQf5?$)+)Mozc`s9&+pVi*Z&K zeRT8cVEWIVOxxL$c|P&C+#L8FmG^yE-8vW6cKmKQo<(t=gP~w?%(&TY%({nOpPby7 zeLu}ERL%5+3v(@Jvi>L6fUxnY)bw+C`pBqJ|hhi%2R~)63F`b;`eR1fRWWXb3*MWg-P&RL6F0U9@ zX(&$U3l#>*Sw4=I0=&1zfjynIMV4GA@TJuonTvqt-!&|$tVY8h-k`S9Sz}B;Vv{!# zZ`PY}`5etZ=3X2vN?d+Gd&5gch(3)6v;(TRM_P(h;)+oLwGbGcL4I`$xFf(g=FS7R zFwC*##FG^0gQ0uAMg)DwA4~;Mxu{F!QjMWvpA~01(Y;5=z{0QF7?pfgKy(Znvrb5K zFVyiAx95zy*J&TLKLCOqdig3Yu^)`}KG^l&kA~CjcE!&P(1_}TGR8Yy<9X>ui*ExG z(H1Cm?6mvs?i;A>zSI9}X4#pe?^eyxr;Hnq-MdX|PEQ}Tm&KPp;Un>c54m$y zI8^Ezp9bYDQ{q-h`4~!^~=YPY9jO0gFNgjHO(B5uCzg z>=X6MLf{Gx!E8Vdc11kcnrv&?{5HWUw(st?YRu7bG54+1fuR^<*XQIFGO5ch$Tb{1_F-MOkOw z$@O$Z405ZS;-Kg`@<74MXQxN!$0v?YRMS*6=BD6b+MPq;J;!sUnkE@TJ8v(^fP;gg z^m9q7VGy5!4{P;}o8mH0CTP$jt%@*jV?v^J*n4GMOhX_aTMau8Sx zVp$r)eDGK2M%iq4mq+xfnobv`S``U!MUEdm8BzxABX-+KE!JQo6j}9xdrf{s2_c?qV0$rV zWSnv<(v-C+s>ML#df>QNov$52ZV}?nIeYd|+pmvt!V_G-65fh95ni3pY7UbuX?S+m z){&_^?Tuq%TALh#g4f{<#KEQbp=5~&L^hSKWNRc)m=ssQ>Q18NTLWWf-7&0|tW4r4`r2uf@cumY7zC1i1K8PmrRCH(D?dnPH>4>u9R5$GVPQl=hiffy%RZTyE7uY>IOpjQDyq7fJhLb0w2AWz z_8mjo!GS79NUZfF32@55A1CBw0%LswCo^KAz`2xkpx*%(g^u8Y&(I%AISdSC$evtA zM5J}sjwB|MNUd$keQLy1tI(3Uz{2<^=U^QalO4ZsBEi1RW z9si(`cED>QP`^0^E)0Iukifw9i~7!Ygm+`ASiU4E6|_Ai^zpw_v};Bs&pC!r5qBDT zj5)+88f?cVyNw~~nx1XmIHeZ_XM|^tihPM2X`w}0P?)9;3De1BQQZ_>a(qM1)QZmI zZ=k~WgAes#v90FFF498qwm%+wLEzm17n7mT$rFiJf!rL5L={9JL)ybDfJzVKTcfg3&#; zYLisdGg+KhA(Fx3m=@L4MjG?oi}rLbnPKmW8qYkvrDRTyrD1c4!qgN}+NFH7b9>^Wc)Z>? zUhpyGZo^domyR0~>$41~!iPiu&G25&qc^Rphw({6;A?aXCcdJhBUHf&2a0c&-lW+) ztqzH$u9WOG&lO7i9zB<0*hqR(<+)h4opD=)7O^T5a_g9w&D2_Tcb~4Vq4uja3xP+_K?V0R=I^)5Pq#&gc~;B>j$sE2SZ?q8+}((97bV()+_Zfnv`!HN7y0;UciFHwsi8=)5H@WRY{cUKn`g03yc*a$?MoQmrqJ%yg(9Yt|v*4 z{*3$DAI#8B(%|f{(4j2?CkYU22#N}U1Q+pyc}&mAH<$fbVz!R0WPcJd!?lU(s4rjrJ|$KlWlFKkJdUL zMv>CK`-FS}FEhO1>PE1kT6FGlUm13V9o(XG3w3bk< zr3s=+F|bbDm7%xq56o-R?bW2GJL_x&VrJgy*1U%Hl&fp0PI;;hcyi%tIZze^2L|kc zyQ&9zc^lXqH6aI0tAF5ahAMx7H-X9=Ru|eF^G4jkXi-$6vVG?c-n$xS180{;89lQ$ zAx10VhqV(%5Zr~Y5v_CC*I4e2l^kh}gQ5-BhB7o&Yssw2;8n7}>gV5QNnBD+p}NZ6 z{4X(4K2)X(ouaF5HBI!q$IidUH|+Oo2#EJMp!XE7t>t?AlCzd{FF+##s%U$azgM&H z-UM>8o#tUyW_f|8We8oohu`n(V;-GG?D@;xq$t2JP-J^$yaGwq%+lF zl5HUcX<}U@ti+aV&CJzopw*C7($kgnOA9U9o%jfyAEdr;5L-4|4Pl^6JK~U7S6b78 zVs-p|MigC#v}{uCc-^c#Ag7ny))+crkR8jU$4N=(U;8S4oSD)J>m+lWpnrDU1xw73 z+aiZXv6Z!Out+lAg^+fn^@dU^qQelCDe?e&mQc)4kr`O3L;>}ZaKdV(t zMk&!jS}$$aiv&mLi#vw_EG8JFv5+yZUqx*9} zJ2o{D*g6=%0QX*mVTe?b<`4y0G#gvS2T96Q8H!UX>!%y9>ls~@6k>#T5XsXDMPb?s zFB7*-GzxV&+qvUn1IVpUjRez$-!s|xI+lIFN=4|d1ykiq!@J6b z+`L{ezEW~sm7B9xOff^KCS56#M|LY>z8QUBJ@K=S4Y&#pjRuiXo5B!L3YMxoVqMX> z=Eq5zt?>Q|DG3k{hnkhB_LuTe6$yr$R@2WAP>Di^3blMn$yYRA*mbt7>7NZ~5aNZqbjC7i(d`vq2nF0)^oL=}D3BPhpr z@eALo$b$qy@T(KZ+}k9z^8Sh*qfWhH#~DQamZk4#w~kCb-Q)2%o;X=>DZL-fu`&PC zOd*`zPK~aexcGMl-znRU_}w&2D_SD5FZ5a{KMG0{?^`pibu4_%^+S#Zde-)pQiBoAokF}iKDZP0r110Sbvv(Se}0Sc zuGT_zm%^f)a!FIdr+V2hr4U#;LyZCJt4g)1$qU6)$9-Dg$a3JD;w&yPQb)KDTYKaq zU!YS5W?M}t=J3jUcT<`-N}XA|nFw?b)7D;13dFu67J=V@DCm&0XkxrX9{fYB;tS5D z_S`6grF|)55R8RE@fp;uWbqZqM|6PSrKnh4HjZIcBe#tqX^4WC)Ve0tYy9i@s!(i~ ze=qwEb#7^g!BW@$YEbfitApYjZAE%ao{%WZv?VAm0ErZk_rq@tZi=1xQf92q0knTG;+^v9k@uXo=BjuVPS^^UCW zvdsuSOx|@~WAi-(@}@pT9u4(?^M;TQXa_ruwJDJ&4Fn}^>!Nq84Xt&>qubYodatq4 z8`GnPi=KC&oD^Fq%PHYJ&83t!vn=#BqR|@Zs1b)Y;Io(11262ZP1uX~^XNSb<)mVvPUr!mx8A3Wwz`w9v?= z;aMEEhA2!Kpem*fw``rf=);9t#m7svG}GIbB!Zi#Tz30+6mC`WQ?4W4l5(8W&mu%x zM~37}FRBR5m1IbfQo(4gCX1^pG;-E#9vN$R+e*0#L(c6QHtZ7#HR$ePv*Yf^fW_aM{C z46c^zlZJ!#)w!?ZwiAzs{9QPYTDN1X)?lR?-AmTsn*6~MlCNf%jQSs>{ATIIM|yy| zx^ssC?>BSc#BYVHsBTWiq4;xm*(8h34{y(-E1?YZulK=B_iAQ)O(wf>cYMeWp_5af zVFw-+q%dKS)m@~0xADX%MBR|L)a2WpM6_Uy&{k$XwOz%1Agj7j>rUQk+EI>sj%WSA zw5zf~++s@zcZl6p+`of4m1P2aU*&EE|DqPQAxevUZ`7i-oHp)JE*dm4*Z8Z$JzB%{ zk_y&3Mv|$19}zV7t$|CP_Lo?8$RZ>=buCDv)rNF0q}zfaIV^)j_exe~0NQZa;xovq zu3*cF>tw(xg<*#7pi&SpC2K?{0r#t8ZyRuIUxP@yt^$FCH*7meMseJ8VV8B;DXy)`?48gf_t)iLGD zg{isC^nXzN6}1e{C9P4v5a#qYu0<8GYSU!<`qY`q>*ZL=T1(Rk$nA9Jmn)?nGx8cNwJv7@f#O#ON`VbPQlxesxq5~9_@A2xnyDI)V=G-6v(Z4HlWp6TZHh19bStamToxzitO;F z@&t2@de>$CoD(m3zZzD$kz-rUttn1TwWPT;+3)pPG~`@9j4%7+aO#_`#}H}!vYBm7 zU*Uf|nyWen*V!kuy?t)O{xDvq1(dpzGE3Vx z7TT4h>^FYIwq0-|7$vOZE~%y9+V02D*H!=_!f%wQm6xBQgv#t1d2)3X=f+OROEF9d zajdKHI%ILHc{C@x=>$_M3>p1m&-@$8oxcDbk}QwOl2b$BRL=8^#mJN?1DdkVb~(=! zMkam)oCLKi-rjy2=G*x!-5zCG0CU|Y{ML0`MxnCdpG*x^)V~UyWBW04&kIj z^`Q|MoBe6-yK?O>Eo7q^I8J0$W)#Zui4K9}-2S@wfoQjDxiG%qpTbV!kS4?IS_R@D zmkrY)=cs#~P>*VQu_+FVVm2XyiQOi2$Pi@`LJQWgk6VonRN}o=$_|h(hLb6W#G@lq z<^EL|L3-U2Z7kdF*4q|ZQ&puO7?l4;ISO|wo$OVcTrnvdPw}>LJr$`wo(EKJOBMMBhUAekd0vF`a|*-A&be+_ z9_g@K)CB}BR_0Y@&cSyKqY2_y7+{URz!ClBS8J{lM71heN{_-|CWBcSyY6*lVQd7p zMO~CUoMUv(E-Lg|$La}5nsh60YT?9w19iWtPyS?8YO^g-o9bA5&W>g&$Keup&rJcq zi(}wTn|!X-Y#=h>#;kVi2&T2|JRwmBTY0bD&UBW;*J?XD7HC5=gs9eTWdMC@J3&O8 zc%zjzj%dfO(T2S~IT7)4LH!P7;|M$81R{kE;~ff&=)$Mv2%C)~=cQM;TYH3tT=Bz> zk~Irx?e&Y=2K%D@^UmG7uXiSXpl_JkaCoB&+^s2r^P=DHR^MOcah#eGd({$)7-gQO z$lbbbzzXXE_o@ZtM$<*^SBnf6q1BHEHI3jZssc45nPHSfUEp3_FO$?vR!tWYa+B00 z9!}c7S`o6G<@W=iQGj^b+!wGFDL>35iUW(b4 z%z56Dj!m&00S&f%p1t)U5E1rVXE)mk9tOKinU5o7Mh^9^fRZn2TH;ouBX41+3Ihfv zL93+#rmpB?czK@DHeEn;!tqRUBF7_uQUpM4ecdMN%z1oxcGf@n>&dgri?)iY=66_m z5~HCnT?r;sj-E&oHAwR=8qS~p%+Z32X*^8GFo~sb()l)mj*heGRHivKO+Gq+R=*A2 zxhJ-eG@sRT7@pPlo-jXnUO7$|;j)CC!GUP$xc3{sccOK|G_@-k5^EagsyzevH9wI` zTz)K$&YwO%ygYq;dUksGQ~&AtF;m3V$4Yvlw=$3yQ6JBhzlk3k*Ga9NB=DC-K0bc4R^i#xbk#A|j$F5a*g{;vGolb`7undgOo zqqNTk+#yg0^l#THMx;6e?rdzm#puvmTQHbGR>bgw!(bh3P)wA52{XUZMEb#Mmj@fB z>UBJx9&G%Tf71!wXl^*?hPA%vLz=cKHVEubLk+f{xQ`jJ0!Y8AWCZ6jz`XI8C}Em+3TMnhxT!F;aQc90o?Tr_Z$+J%l5(@_jF~(cD9=bxK*bq=Nc+?8Pr?Ll7O4et`NP&% zf8am8I9ofxzXuPtr$7;A>E1R<(ox;?Dcab9zXuQQ;pgtd`#bh0{oQ@I_toy+y?b}> z-Mja27s~DJJh=PdE3xxACSZor1tGo)qe+s^07EvETlHR!MSuR2{GB}i>G=HUAf7IO zvuGV(T;ebIszPqH;QPK9q8Zl~miW{E>Ts2=*xI@-Cu8`y%ERgP0g5xW-iobE{I3qf z>6)XD24*b{Wxw^Od?L2;ktr(QI!%odAVmW4b4P5EoT&Zv|7`>OT>RYn)1TBMfIO^y znzhBxU4BRuAlzo};-P?sfB9AX{2#5azWgmY|2kXmCp!NRAKq^~|9D(>_wL`lxASoK z!Na?7{&#nF?|voje>wlZ|N56RyxqgGov!)Q;{RR1_8$Jfzq|JZ|9_UhP0ef>10R$Y zt7UI(wl>jbc%M?xEzdwVPi~U!OH)w_nstFzi}+A;DYzUdv2eu0%1159Hkb<9Bt^V; zpGu<@#RUT&qs$j%!bJS_qs&@wYT8Cfl3pg=wM?L_0jo?jp2Z8_D;q~Bi-g;9e?t}t zRD7yd9Y`L*a*$*v+5nPMG3SbxRHN#)EDqWX(cz_K)oyDZaRwvD><&c`HVMwGK+C0u zyI(HOf2sQ)c`TWD@b;L0t^5D(-p)M_|KEMM^Y9D)|19|b&)c&i-yS6CcAUNyy!?dp zUy>3Kz6cR`yMU#S@w;-j2mkEhKdBf_BC7XzoDFXreWi@KN`gf;=yXzJlRaq!C}t@0 zxL^mQBU3D$K~N}F%toW+op=we-MkOPi&-k(L!gs$TQeZ?|R}n&A8YL9`uZw-CUkVY@?qnikh31uetN|6$+i!zUU@0 z@e=HWBf=q|~y<8UbH- zr9e;WZ%}d@=2ve~hjZ88qmLdb$u7D7Y#d327-hsIsr$Z|Bpg{u8Res!$9Yn~SXfDikuhKWU|~Y;nbC52 z4gRhMUqSpWZREUk_kJNL|H$+oo{$eE{MMBJKDhV5qyP5q-n;vS{`)-qzoK&luCzn9 zNjnv(Itgky`b}|ar-daOAt$$9M|tNdQ%o{EP-aElV7W}7MGh|PQp_fe4seXw(|f~HqpWz;;SPr_m86iUb_(-m)SD;!1*FI{)BAy7vpg_i{1kUE6l z=VQ57p8sJM!pAWF2pq8D{NH_WcgN@dcJJN$a{fQ_`By8gcVYc`cYH==;6FH*N_vP9 z?q?~VOnRXe)X4MdCHH8e>yGM%@-5$aKH!(IKsXJ z1sTj>SM=I>i?Epw+DhK%DI((a8I>bVH3Zm0{42Lq(_SDLwmSYG3#l1=6 z`HsCgf%RjrEsnOj^KN%%hrS|7d0^y|^pl24>A+|z6bnJWw&^8MBK`gmcsp@Fi_H%I)sDVdomD0D3D!Ngb}B` zOUDZWrXErNA=sfD>qo+2x(~o~kWQob(>PB6a|dgvNsOf;8koWrnUeLb>>S1M8dYhv z%DK4>P565$Syt>?72_tM$Ko4?X(t-ov}QU+BNj@mHbnSo_P19wZrVN98ee zdX9XH^nqfIrB6piS;%$NCh>ft{tFXgk!KR;1jZ9|dQQhoJ3dh!dX5B^7t;6`9rP+0ToY=~{ZFNhiZnZtDz9j3DR0Dg2!oL;xR8GQy9Dktwts6A1mpplg z#UdR#XoUgw#4RH0YTLuQ>An?$;l|mQP7g{ZpSRGDoRcqtu`lvJiU0jTFrS?Nhwr;S z{=fV1i~Y~%=KrPSukfNo6|4?Lb*tp(BTm?n5~J_!kK&qhfKn7bHYpDp)s<2!d8t4= zAss=J((=~Yx=ry4nfSLD@X%z30%8~L5el}YX~P?|G*vWR;42@0F5A8pPEWMO)+2!? z{$-rDt9_HQg~pzpPE3y)g7cVgJ|xS`0`U{j+QTrlz;JXFT#X4;mcHOf{l}AOx$ygg zfkqY;C`vGjV0tR$FG+A}s>919hp}=R&By}7vJrFv@XV)rRr9B=irt3cwk#MJoGnk- zt=?|AvMBQTzR+uH^fd;tl#e5-Fwc}uJ?XV)VIDSLvdUDUtE8B``^8ty$b%n|AqelT=qX` z`!fiO>sE_GjUxWu6?@{YxF_z52jZdlR{Xoz-DyP_KCaAWaB7`>`5*9@#I{?&lW*MH zLve1U0NVSH{KpZxdVQh6{@C=Nvi~lxk@<+D+Y16#(SN%SeErXZ2YdIwod3^5|6R&; z{bC3HPi6<++N}7(*UKZk1twX9YdY=-09Zh98xwITq(ezVwSdEYlV|o9!NNZ|{*9TvV8&=OiaZ z^-ZqRP3Sgmdru$+6jdEQ5S8tc0=11Xg7aWj%G^D9NI59EJqo9vU2)Gz3);$kboW}} zJNn#;vMuPx7G*bWziE=QGUCpubDT57O5R|mr&Z>Q{i;3e@+RvA$Mx2;R@>bcH`_3i z6tljE)3R@w){+q`{1eR4O#2ptanjJ<+v_RqCw~3iI;F3tPv49G;rKrafdolE$^vw? z{nvfp{_~6f|L4d5ywb@(VsjYUh0&U^6qxPN3+1-*=s@fR_Z8-*aFba!UdvbID4V4v z;BgkBc^bnDict$#oXV^p9+rJOu$GSaLsa*8R%B_sr1UojVwV-0SxVYLt;_7GUU*$R zrdVYa{vVh(l2Wmef*JF8A zYg?IO8qU)~+7hdUCd(^x@L-bwKZjO1$XIdOLNda!Hb+}LKc%#GQu%uE+ejkO zNlFn?jx0N*BT-D48Kt0fAJ{s4@*d-$%Hu^hz}};rJ1nJX&1A}XMJUNhA^tYwv;qK) zEqZQZmB&j7572n$ZI&Q$g&2dJUJ!!*v`y=CFn0^dr7aOV&6tXbOQS@aARUu6BeSCkX-)D_N9@(Z;~XzK1xdhbDQ5%e z$qch?hlM(~crCn@-EF36P@AL38tjm_ z+(^f&SjVVxfS=Shj5V4~l+lHjg#_X(xrwnEbdXF@yOER#$i9mG{}=xMbJ2gOF7)#DsWqQ2i0C_$acifT z6brMb&I&djz5+Z<;dcSMCaE&o1gwFmAqw*d-LYqY#ak$5j*&-Gb#g=wETIjrFgFgl zI#2LMi)2_|Qb6Z%Fi|Ocq-8555Jme>*&n# zKcAcbqid+JqA%Bb05SJ=8ew|8avn5Kxxv3*{j(qx39G?2ZKOdZtkJ+{Dkd zB7!0rqo&8oY{rQa`h@BZ>gok)Tmbg#PibyCwx2r(GNroRp1_czJqe0`wO0ECP#9k9 zTBtLsAx?La(rKz0FsfPcO``X3N@tXAZV(nQZO#l9PTMl#Z|egh1?X*6_yL2fyxVsB z){z4m0(;8U{a4gZtxf-Z#0{`g|8u{h|Jm8w`GWsHGyaz=jUuK=!P@^pj0fH%BQY03 zR@_=$vsymW+T7d}Pnj#;-KQ8l+$BwUQm>F9L+)zKtiMT=__r)cJMiNQ$hUcfF~{2$ z`@$^Q*lAui@Pv}+bu>)n_$|HETPUTgDu--(l2kTCHO{7m{*e!Nf~Y}pvkG;23I6b> zJ41V-bMp6D@-`gfRoXaP=rzH!HKv>Ab-O|)d-VV+P&ywy1xlN$mR9isEj+ zS!<`9p9%jveoDWY@fYnL%o_Z~ z80M~$NuQx7QSLI#X#fJ6doRGM39R>l-HfoiG2bXjF96&gu)X!y!x#39Q08Z zL0vO{yRA=|1I9{L!{x*`!p%hE7FUUAf1y+fE^OO#>kdGe@iET3e4Yxms{%O68{^g!-ri1omPMb8cCptqRde zk&bW&(b)=95`s=u+UE#kC+4BGbH~X|EZTAz+kqG6lc%(1HSvhbN(ZB?+og#-XrIqe zSV|}#+G725`!@*tx!m@mE<)#smBsrqcdT|?|8j$W4*idI{`oEZzxVF%?Nt2#9(=j~ ze|P;4T5|BRtAv)J;{vf2P~$C-)qLrN61ok@jKB$7q84BY6urnCz(Nl|{Wn|zz< z&~jtArAQUT*DI`sj{d_}2!h5pz_oDHISTYfwuB~WDw zEuD+KPcRpItL9>F>0IntbFpX5#hx=4ivCzSA7t`wO^7ssZ_Te!j*ipATUXO7mheH& z+3rS9Y9n{MKHaq2P-joq*{jsqb06qmh+TqW9LLj+$`;DzRDX#VgDlLWQ#7{BXVbEp zL3GzQPqnAdc`<wf@K5|Fbkfv%ud_|F`qt{@sfHZ}*G+|7W`Yx4zy& z!%vheidi|@`WBy5)qM4){Aiqvg3g#@>u<`y^fME=;^!IK_qAJgqGNjT|Nnpbq1>$n-rU?jGObowo2AMxDWW8&it^fp7n2+Ptkqh=zkRg2vPOF9Cp3^j zD#?Dq@r;7Hud#5WBiZK(3Fx2yY#eQW>Nakj|8MufU3mQ^|KIO`|8f0~fLy`PYYEGe`G{*KVU*qVy%Xm>l>+kPDc!J9x} zzmBrB%oE@e*~aY>*ubz5e}S830nF|Vetr+!)NC>i;%J5@$I#d_7Z0O(A&`nl`fCF< z<9@%gx(31VoE#sRNR6Ju5c`+$!IQ^pnp;F4`x;^;quovVSZI~*=xt}9XKxE zkT20@hh)>uZ9He|zCv-3$V@Wlf_+4GY8nJ;di^r}!~h8-!;fo6Dvqb9DLD-vzsmd1R?Fq~ve3CgY$SiEVd7gTp? zs%AYDa{sVcRj-M)YHCklcsBBNVL>bxJ>d^5YDcvT3mxX0ora^kFFvhG;9LavPn4xM#FYxUl5gU^B>N&i@~K@4nu)mF0=n zzs09O%i|%LqM(Z%JIxcNEz5GEZrk!$a=G)leE5MRD4|ROEPynviZkysFL1u|e?QH< z#aXv~#Rf?^PO5uOR254kHuin5y{^B-^ASg{D-033x?-0x^!w*+gxTU@&=uyBquq+D zq=WwLJ%gt*Mc*Y77t>0sYz71TVlZG&RP0PD=6k9PE45bWd0Jk{0-?T%Gs%(t%FRcO zL&w>~Ws9cSEE8*vDMe*)Tq8u6=0k-NE$rkIik6CrVm;i2{~zqHqgc?B4 z#}DHJfkqhjqwiln`*4WR2uRw>*c99yryt}DQczfm1h`V=^Qm-Hlyp*U37a{oX!mdz zAM74JSl7j>99e!G_EhzuvQ<&O7vyW+i6;f_>dwfJRLgk^i0hN2EN`aKlk6js6rh3l zF&^W`N2?5__v>OwK;($NfO@J?F%y!zt}95Wc2kvBg41gW;rbOiqo(fQVe|dy_Ey<{ zKz2dWQ^nVPpw)AdoUZ5dgm&y~^xzxd5@KA`jtP8$o<7gs!>OM}-!H2KNC$Sdz-P1M z>XH5{&eMhI-Qn8a*}G!j-WAewk@WY-qCzu_Sr(Plr_TB2b>UFO^9zb@lh9I`3bu{n zSSK(v*fc;8lW62##?OHSWeFzuff3>&&G?IWMPEF+f=xf&a*vFzQ%ptXfRW7MdcmY* zIOD%!(^?el+hMEYf}~bVvW6YFnij($o~B>3CmO5*ZTb{6MT-LaVTb{qK%%KH9CIA1 zF2io>9f1J=K#hlR;Vp#S)440}MWJbZRlpddH?XzQ8o3*l@rwU`G+d^OVuk#m>|!`z z2f+4ivx^@e?kJ8)yf8O1{zuiy|0|z#=K+v^tk%rBwC{l&GmybHvgmN9m zA*Ubb2`3ZmREC{jtyVGX2#K2?7o)vtHr(U;ewX@ly_ik)vI2cN+~01j2hGH(GUJ$3 zdtaIHavS$|?`_$}{@VHU?>u0CF>s&QQ;LZx5Y2egdsnGs2aW!=Oo4^1knyKrlPD{I zVe4A`u8TRN>jIdzVs33+)Rti)u4z$1RiX7(B{2WS)tVCcZ?Kf=&;CDe(Vxbnq~f&) zM0y55dbZt|c&XbrdZlHNB(NEP@1?iK$R&=9gtOJwcd@8?>LidD#cis?zPZvTr;lHM z4;zJjGk|sc^XaR{a0S{k`e5r7zJz~k5(+jW9Adnt{Mie8tC?oO{)es z{O*Tdv|Da*OTFP$aMJp1xoNz3diAk~OIbs;+_KJrEs^!2!NpDG$Y@jzwcvcsX{V`y zO)c2(ooWWfteIaojc$vMs~*2fR(4mTS_d~&^goDbS$k+e-)w$SF}bMne(~(bCugsE z1ZIBz;&Slx{MobfH#BeP_;q~oZ-e6}Pd3-dI6$(wdy%)>r9S%Go=O%-H&#AN{)#ws zDf!CjBYQ7!nf&|VcUuo1MN~^dVx;arpAIfgpFJfOc$!>!bobuw;XgMuX>PBtH*o_2 z!Gi9s|OFZn?HN@is`tDjN*b3 z0AxzHSxdTb2gLVV3Odmy;cyK=j4N;b08CEwdRihF{R=oCX$j;EId6PvoDMVKy7S#J zR7A{B0#akzwFbL?xXnE6*XQYQ_waD5dD=y95h3imGuCYoW7<3)f$37dtW3Cz73T-A zdhHjU^QJw{sx%N-QGX_PJq!V}ejqMW z=d%uu9=!C|n+Yb%q+Z2L!Op(2`+inSitH_rz+=dZSUOZWQ0HoZRFG)8lWC~0 zEdScR%fPH(fA`itTyq=M=h_v6G#oE!iuekRR%22}V+8RZL(sFu1d24RWuwHUn<`1^ zW}DVSY8C$;RY-XF8Fp26g&zGDt{r+CC;9tIM4+eT{~rHupzYDivyyy`W|H2LB+laCK(8CXBQ~R z;^fWJ*^mGxQCqCnm?zkt*;mQ9sHVvZy@W^C#VSkQcN>mH&-qSEZlfB13iGyIVXJ2Q}dmOS^oJ zy+KmF8VHFj7oT1r+!bE6#vzjTl?Y;XDO2RuBRp6$!mWBtJ_f~Tv@D@_qg46g?0=ld zk1`^Vz|uH2|L%KUT+@D zBHd^}*D($znM1Fadj^ORxC+K8f%x4U@(o(Ht zxkYupiTC@h-G-l5>#OK2S3U<*Xne2zhJmKuI;{1ZBwEhYK( z#Li!Eze_coh^wIyk=WPHWpFsk;H#*eRRiDy+tCnX4WU>YKdFbTj%F`9T|gx$ZBui{ zQ%*vgD;G3XGc;4)aNy*;Oh07BvT~jL7@}O!n6H#qt|=kDPh8GY>|nER&0$*j)wGBK3?5uh{){%T2Hlx`~<1q=VD-De?N8t6rVpeuuCA5Vjzt^BxRQ1 zU^30%_gE;X90HD{)paVA$}UnB>WRyWkEgijnD@bb+mD=p4|>{_m$iMmgZKeXgZL8p z%+MnmnswQ#p$t&kGmG58rFHAU0{Csk&&=A|_x4xLEA&Mt`H)N#<;+P~^1eeS_exnd zRhNzlZeLO3A*VWmp)`RRV%n#{FcPEi&MAxKZ?t^!=k1 ztiAD-WW4AE9l|DPg+Mo6j5x#>T218PL43HLJfPd^Buy5WLxX9+OV5qP_^|LKT@=_i z+AFwA6n)s8Sw5K*pY(Qr_n+T=6<=kOh>jLIKT448#jSl|*!YHXh%VvT6!pT5WBv@1 zh+|g}-C2laLK6;40x#_T@}JfxUmN&gA7pOt$1n5(X48J$`t$|50<}aLh8#QW9)gE? zeW8diA7IStOO#=-uE!sG^?ZtQY|4%V^F|bH@dLHkP5HHb!Fka!!)yDprqGvMBP&ua zj^l2qFHSg}Ur0uzRd(S@xR&NZUvxx*M6zynRSbkGH;16)U!`VuHa3<(u5wfK%N zssb4i7jk^0Oo=T=^l9^mf?ycEq&MvLO<%%=1T8~ROhP$YM={qH`t)TkwJ>9$9Z|@8 zr|Pfsak3qp7*E#m0HXSohhZP{86{$ZLNp0dDicq}Np#RxNCKJ`3^t2woQG12rg=82 zrlUns-soI8E?}Ah!CVv=_k-P-7bq{VL$`vNLZ;9b1SmUDEntaR6Yz8MS(n&bg74}N6B=R+;B|P6{Zg&K2kb}m~_U@!_(~=g<@LL zr`boeqTrxW3eo<~IOitA5loU;ibzXLxH5^(Wac4QC3^}@t@dE3AEl$~BH9@yelo7e+#!p9c9kQhchup8VB@Z?4RryIQIPlNj9Dh0(3fdGrepxiE3WRO07e_- zO>5d(!P${9wh>2Vb)BN(A3emy}(nxwE4 zu&Z$)QDQUV2QMy9cmjXUBYmu?rcRZxjMAl(x=JPAEl4`#q@G5WERBt+Q_raGl)_79 zRBdrXH>zs_AR-NkstM1U8k|*>Gl0PZ)(Yg5y38UmR=-iPU=X3k>4$7Y9(4Gk0-sV8 zEr=3lFleO|?gbRnFkB7QNP2;Pq_-z3F- zy-A9D3YBWLkGSRvG9OM>U4-ijy{2t4$!=n!{88E^^9?VRsh?4bNHXFBNe1|d8) zy%zF97;K&?xF?F5Q{N(wi(igkuf+HS zC5&r>1vrT~X&li>1o<@CLY%J=W@54|i44l7BIj}>qI@%m_tU}xh|O_tCR4l$;^>EB zm99`#m*sMc(dDQ+2Xf}3MI?PJvj!7RPZq+`P+5)}r{%QewrNB~>9hI`Lj1 zs2sl6n-omDR_vJal1a#O3w44(7Kc?nayx^ll88j-gsffmdQW?s-gfj1t~NP|RHNTN#I zbQq74P0UpaGhXq5s+L3=CQD?lY3H>5$St9|{3`LQfIY%{Ul=f(Pqn@7@O^f?ODspE zIC>6S7-P43SQDJA7EeNGEmq^c>R$5#tj20qxC>^MIqi#Lg*&x{(KAImoVOS;^!lcb zbt}>hY-(5(_3LbmOdW0Ma7 zJ5tutX65d^_}+T90C5t?D=|ke;tI}aHd1GBU9|_Fr!yAAM$A4rmSw%Mp|zKe0=`b{ zn_j(6jBl9$RY0o0O5Vr(M(d_YzCaIzk1g>)sDwnyM^3Ff6yiN5lUagct8jS^5r@3> zyx_IvG}>a$oJ?RZ^X_(i02GwP+KG8l@P(n%@2a;=l#xrM(TGNI@8S$%aoDML%h~7< z>qusrH7T(69Ju|Da(CHfpJXu7l5aEw2t}}Q@koHfxP-|^>W>g@Oi_87?Bg63R3iFAgw%Ld?#`Bx$lsi2o1_?4V@=)kXi%zsvY(#FkaXjDO%08P+g$&K^z#BBf zES;@VBbUl;RcG)PtAd0#nRDf!N&QwHnk>-4G|f=jOWqSW3td4!(ZS$5}1wLE0`i11ww*0 zQn>sR=J@lgDPOIj3kN5JC)25uRSDICGIpyu!CMT*#iE|M23SecSBb*W9u=&A#>N{3Mn4KF$uls<(@}sm8S}RhdGDp_eb1)i+yuNC~Z{8MUDJ_^dQTs(5dx z{fqPp7lgNUN(Tf52h6L^Y68*KROv{I`@8i`PTQnmn^*cvkpvvR!lKHkbSZJyvd|z- z1f(0|iF20CpZK1&kj~V`p_(xIMsb#Kwopk|OlVaMBcH}svDuSy{L5x0SgDtcdXuI^ z*@*?wxg~K0_^}OZ*H%&4no&aysIu(xefyQ6%ka2Tm=_;31Tqfsk~bLnQ>bgHVhTUD z1PJ8%b$WTOs3#1`xJ*`*r+k%=mS$XnILihwXQ&7Zcd83sUD$Kk8hlK2mntcq%J{EmT(s@Meq^FL{r0-LOHf z6Cp%tZseUsVWqrBU8~YD!IdgKvftA{)v=m>B+R64P2@xu(chL?N>NCB=cx&ocgeq! zHCIeiDZMlWjo=e!?CD21=;XJgcBmZYVVb8C7F!p1IeZ|UHD33yN2&k(D&fRA6KRR?L!hSn0`_E}Yo}w(oq2#7qauR_b-FMWt@yVxA9` zThc;veITcynjiU%(RGo5SdDttj1&>?9jO=jI&5hL{f1#5=QluldT^1aQ{V?v>RFW{ zN)nK33L{844+iQ!QIN{nD#6r(xG+}za;Zrj1PwVo6pP(MW)IflRBa$Y=A zf6()%$!M96i|8={TP8rJ>GLDt8jBo_rc?@w19Sce{oWSSbWs#^Wzl$_mjCg;#{coZ zD2NKmBk=+OH5EEL( z^;LW&UwJ)i2!L(~e{TJd=4BRLF0YGe()!opI**>FOCa-$6xQRf@cGdsU6$ELs{3Ez zElS(*81Ie+Jm=5xkUl@EkgHCJ+*AMG=gabA1!Mk*O+CxrE7UH5skIQ0(!%N0j%bN-{2i6>8iL{8GB`{TlnwA#v6hC-nEOfOd zY&9x(4ZO;mbcg|8@-gO*kH*kJV4GCWJiNXJw7!-b(-BSSm(?b4Jk#O|_Iz~>`*W0) zqh&_BEKZiKKtx#t3#b0Js&O&bLWpaMC*FK!7M_dGW!j@G>p`nnthw z!3?_0;xXEv3$fmP4;|eD- zH>hjCHvE2Z|El>P?(Kc)`~PfBoRWp;~|VeDDYO z@4^1V-}%4aB>%@h?JcWv4=#kgH2)xWbT{we{KgpMiPxGW?0KcKfZT~riup}R7RS!0 z8&T-t;r_wb(M380J~0F04Yxh;Wm)uXl@1?Sc<|p*Zt0i6x@1))-c6{!zzB*m!8oKI z%Y+z(bDaRE*<+3PO$72a@(slhq3B6B4Ohe*yQ7kVRmEhnN=Q9Fw)MCw$Vm*+1WwXH z$#myJ-q`8(I8PVmuvU6}^)S-r=M*(=BUTh-Ii`Upv!XmUL4GVo;{;)XGj{svQEz)} zm&e~J4Q{?1f|z~XL-=(GwLy$R_>usmNsYG$_hOY!v1?FA#xY8E55@hbz&V56@K5UsWgn0#3#|9)`K#zSdU^cn^6cdGv*TCM%h#`7qAT|CiziU=i?bI` zUqL&k&re@m#?UT&7g0=abn(OSvuD^C)OP$D#_$Sz8J(QJ{PES<_di@lKb${%atdEQ zK84;LKYn(~jls}Po*kb(??q3JpC5mJN^hM*Jvc%v$bF37{BVk2Vw=bC|C7tJ^A|Xn zlk*psui#@3#`@|~zx?Lx;Sl%q5R<;|jUu&xhZuX2@V0ACZ)jHo6 z$HwE#*0XXEHnIM+D3|8bWRx$a>O)$VAgQSekrtaulCn&0)CUQ+)Xx^_$86CeGyMoI z0Jd|f{KT-q{c5ieCLa~%GX~I^UX7Beegc<* z7zkP>w=#8tq7;i_syhh6?Q)d1BzYiDG0lI!Wx@&WFoZX%>G61JSo8a$F-MmluBg)` zG@AuNnUUH$wt65uyF2)f+U4{T$p|}qNE96W;)bB)`;HS2xdozMbQ-{oM~2VMfe~ZUShRIe9mp=| z_+ZrZGvy)#ZM1>iF-X%+T&7R!9RJvAms{ao|HI{e(SaQx^pEGtl%jQ2?{8xTWpfn5_% z8@q)^ltgI6hBQ49oI$YDa3wuW*eT5lE9SgL${ocxRcJ1o`;M^+p&EfFLpx^=^PDOV zd^w`J4A6M~s_)z?Y6*^_!3jNnmJLVloLVVCr6MIraT<(%?V1vcrs8l+H!s9Ki%w@5 zrkqJXE?A?B^No9--3jY$>Eqwned+Xtl>1Vh*NaL4@_6a^+uL4=SNt+N!=SaE?H zLtE+tkYslA3cW6EV*~NPG}tx0fDIk)1+{R*ui9_g>o1+a-nrwJcH5`;Qr~o24u889 zEO-RWOXqIYZ3Dm2DXJLy#y|tuP`Af*F!lg`^A`VkXN|?lrs569nF%lLO05DDVZj7q z0$WT4yD(svlfwEnXXg~|uF?*9l$KMFTE+3CME^6iD<_m|#YDt#O)B%1`Udkj;oVj# zSNM+U22|NyKRh#xzyRu_*5SZr(5vyLDbz1XaZN{?&W}qAtc2lB)E;w5qlL*hQt)14 zCs|#LyM@nrtLkgr470u~)cpFqgEwTnu}wvT#*^#Jkm-u`>bfoJB$XzH4U?zVpzplR zt1m@icg{JzP(n^VZhe6%g5NRsF>ri$5B!l$#(}Zi>a^4bxmr8O^;`Hi9s4RY_DyG| zHe9G|-FR!q={FrB{4(da@%p~?36_pQP3lm(PxN`h#O|Zp99)qk?CGX<2y+ILDUaIr zg@qF3e3~rK4UO>}DrC1iZB(ERt8v@AzH8Mf_Rh3uKX!&zTY0;rCGW(@y<#h6Qr>i& z7Hg~Dk>p`LwGX`hI>H)ScxQ9GY$%;!wop(ud1^$n;1WsiPM$n_*3q5mAs5KY1Di#?q){M=g_{@)3N$SofM9qSKQ7 zcF)v>MuowAvkI#Aqia|~h4&8MZP%mQk4F3HZZR16^lq!VHgw~bjDe7=&`H_At5X@9 zM_tQ8IPK7JBB5d7k|vi;@%K}(v0L5o+w}sd0P)A1%Yd?0Zcy2B`S3>dSOKU~+7AySiivLiJUnH`uWRCi+fcGaQ)+d=@oGWeQt| z40aTzOy9#Sbh_Qp`)ejlOB&rR_RW@(#@1}pFWtuhspL-dOcb}+6I9|-gt3v?$$}xN zrBf-OS3=t=;Cq-i(E;OJKcj-39c_Tw;R!=myJDISIL5oQUb?_&quCGqeJ5BNQTd*d zuRLtcj$t;W2J>j=?cGjNjZk0Qt)jafS}^>B(ERw*JNPI_gWYOJHUgsf!K$fVc4TgT&|aKi+!Py{NwcM++h;D|LS@20DWatou5N%H`_b$M z`Uf*nD-@HEs%v`9t*x&sTnE&C|8SG(3ahE{-9vQ$K8kkCgiR_1sJ`Qs#R{M-{&E#n z(1&$us_#>K5Gpsh#a}++ZG#pt!c_5u_eUiXBFu-fu)bpuC+2kftZ=e}lab1y=Ybv= zMl&Ii7^VbY5LSs16I{&G95qZ7`}Nl%%R6SSde&TYfwwDx4@@{i%b*zkHOSXNzdNcx zjBR7GKcu-v&t^?xFUsN1hN-;3sm$$!>)ZFFavJ8w@l)F9lno^rM} zpjsR2H}vWbMb^RQ+f0z|m}tih6z2}Qchg>EIH)}8v}-S5FjgJ4!}(TX+mwWjrc028 zI^DOb>UYqdbvty__eP^wLzz`u4UujOR!3LfgaV02+^pz5nuwr`3{Zoeil+>$3*nYA zsTa2qg#tca|?PMXEJrzHuaZV`Mo?osc5(fv(&?iGQ7EiFE0AgQ#^GTlxEi3oMNk-I?B3&=jTt(p8jZ6U{YH;6{vEx3XU%?U!6VfN6*tm z0$U`B6n>4@0V!5{pHdwR-n=@yJnctskU)^3a-#Y}fRKJ|&PxqdzBrfXuk)2#?V2j} zOSMYRU!K0OrU?mEtqv{|nBza6J{h2Ek2B925bWP;b8X5uRCak@s|xnfkHLztY<-Si zogU*jFP1|y)x_D?CPqrus(5nx?9{J9(9gUoRLO=a2>%N`kfxKUVhiUk{>K2CdG++!`5Q?@ z!HKSc0)coq;XrA(F8hsIb!XqdIDd8e#M&Ml4C+^+%hh)}LoxOx%RR}hI>1~<0VXyi znZh%#|G*va9t7re8XaO%@~^x8kx`A+_l7}%>5yh=w za-?O+LV>}6GjZ7AADof}y3o>nVlW!q?P(99HpJ6^KY0c_VUR~2C6fh%>fNmAnt`cW=GZGmXlw*Ky?X0rD=@$J9{5Y>jG}%71-z3okSOO@ z6!me|5{x`XF2usb;0?ECOZf|a-|vuVvVJ-)XtVJ-Q|KZ(KNZ0_x~JqKygAQboL>BJ z4#)W%&j^LrGNd6QVAZy`lyz${S!Vjt@e~88XrTp{h1@YP?wL71n-$@T?blVs=?QZS3QmR zh`0?{p2}V?XGy+0DKnT0jATwfE?}9-<)soY=xI^BU(Qv9YSIN}qsMRuWh&kQPoEus zKRAJt!=A4f!14XmK7TpD?=RZ@=qJ59+M2&cfdi&6Q zTQNs&+r-+0-o8(7`<#*Uvc<`?-Iu?imysh1F+_n%gm-CMl^pE%*ymv7R1>HIS(mms zVy242eRm3&@$sfl@hffj-TV9g^ckKfP{ZYU+i(4wU=7IXxD!A;N_`0q_WAt)m*c$8 zTtxl-!@&E(hXmdyUcp(#`|ABh=HnKYk~@vfYeuE*cK@4=&6`Qp75?08y=fD{X*hzy z;k3Ougel}sG%-p}C5E+ew$go7NdtrA)FGUW-;r-r9X?~U)yOYio}Qc?KWiAfuoO;{ z2fTx+$@QGnt@eNmV5>Kv;WP9GJhVCx=AWD@tZQ6Xao?&U{YY^oDyL&%F&7L;UAcQw zE@A6PONY3+wcNn1X2j(j+^uk7tqOPIzTJtmkq_{emu_%-dB55Bm)Bzgw%Jub8|{+H zX4LV_V|D+&$}j|knQDh(k;STdA8Xb{Ai1Gy^t5F&uY#RIKDQQZx5A;KL81Mhn9~isPJ3!ld1?T`8o%<6cv2fkNdXI6kdKAc znV#Lzu40K#xZy|~-$jYXX}uU22Wi@DSOPc7G&wj^F{?s5PThKR=za1b8!wY7tzBgg zC}A#~+2}T&3$5AYR8)#HMz9`PV5frN4J&;`5Wp2-(mG88;+CcV=ubtM<1h45r`Fb6N-apwQ+?C z@!%}KZJXNL*c0=yVVAx&C9u5xzVTDDlFli@z$^s^`z$fo6V?1Fsb=M>Ght~2^F_9D zb~v!{_sDc6X~AIu1^s=9x`iQqEzn(SjdHlR2My%^c!3ZV#?(_q9g4-va?Y^8?=sxe zbW$g?^t`VW;HDa%=t?>7nEAaST+I1YH(cL$_Tl8weFZOUVNr2H=oG#G9`zyFNa((} zNb9AN^Z?sl1W3W#ZD%ZOtkE`=iUQ5VkTakW>LK=`_vy`%+Uc;(-|oNj#+~s5JAQ+M zcEI5uS1`+$Y3S|f=!nxCw(IXHtIKuz5sg;XB$Ii}l`GOIGMmOPoL*+9aN4V=Z=&y- z+u3QicifeY{sg^??pF9ecEVpS=F7!^Vf$hblc!@&=f?)S4LqIu_8bFZmb9v!_EGzm zZ*$w!bsnoSuIjbtZb!kMJ8gfvp>6k7I_ErS**W~p0RF;n2ZQysRkvu%>E?IsZr%09 zK4}=b(PX;BQ^b!9p?sCBh%k1TWY)s9R(uc%^;xKpBm?DJ(116pgdRQtkY`Jhlr}%(Z>&>Rjj{r?G|QeeDc)(cCYna$`{wc3?1-{^KgV$)mXu7gX|{Wr%YB2j1Ddi5YUv9 zqZyFkQZa1Ne7dZd*D7iAPKqk*j~Biuuxa-Sv{Y2KV6GLKx{wnNTn7^fTo)!KxC$;- zwKhjishf!I;2OI@k1U^d726{!&t; zoGmKx3xI1FHL;lkvTzXVOt)xIdU5>xG_vq6_~+0RdV2i)?AeclaJhT2$#ldS`TR7x zHJuqvjq(>+;9S^1{N7^QWj!H6tFa#6aF9|iSs2hqr?UNXoNY_%S0@n^XP{s@B!v^p zLYRM)v%s?6gSf(qv#k|*y&}QthiWkS26`xvHc=M2LQm$DzeV6eVLL+(R(6m0!jnW4 z+<`Fz(nwYTlj+0zu{N-QM5$@@@Lu&D(rZM)T?kb3%htmkx(z{J=7{B;3*#PEBcgyc zPsamdb&sT|n1(vmCu2%7M7!bMGzx8W8t*twbbT9@%DNi>)o93iz4KazMg7|Ucz6Gs zk2}%b$ZN+EW)F7hfD)_bpG7kmIBPHvR}!y!)~pzPGFMG!p`o5J#U5g@j>0<+s#*n*arh zPH50fnw#e0rm{Cq$GLiHlDGJmA*`B_Rjq@Zj7C7&ZU1^ElxmLwiLd7A2q7scpo~&d zl9vm?i6UJzAV^~GDA5V+ijv4_Vg$E>ksvLOhZ}I!|25rW5mIy%}OfPn^skiUn0r~Dm-xXJVCGOfB1y{^shOG%NGX$b3R95!cd)!33^Js5;us1&3Tig>p z;`ks^``xk;c#4!mKEwBbgy}^?Z>tPsEz-m|ky-A`7a1BS@D!SNp+e$ZR%F5;5vF9b zDqf=5m@;gm)E=mhy!wZcCt&H<;h0OpTwq%!q9~_o~v(N_)yX;A|mA;oP`* zP)Z(Xxf-Q;Qf7tUt3tUC5Xjh=)72!7Q;kxr)vWoo5u&joyUOr=ZRQkr3^BCB)xFms ze(05d+qRtx$VHsjuH{_Tokn4jn3I6)=28A;Nlxt#pWVC@y-B01LfQ0*Bgff=O|ruS z%<>%;z>-{>8H57(9k^q}dRR4BogreBqXLf2xcES&0+$tFMie-;MVb06&=(UGH;IXB zC0Ofd-AY>g5^|#}L5lK236S*h7FBE~Ki=}v(ObOvhM{23Ew7l1%j3(}7kw}36fiJ^ zo=PYIlk6jF01R|t#$(nXJ@6+x$=wW;#(I85dUUBNl9 zrK<83ipS86E6(Mei4t5n#*k^$cX2X3!I$YUp&uPaqhBXu3!@FUFmEsnR_7QWa&Mqi zpfdp-GS>t9@oTvsqur6nFC+bpAo5qm9^NOcVFbl|}DoMKbD3R~p zq>{7%mQROriyo3s))LP?MMF7$jAWA!vI>^QO=u3rF8m5Sg2mZ|*aY`>XOpJ9(e-X; zmx|(uRK*@)H+_z!fG+IQ&M~V-s}$`H3x?D8`7Hz!{ZM2aPdsJFf*P$OgI~2wWnZQe zaf+;(731Nu26}4#WOyX#NwCad3e_G(@NZy#Y|8x+)Atl}II0pJhUlSk0dRLzt9ZK6 zQ;yp8>N<09N^PoQAWGGhZ>(Wv>(wvylGquVT+>N(KwAimiSjhm+lW0v@Uo9Wqx6uD@FmC5Po20(WquU?!p_8FmLrn4M9FX(m(B zY+-@NEy`uJSYOl43|2ixHzBx0gp89wPQLG!E@+PvLy}i#V|~MPvBC>U(R`-9m0eGd znN>1|!sZ4^A#Sd3zCI& zT8leBBC|!48#Zp%$~d%4#A8x3;L7ErZy#*zAw74IwzerTDm#bL|6@*c>`!u$nZ{vec zqkG%VDpc*0h6SszYcP7aM$H-rhYMzcCE5nbFNd%P9mCgpp~)kSO~ZIJJx$Kozl@l; z4_Trk^5Gn!=!pRlqTV21XH2^#Z{l54!v4zToH8`ZO|i6%EUJ)ptwI(BGcD?dv^Ok} zl}thflW3-LD2NxJh{56#NTgJ}s?nsiTokzVN2K;qWM1RJ6MO;mTd{_cbdZo}aYp8> zu_J*Nb_r1aDz-fEUJ9>o&y)>iL0y6gC3efY0r@;T3Temp4+DKhcpErja8ZZqG4IB*i-? zK=C{vb)scu>2nh-dG3Q;itfT*^wn4Ilk%!+xD-!GdCA`76uvugo!vDSbRD{4F242W zvW<19fgJUoe_E8)y0u1sDD%@{MLb<|UEd8+HLTk6J4WN}>*OQ)7*fdae2O_W7Hp4B zBFwNvnM7)V37+d1x^a@PhI4S=r#CAMkr?YNA4=ofP?PL&O=bMCN#Bv?HoH{nT`+g& zlcTYuYPIeboRyVwX4Ag#f<$BmiNe|Y!$ik)WA7bvq0i=v8-~0j5FH6j0VY}vK=CtBn?Aobo%inwQWoYa zd|6i%TK+kq%3#e66Qz!Y6)uVd1}(L{X)h#0qH{-IxkV z9wwfcJuB>qmBnt50av71A4N)0FFGP83+jtxj4tY#2)v3YyLc@-X=w7Tx%ML^M9Ja2 z5e%~!cw0ZP`et+FDeAKZ(&UrkQg5u2`K@hYpLx=Pl_{pTMJ*nOO-b4vSS~zt zBhiEa?SkE73;x3!n}K!jP_j;_i-+k-u`=4);4t)&M+HS98R=CQ(0d4PQ%*-!%R;Nq ziBc2TS#k{WMm>oQdf8FDp9M;&d@1QRMmIE|scEq^Y(3Ra7`!(;Hcl!Ws0xElU>dRt z<1&Wm_)PAU3kFsmXO!UQT4lM;mxw}#42h^?d~lb?jCbhL?Mg9hJr|Un4-m(VfSlY! zd!1X{XXrT5XHz($+ng7m<^&@)5>3=E%f{=pEmTz-b-vx*q!(GY^`FR4YC);`hB0SD z*&GP@2hny7^0XiED@n?E>ivsU?E|$*kb_cmExuiFXLKSG;DoZpJMv3~R>cOcE3h^a zM(KrmIR(3GgGtv$zVP*@CFU$&98aelQP^1{GCx_)W;Y@xsw5_Jd|UNdC8%~W)_^^E zT9h`!Bh*0T+{y}PMU^eC)|6ICRgbrbB)3xsjgwxFi)A2O$>k)W$U427EG(4jM{kG? zlm^tDR?9%dvrR9YbEGH>n=HMuXCQ-w@KXv!n#~itLu(YTAd)b zayCI%l6B_~5kpC4)Xr3zF=7NE5|MKctBTF5m4eN-Xsb>>xZwn0Ty>K(B2TufX|lyJ z3*yoecH=cuEa`nIcj+y+wPtJ=R$V&iRVMoaHuYLEcLHnx#v~yf!U*tb0{gNAP6j5N z5OpgUr~-sC(U~EIh(DLV-7JzJdw?E@Fqhi%@8VJ2i4oL*`M0on$v&<7n9}MzEXE| zUy?$@>J+)9n74b;em4xu!o}a3qNo(-3jY+D4C#}MGQ@@f(DWpr`4YB*lR}ptkBa$C zI4}C`=Fo#u!;wYR(@$0&Lu>GrRM1sX#pi=JuP{x=C;FR$ zfB4TA=bSf#KVcRO^ZJ^plN3NojNd88>@xLR6+OhdO|O>tccX#{X`8j5aaK?{dIquGK;$5T#u0Y~ zXn^t0oy-w zwT=o98CgdMHgy~?w2yBO-oc+VIo(^-R*~Ya1C$r%=_5hUv~N;$7-{>21!NJX0JCJ? z!4dbiCzC;%A^fxg+t>nzP2+L|(bx1-L0q$9wXIpx7al6%uktz=T)kgzWkz82N2C{nVi5EvOa`#uv*8)>5wxGo*O3^f3b;zY z!%FB(AVlTohSKTh$IH<{};^;AIzb5O_X3DFgA|oV% zTdk>LIL}I0n-mv3{gH|)Zc8O?PmopxFAls$=!Kz>?y72u1kisN^}Bg z0`$bVz}Zj*f-o$VliX=dUCeLe{F$Yeg`k=kF*_lvQnW(nr!|!k;Ghr%{WjD)CA(L` z>Q#aI7XX`N(OL(_x5yaIJE633mReEGXXZ`in%a&R5el>mo*^?_wAbS3X4%#Cf=`CO zb*#-1+B&?s`xsnd)td~94=Js!(zf;#0tdOLrHeYyv2_4p(`EXAU8K#6c)eK6tNz~J zxBwP@4zmlyy(q8t?(I#pVVRUSdvNEZqeXnZm`(4<-*}YhAAh83EcQ5Eh<%5e*GY5( zCYSx$s?0lUz9MnF#uIfi?${W6FOfKidYi39R~aItI)N)(7mgcBa@}e+YKk=}6>~*t z*-?G!sn=eD2yynp?gYazWOK%CLv#6v99fMM316{GasR#~a-5D|gptRnrlY8{d(ew^ z5A2345C1aAbh~Uz&bI; zm)(WNd)jNvze#C4?pL;SQ!BnE8$qCI&wyrk_!&kt8BWOSrgc%9lkb%=%#}r`{4#Ps z9;pLp_28rcv8WoE9E|h8mL91>zH`eRA~cfl!g$g5_##{Cpi5l=SC12j#*EPBs#4aZ zE^O7H1LCn+hkVQq#EFGbE>0#^yg{QO>NsC(|JEoao_!>&q2>dl^hTi&7u+T5{6=b9 z%0lduX_c9inNiGMTjjYcQog_(7k1Qxk;7BmUZ&JuIGMXJxL^UR( z^w8T)!rX0Umz_6oa7m?*>2PQuWsgA4=1puXCZIk0{WNF)6OgcqC2);7e?88sQG#A| zvEE5!!>0FL5l4cMG zX?cc{tP*c6l~f?EIUkt`hGEkTGn zZXsHq;9CqPmag;${E%c*m4c`wd{+`&U~9~2LOn|FaD1&sCKJw*_@`2Qw{|bB>MnY9 z=B$S4r{Npb6%=b(%`mES)!LTq>N=jX4vzl9Oj%uW3xR(-@k-aoK;pJ?*Exv#cE??S zwsfrR^ga|??1`@pKy&&Ickc979f87>Q9LFwpz?y3E`kxSgyT%t!d{`Zrz~UioPE3h z4s{3cF)!lNvlo}I!ktzCV3_a&nuJ@2Jj$chP@$8F@nmh?TR5Kx1m=L|<2Q zACA)x*(gQ$i4531&4$4BicG)UMs%@;8Is|0f`Mg8Std6z{o840?Ox5Zjk5x%xh1r~>dvyFil3gHzj$?e{G>eTF<}r`QCm*I*@i}0s{5jaToq$(@C9X5|W4EpfO8P0}X03o(a6CYHRl*GkVj( zCdHZ8>uzW|nyv14csJmAq7+BC9ng5>*20!099SpRwOVo)qx!~#MAcOY?%&nkL!nSs zH^T3*)Ojpjj*a_EIKj(&o{ZkZX*oMPYkz*`p7JdYt&jiwiAV9S+cX3xBN3fmHWb$S zbazZPFx~5|y2m|<{v?yxcRs-%y0%&e8NNq_Dy~wWvqN7*pdiA68A~Kb(EQ84v|JS& zC##^b!Cpe&YY*@w2oG6bv}F)R*=6TKS2p1UDAI}SRfjq2Zp~lY?~w8SXa<$!kY!yQ zFC&dXqPWEHjpuQa$#~qcYU*x!M+=iOuJNxruK20YW#*0Wn#hmeP>fwhC;J%FVFlcgDGlY9b6d8;PK`5m_ODK7- z(dKlw%P{?mvC|P+Pbv{s*rbvUM$TJ)yFqrf@kIgISclxBEDHo%ab2v4)aB^Z4D3D9 zGE8w%swYei8)}mn6wYJj#m91V%|ueCSymO>ciOFvqEJ#KxYhfN_Tnt+5Na26gcJ}3 zZhq`zH*U;GpaL~^Ugw4*lDmg>@Jie5KyAPV*#<}bbM{Z^o&;MVY*ohi%$}yX3ghS7 zaE#h$+K67G+cFI;tF&xvW*rgrRd%Yk`4KRdAb;e`>9i+t74}!*`v|yzvb2gd(0|n) z`+-?WPvJo=%7QFjb_Z3-Ko<}dhF`<2yZ0d}_sV6y_bu`$AjWzIc$6vLnXbi zp;h|(m3OcscGuTJn-l$?>vcRq{8$s;%04&j?^ngh7OUzuN;d48<2Zd4<$F0Bof?wN@i>`l zY@`zeiu~6&x)@!TMGA}Erc;{D{pj3BXv0N|YJYDJdEFR8BR{0*UzNtN754b8YVYvj zHxKU}?D_N0Q;!|%)UQr@4uVy;-<*Ksuob`s1F}$M<4&DSuWNCOR^_?^9;h+d!sr_D z?%R9qZc~XR-rAl0V*l{4dkc-xnh{II1K1^t&i*!xv99c(wXPyJcwjup^10>fDMW!h zLSH>Ww^6rgB$LbeVTWI=#S6gK63u1mBkB_NL#?}#dTr3V2emf%oLOza7oXh*y?f|n z{(F)xFkRaM7S;C7HV4fvOp#{}Z4Z$VFTc|-bI+A&&^w=|K{eUvrt7)t4(KLP{<g0@uAXDiS)Dw;FPs+**;tA#o z6fR9)i7JE9S6{8e&GB6Rv}&8?fpYsqvkci*m7;`JC8J7rBFKk*R;b-fvTT5(B9eG^_GgTFgu*YxIYYSNTc3w+z!Ds0JNnNcuylDd7_O?@{rn_X9DHIye! z9tp9M)6l@zv!%7J_?c|Fj+Ibg93gC<>rkI*aE|9hcb(3h@6Yf>?v%96utBT%zoj?B zA}Pmb%5)u)3@#8@Nzz^jM^j-Yk_DfZC3^jKPKta{7Sr96fI`mV8+7mpVH8zXwQ)c| za+lyvbb`#ER$P0&gjk`i^`Xye6TNa}tOux!{=75dCyPo*iT18_Rp|&+UKTeDCQoFB zUFr#H9WnmcheVPT$8p?*-4+abPx!E>oEe(9qOQkNtYGDYSg3Sdf<9GM3i@@O`<`Y$ zC*w^yew3fZGOu~_Hn8a7$Cg_%0VJ;VkYxx(i6!zqeLE&1pC9$Kx}%nCn+K=zAxo|n z0Rwd_A+}3qEn_^vMCC!?-Y%pu$|a4)6t_avhn|iC5WTm&n2vQlDs}?a zb8+J#^=gPx%8OL#1VdxhHA;4&qsRmq?>SIi^l~x}#I}h%rwA;RK&Y(TD3VNrx#`Yp z6nuO+O{xr0wDlbo|L2d}>e+yV22>)gnghC8zK2pEuzQqSVs-CY);Sx46N?g;3cfWswl2gxf>(Mt>rN6H}Dze>vyh|1z% zCp0&~U)zRbiL+=eCTnXxnpQ>eG{8sWagOb8M-ngnuSLf306dcQphs#kV-Az4si3XS zW5oZkjjvcDL~U1*Ah_{PXiEBuwqXbe4a@Z8ws8y5Lbx%~cYEh-w_{)j^W{3rw;?=1DUf-bB zqg@GNagCnciJSwFRTp>!;1Yt$C^b_~QlW%FpgeyDtc<)&%>=8B%Ai zr_)zRk<>eFmR4UK$DmBwlZ`|u>BL@Ag*f`I%@m*vira4uVP!RfmZ^`cn|!1N>H(H5 zY0?BgVO}!SdJGWuk5n1uX9?C#l{t8`hy=l&SyUggkXn@r@@=7V1rkFi6+-nOd##G{ z{XZBbq(>>^@^lIf%(s$1CB%qEVzdARAWhWJN)fCjOxya=wi;~9rr@N+1}R=o6EO21 zC~ZPAcR4~=JD}}MGfs%D;4(@59n5>;4(=bvEH&*gzb7V6z?wdbim>nEK*tm`VZ>5; z0X4gebUH=QUX}febw?_T7&f+&SkTl?E8>M8x)~YVM%ZGbu%R^qTzRx%~Q|G-2##%SKs2w{SwvL*=cHD8KHTe(O7Ai=({e$?<*TEL6p1NS1SiMm%m7bKR3QEJVjPdN8Li_6~=8YC< zN3%_qX_X|wI*ZLzp~Eec2O69@ey8ll4W{bmikfZMIDgaBvQ_^gdvNVAX2pO`$&t*h z`_!6*fi`9x={ElpkY3pVn3-H=-^KQD#-I*$j4^W?ATx~ww#fy*L&DFW-W%E1+1Z;=VTqNc#+?y_PX~Bhx&2pOR<mHW*aXVo?n+&Z~nNN_&!7v?r-Z@w03W{2Q(J16bXV zziT`FIGQfU>41S)us9|K?KUO`#IR4!zs$j(W*4evdXZb_Rz}YN9>!{{%#g--RkK8f zO&Hk0e)JM^VT-i5afaP`xdMc<_G-pK_oL_Vum1`^33W!BDPI~(r$=IH?bjlulsBDB zaz8qDd@|rG<=BBBI;fntlB?Iet8^_6?1qA1a+Z&gT0r*JG_a==%Sg#>P&cjKNV)`? zf7i35wi*pTc!SzjXH7AEv?y;R-mEv%@)i1jtUL@a%B_Axd&A3@h(V1wZgR%6dHHQ2b%SLp)#!Bp?F(FS}Zo^H1r z{iUHc;`*SB2~XE_UgBu+X-q2G3bl@%_MqK;XSp@fToW`s{Xa0v&K!NaX^y@mv^{q3 zHmx~5{jR+Z-1?M{#8W=x&Q<9=BA=w3Pf&%v1$>4!xkF_>0OICrPQ$=hr~Iyr!) za2oHt-PubQqrGwzf3?$#_``KkEpX~~9EgW>a#(wNXK!}1hsStN2tW)y+Ogk~yjbQb zMffJbvfDk=eel0iU1>S0q`u{0H80@mk}zdO_=R=J7&9%+N6EZG50bhYskwMFO3Mv1 ztI0jlb9M);DgmpdY=L7qgxS>R>6M4T6`XlJDG{xKmz$w54qvs~2Buq@Q1G=wBo84*-8jv@Ycg?gwZ?u40Gq>t6PdQ2p4uU;|R3)f}tGah;biSn@D$ zdG_jQnO>zI>zP6LYqT3zi;#OKzo!u~C{lBdgQD+I1d3n3I6FCia_YoHHBCih_6#1T z)j2Znc!?s~z#wxV2?zneO7v?fizqc-mg^EEpx~E5L*t$3vdm`a$|L?nm{T$%ojUBg zMOw`h;2T>Fdk$qzs5wSN{CU{roQ|)QMt~3g)459Sw7cu^!|G1c)k3X@R9K_YxK+$z z%m{lz1*p-o5o6M?<9Fd&b8#wRi(`!`fxWr{xdtq;0CP!_Y(cS1m6b1#8lP8NNCb*Q zdd0n_n4pY^$Tdi?n4vPwxfO9sZ9=FRYPt==6I@|<&IADey{uY@N!yb|7u zI6$!uG@B#u3Omlx+#1=+GsHOVqQN8+jDd$imIO`3Uy5`?Rls>HmDsr9@}5;!AhgcL z>vskMZ{0Er1-;^Bp-__RY%CE~n5PxZp5=|cjY%9`jdg|r^5%fauS8T02TR~JM2a(9 zWHziSac@QGdY0r=KUs<}SigFxQ|pt$LlG2vW@C%FQOi@G)^2d^-L-znr^;F|9ss^T z=+q3ykdGZ{H9A`u^dDYWGmn|cYf9m`DZ2UyClOJ8l$cGSR4OCqamz109aDDU9z|M` zS^siW z5h-kTGkIoD&|!aYkK5D`yr=!M z6iK$#727H(PKx|PnrDO)$cS1x_pG3eXene(5bNOZ|zzl-6V5U{=+D z#s$UZHX4Zmla!x+Mka@MCGJKH-|VZ;TDd2=;pBnOIm*@=tV(1$u;DX0bvJMLiZ2Fe1VNfu;n(Rhm~MN!PsdP-=i$t4y>Xl^yYw+So`H(t`w@Wf}V&oot

y>w-0k;31nP{WrocQke%$Sx(n zUD*>qPN&+S-;7EYEjnEe>1$dtMR*5Jy`g(5k4DZd`YnAXap-b zd){Hn63Pq# z47`Y5SbNFA^FTuX%2*}Pm#GQ%v<9k7BUpq6Rx^xfrpg-gFE(`^kH^a7i%ZXSGJ<&G z0X^m{B{@z|y1ki8a%Y`A56sLv-C70MBw1a{UdK~)#FGnD@JN}hk4&{k?y4T?8LT~w@;-=*gJil&ZmI!Y*@$JXoX0cN0wZJroENphSt)?0W@hbVhXAUyTOl}b zOuBFv@s6lOO*OZ24*S)#7&<$CT+nychDN*%{xUfE_#n=x9(OARRK)zoTP*j^$}hCV z(bN7{BY{|Ityw1lVUk{}kAGWc>4MVogkJVmbd9I&v9h>#s;(mF`o{l+o&SVSIDXaK zo}X|)p9sXPqb;0Xw4L8M{KDEDrl@qN>OY>Z8%akeP=eje7(PqXBlsGYrS) z=Lo`eC6QfGOHA>wT#!yMA>x^;{5(K8cBM6QHkLzeW~%ceuJ%_PaA?)=EA(v_d(beo zTyh#lSyqs+WF}Pe&qo$8DX*&@VK->n@NIbvsdnS!<D1{-8o=#F@ziq0AMuWtt zO)xW*)T1t>ZmUn#A{wbU72V$;I#+OkXr{g1U&?V*Wr$F$rk`s-%{m<_H2o;cPtVv5 zTfxCHr-9Z~Oyrk+P?`JW62wXsP!pMH^v&K1z;@VIJ?ke$VGh@iXxw;4((y*|6sPMa zSOB`aer^(02dqn0Va(A_e2F4+AII^}4%D@`CxSZPnl3w};r<;qOCYCR5N`Y9EW1aeY#9sWZ zkPBVuwY^HF?+5ToC*k(t&jwEinM`-}J7t4q+!1eSVoNo%(y|d~N=ugcMZScaWYjG^ z5tRV@i)v|-b_3HI650dTAo;WFS?eo}I}^OSRrKb`(cN)R^m2dMw)1NCSGO4NW)n$w zZC#WvEo(|F6jn_D742RE&wSJ>&b+uSYt2^?+aFnG1L~8uNApLzwBtd zRh9s9Wgre_>)Q?bM$RZ$rR2%A**fcLa9!)MuYrr9xf$jik{V2}%w^&&&wnR+ZG0zUdg-Tx#RAyCJbuzR#l+16||d}1tdhz>|E)U<6v!a{So z#yh|^D(nZKKQbwJ3Fh=4w<# zwN)`$3uFs*l~>N1t;WVv+jgO?*CBJeg@3kIXTVP8v4!;+Q_C>wDhKSOj07?}@?070 zH*M~Bs{K7&g^8sxS;;mKjfo?*Gmik3xXmTD)mZa-Ng#D31*tKw%+&>@KJjX^ug<q_@G$=CwVJj zb8>Ui2PL1o%O)Z^e|dWuT`6Utf4$EZx;G2kTMF5YyW?|q2m_a(D+eAG~ELfw!b)8uuXjcLIeYulJ%(~d~?E7{bIVAOJ}X-7GsBc6`}^RCK4ZkPQP+#z;1 zN&gPzRGtIy_oh%F_!q6g4PYSh+?Z#Bk~Z#9E*duq*Z8T!J=!9~j|v6@@W_uGJx8Q( z4N~%au*UO3ijeNqb#EMR_VIcpKJ|v>uzcm*E16Xw7Id5?XOP2J#hwk<*^o^eqXNSY zMG?Sh)|^DoZ=QJDfFWcJA?><`3{u{(TVJ-&(Euoct@8`GMa38DPi~sm)q#+C|x3SowDlIRe1O$L?Ra!hEtu4l9`G0Txgu?NfcW~ z{-wCsfoe_Fq~SCO!Lyp~$Y_W*D}UHiyV8zmTk2C9L2*#gl6yf#)=yaZ0w;b;-O6V7 zp3)u=3+t?`_dLwD(J4^K?Q%$~v$YE0cX4%{P`>+QC}pL60x_6vy8 zjS|~tX-!FLswFL@Nx!!j(NJ>vJh|*YkEDLb4QC;XU$M0P=WFth$8*)k;6|b(b|}hi z#2=>1v>Z|AZq-$%s!dy>7f`q~NJDNc%3x8VUk0$*A+QU_rJ9t!O?c9bj}YnFBF?`o z(dUA5B1%+_;iiBZw5pO1?7|`(&v0#eruPrHCf- zi#_viD0lv1jFqv%`4*gl2&ZyY7OY05gbdJ>b+#9)LQ!OrmoPFKeK>X# zK0P^oc6xbwJ0lD2?7nLt=`iaPuzPed8{P^-G}hQlaNdab;XrP?YCD89v3wbEXRWJG zK?3wB*^;}8hy<&c2IaK6t}X#f3L1_Avtf6+ab6t8Nh89c6BxT6YU#Un?XN9llNmTp zzut%fWoRQb)$ke5Og;qz8dr=$9w!2}c#~Or7 z;DMQtBR-{ZZj(kkZqUr9EX{U)Zf@8WZMM$IH7wZK1h32v()S}Z?CwS|zfETKEu({4 zw{dw3L3X%G6$ekj4-j>NFP)MW#%{2ZUWCRH9|O0B-hhR;D*uU!VeQm9*{e6XVN!OU z<89@}Cer(O9?-ZgCh{GF7xOAvofFs}hmE?BF%n8zgUW@+jNyF48qxZVN zy+8?^7ybO8{`{&;)7+Fete3b+QRQih+zWLBR#+FfUoRjVO&9sPUSxEWSp9eqXarYL z6$p%EiI9A{!2M7!v)oKppbJ?JBGW>bcsOhSY@s(f%kP785ezwP?gAYNvtfprng_FD zoOZme^14brc?v}#ij~hrL1a~dF;pqLk~!bEB)%%O(YavDm&FIa2I9d?lJ))gVSK=h z`E;zj$l<*!VC1VnOWcaY*;aN!Dv)0?bXuxl>Z$>PnDdOb>HeV;j%Sk71)e}HRSeA5 zS8Za>oUf0cJsX_<`Siu*MOz_$`5jh)zUb#mSAv0Y(Gy9U2J!Bq;r!D-F(z#_Pe&Q~ zC9x7tu*AmD(P=TCOEyi@NoL6_rzAh@>xBH;aPp}Mdky~D`Q9zEz3A!8+ewE zd%y8>2O1Klsa?y%7igSc?U{<-@;73`<*%cY^XD&*FV7yIJv+Pnaq#^72{Xj?ua)vd zZ)M;v#sfT8{w98GA~>~ilA<8l64>5b%;s&$hdzx2e{5pqdzSqRX9x*qta3`qH%L6I zv_!Qu%pj_xdl6@h_`gTE(Fv$1Te!=^dZzgTL{jt%pUsPMAwF|2@hb@{qTC)Z(jEGv zAKk^x620Yk=;|E@$sfq?L;0P*lWZbXIV$_?iaS)*5&he>iV>&I6?b-aKOhM4?k)^w zSX3!|V2Gxp9YX6EEMVq$nm9jL?ZwfKdG$J-&X0Ef%%ACm?ld=Ca>LqO^f^siFE^Cq zE*)DzwVq!h#pbQ)X2RW6;=`zhXq%96?5eHzh)n4{6j9;!~}!jzSC2mPOG9-JNdp)Kxrn zK!g|7=PahYxOO$4WYv6{0OefHW;Yl{y+8-@(Yh7=v!)psfbf|#Ltd3$m|IRgYHofa zlp|T~GS~cBq$61k)Z3fphkT&Z{2C>_vEb8-^cb({=ZO0$Hg+H%!j!g&j zi$^+rQfD;_q>a76_c~oWQQ6nsz=*c4@FlM2@pO7BNtC*30^VS#GAS|e};8lQ@FQDV> z;M3e8%VNwK!!5bwF3@(2o~tS)ja(;~vsGQIU9|CAD0hz0&qmln1-d57id8m9clwzD zKU11>^@O7OWd(=H|F7I3$mM+KQGC_E*djdBy}x+-WE!rNB2bk5;+fa&5)yvS;i z=xS~yPl%N~$0~O;x)895g*dlv%MvP09e%2Blrkv=_jVDFnjZ|!aku5UvY}kxvm3YU zAe@|dnGb|JwK}XOVDtHB$cw!%y2ox!{|_GSKREoI{{JKBzpTG4Himjg`mAD!@dhX$NlGS0ha{&-c|6T#nY2LwGdMpu zSr~+bu6fwo9-0#=v&}4}!OfRLxbzT5FsrodRE1xTFr`Wl$FheRHB$l}fjU&=oYli# zBx>_HPV7Raip@jU#mpT|hJBhW%N&|z&xUaUQ%9{5QjR!}b2J_$_r`D@iGy?k~4=d=H>y=!ky97p1RuRn#B+zw<8%o~#IW~)5lOnk$ufXt(E zQ!D{zLTz}I4N0ioe1Q8P_r>m$Tz5;djWJ-dTz2cqs;LQQxrgI`AD! zvQI|$vU$~IK+)D3-7lsmD*L+XuN0d zAZDZ4Wi_L2bb)QRNl6N3hTcJTb-i7=ginp1^5fWlVRReF_oCTm8f$6ou2H$FX>HcJ zYPXsl9Ym-Ca8or`bmA7f-NYoJMJC=4@^g8O@&z^u5Em+VGJa6g562W#7yi%6KfI21@LZyp^VE9tkav$W^tnBQ4olg0Q3TAS#W(F3?X8<6hg#u z*{3LmrPU9ir!0@VLkgfe*)0Z8od(EZ=T9@c%j={NLK!E;5Xv|y1qjiB zol-EM3L!?J5F~~@>s!7`<9LTDtzw9_l`V#7TiIfumVjWF4Ebznw+#7gdAAJt>`9P{ z?m~1inJ`^UHcS_j3lt$lS#arh>ftnm9&(ugA?{|g-4!-hIt|l@WWw|z*#IHNqnYg! zqHeT0y^c}u!HAyEK}`w_Y>3d@rl_R?WE=zW97+b@L6k26Bv}!?1`DF6@1TLfdS@+g zRXYhc0{c6IF6MY+1Xo-BNX5;O9PRnzlqik~v({S%Bki{Q@nLqm7D|PUS}5+Vg+l%X zS|}9u*Fr&PE`%CNg`H|>Y_AQa{q<0Oi5^Z~poh~?wV~mu4*q6%Yd!p_^DdMDhNzJ} z2oOUSL1w#rKCP5ak9Ujfi&M@V@0Q*lCwm;!uMJ8kP-FgtX$j1|j|7edi(O6Z#SGV( zc;VX`+>Pe2{#{t+_c5f!3Io%1`lBdhQ8-|&Ws_FPZ`sO|xUM(9vfzGmgRS3aTYWO-?x^U^{q$N&fNvxtHC%)7Lz8G;#yAQs#c>&4imAm2w3*~D0oX! zOpr@~#{ary^*ZMK0+uQ|K^}nD)1YTG33`Ac`%-1>nWs#Ct!3{WKKJ*Lu1v*7v80d* zqCRg?mAk1OR$ufaG3tuZcF z=z}jQ0VkVDhWy-1q<_(3aUeWiP%0}0K)@w0Q@^Wy2Nq&KrOQlkodc7Fjldcx*l#g!BaS6k`j{lE>)p%VN3E^3-*-@F!qRy4sr%S-a)pyq89nSg z71YO*KkbQD`aF)0nN+fem#cRlD*;o|U*VK_R-U0SVHoCT&Ww+^P)@Bcy}I^~X1n)M zZ+CDn=_i&~l!xhvo09sqm%tESoKO@7P}J!?tb8KoC-n%suKR!kyO?-cYmE;tTH3&2 z%*{d`vmi^HKe~hkPfUUeUa%KM{+vWs;~v3t03*hNGQer5^W!v|WR#@C>3~yaq>qiy z{E*icZ3tS9a%lTFSmyn-KlFLdtv*^sn?7%}=&^i$w6PCfSOeUyrM1M;>uo7ur^#`h z_9}$Yl6VuWSXoLm-@}BVW{Eo0yVO6U0FnLA>iJ(=*Q#}09S;wD{=3EgBU>u??EkXm zOfesw|GkDga&KcQ%3D_BXH9Moc*g2R7Xn(=lum@M+BF$)eK()a&eG{`qxs$91}%Tn zwt15pr=2?WHL;WA*u(qN=+{wb-swl64v_%SVJce7U{&w&pPLFrQ9V4?bFi5N%}S1r zSk;_CKVwb?Bh%)Z#AVNa{`GH`Q%h<#kNM<-1otbjgnMo%Lh|a9;AY_ciTiu}eMIm2 zcwZ-kAiTaKZlLgEhSn0YC*Ihe^9Mt@@%=Be(d-v#{@tBA;ovP%HboX6A0JilO$iFq z!b94$IZ59`7i-Vc!XM;S(eRU4{*Bp-;gpy;jNST#!Z$4^#Jv&4@BQ5`#VbYAg*7mx zjFr17-nCpJl9|v8J-9xzOnReGJY~yA+@$!MQ_J#aUd<&{RgnN`v-|!4Rs8w$XZNi& zd{28*4D9flK6zC4zQ>J|`jA^fP8^1O zbKtfQ0@5;jz(KGOu57xP3{^!$>;YfIiF2^!krBV^g!}d4m3!ecb$)@c@+VI67jwiR zc0^PtNjCuhms|)X!AXE-?ES&7s3>m~4z3`#cZMFuZVm=h%Uory$V`Cdym`X=R7bGF zGa(_qPXlEu_N4GiI|36(kd)EncFKNYiCmHu#C0)&TVPt2gf`%hD)GY%gjt6=WV^7# zVk;=8790o!WMMvy=83FFlzZBDa}+ONS>*Q$`WunPk7@a{kqDQvH}){TNo@D*jjOl+ z&>x8>MknkC`=1lUv5=etXV0&HJwltpsm0PZ0d#@&_X8IR5e)f5?nOUh*Y3}B$?d%g}`RVVA_m84u|*+ zYbv=lEU6nHm<21oT7U9wfkpaEsi5d zz|&raqF#lf!e3B3_8~6rLtNU2xE!81zXz$B*@rm04{>fE;=N?vqeu+A>*?JGSxAo{BC zk@g-3OGtMzxC`t#e?Di4Ji_^$31qIpuaIv<0ww%fL{b&g;`p=TV&!b6%K;s~*~=d& z==lao2}9`&TXaL0j|zN%4fdr0P!V|I&XDA>pWv{Uf_3kHbTjA)GoJV$F2QKR^uvKH z+{4DRwA1|`5v z(8vB#b>(vtu2gHagfE>Vj_s#5j$ogp<_s6kofU&a4lE^>sJ(aigx`K8-jo8hMSFB>+2DL zOr~zJ=|OycLxt@5{B;X&_Fczl29V-pv9GS2Qy8H6ka``f5T;fP4Rd0|IVRDj=-A5U zcwJ9(F?A6Ygv9FE7MAdJeB-U!_*cPfC|*_wW~Ac7M3Z46U<#`0@>e46=O2H;tGgc2 z5erSBNnnECsfUVr9(PuZg1hLc&f~Oz6rS=CFACsCp|?&SDPaJ{{{FDPfX>M}kFT9T ztc3T>F-)<}^xwWIi5aFu)+7d zv1ZHJNabBF9j4;%;q$(YR>7 z=J{VKTlC%kFX5kP{r?KCZqEC|{=A?1D1a_mR?R96dh5tN$#(*&k|>?Ucnm|Mn8a^w z*~;Z~Y^G*}0KKy*DH=8Kv-KlL5t5>&f5v=GUlHwmWuc-}MT<;bwQEk|+gqekb^EO| zcGdXMXntxaEzxg?{3;cv!#HDmEY?=y1+UmvNOmmSR%pYm-BzM(S-`E(?Ow&LkZ+%% z??Ecob9=`q?j57FcZ~9$G4lJ;tC_uHWcQAd+dIbo`PjGoHm(5wFdtk7t}yu{YrzeS zf)<3kNZnOobfOWt$VD!4k&9g9A{V*HMJ{rYi(KR)7rDqqE^?8JT;w7bxyVH>a*>N% bM>38 literal 0 HcmV?d00001 From 8f0757606d2cfbe4db6a6d0eaaa283cf27bd265b Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 29 Apr 2026 16:42:19 -0400 Subject: [PATCH 004/219] include salt..minion --- salt/salt/beacons.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/salt/beacons.sls b/salt/salt/beacons.sls index f3e519edd..bce3635f6 100644 --- a/salt/salt/beacons.sls +++ b/salt/salt/beacons.sls @@ -5,7 +5,7 @@ {% set SCHEDULE = salt['pillar.get']('healthcheck:schedule', 30) %} include: - - salt + - salt.minion {% if CHECKS and ENABLED %} salt_beacons: From f7d2994f8b66adc5def2aa72ce475ad0bbed31e6 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 30 Apr 2026 09:16:22 -0400 Subject: [PATCH 005/219] filter temp files --- salt/manager/beacons.sls | 21 ++++++++ .../files/beacons_pushstate.conf.jinja | 53 +++++++++++++++++++ salt/manager/init.sls | 2 + salt/salt/beacons.sls | 17 ------ salt/salt/files/beacons_pushstate.conf.jinja | 26 --------- 5 files changed, 76 insertions(+), 43 deletions(-) create mode 100644 salt/manager/beacons.sls create mode 100644 salt/manager/files/beacons_pushstate.conf.jinja delete mode 100644 salt/salt/files/beacons_pushstate.conf.jinja diff --git a/salt/manager/beacons.sls b/salt/manager/beacons.sls new file mode 100644 index 000000000..11574f9e2 --- /dev/null +++ b/salt/manager/beacons.sls @@ -0,0 +1,21 @@ +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'global/map.jinja' import GLOBALMERGED %} + +include: + - salt.minion + +{% if GLOBALS.is_manager and GLOBALMERGED.push.enabled %} +salt_beacons_pushstate: + file.managed: + - name: /etc/salt/minion.d/beacons_pushstate.conf + - source: salt://manager/files/beacons_pushstate.conf.jinja + - template: jinja + - watch_in: + - service: salt_minion_service +{% else %} +salt_beacons_pushstate: + file.absent: + - name: /etc/salt/minion.d/beacons_pushstate.conf + - watch_in: + - service: salt_minion_service +{% endif %} diff --git a/salt/manager/files/beacons_pushstate.conf.jinja b/salt/manager/files/beacons_pushstate.conf.jinja new file mode 100644 index 000000000..31a12159b --- /dev/null +++ b/salt/manager/files/beacons_pushstate.conf.jinja @@ -0,0 +1,53 @@ +beacons: + inotify: + - disable_during_state_run: True + - coalesce: True + - files: + /opt/so/saltstack/local/salt/suricata/rules/: + mask: + - close_write + - moved_to + - delete + recurse: True + auto_add: True + exclude: + - '\.sw[a-z]$': + regex: True + - '~$': + regex: True + - '/4913$': + regex: True + - '/\.#': + regex: True + /opt/so/saltstack/local/salt/strelka/rules/compiled/: + mask: + - close_write + - moved_to + - delete + recurse: True + auto_add: True + exclude: + - '\.sw[a-z]$': + regex: True + - '~$': + regex: True + - '/4913$': + regex: True + - '/\.#': + regex: True + /opt/so/saltstack/local/pillar/: + mask: + - close_write + - moved_to + - delete + recurse: True + auto_add: True + exclude: + - '\.sw[a-z]$': + regex: True + - '~$': + regex: True + - '/4913$': + regex: True + - '/\.#': + regex: True diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 2353bb64b..ef2428492 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -15,6 +15,7 @@ include: - manager.elasticsearch - manager.kibana - manager.managed_soc_annotations + - manager.beacons repo_log_dir: file.directory: @@ -231,6 +232,7 @@ surifiltersrules: - user: 939 - group: 939 + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/salt/beacons.sls b/salt/salt/beacons.sls index bce3635f6..2df927d75 100644 --- a/salt/salt/beacons.sls +++ b/salt/salt/beacons.sls @@ -1,5 +1,3 @@ -{% from 'vars/globals.map.jinja' import GLOBALS %} -{% from 'global/map.jinja' import GLOBALMERGED %} {% set CHECKS = salt['pillar.get']('healthcheck:checks', {}) %} {% set ENABLED = salt['pillar.get']('healthcheck:enabled', False) %} {% set SCHEDULE = salt['pillar.get']('healthcheck:schedule', 30) %} @@ -26,18 +24,3 @@ salt_beacons: - service: salt_minion_service {% endif %} -{% if GLOBALS.is_manager and GLOBALMERGED.push.enabled %} -salt_beacons_pushstate: - file.managed: - - name: /etc/salt/minion.d/beacons_pushstate.conf - - source: salt://salt/files/beacons_pushstate.conf.jinja - - template: jinja - - watch_in: - - service: salt_minion_service -{% else %} -salt_beacons_pushstate: - file.absent: - - name: /etc/salt/minion.d/beacons_pushstate.conf - - watch_in: - - service: salt_minion_service -{% endif %} diff --git a/salt/salt/files/beacons_pushstate.conf.jinja b/salt/salt/files/beacons_pushstate.conf.jinja deleted file mode 100644 index 8d3f05864..000000000 --- a/salt/salt/files/beacons_pushstate.conf.jinja +++ /dev/null @@ -1,26 +0,0 @@ -beacons: - inotify: - - disable_during_state_run: True - - coalesce: True - - files: - /opt/so/saltstack/local/salt/suricata/rules/: - mask: - - close_write - - moved_to - - delete - recurse: True - auto_add: True - /opt/so/saltstack/local/salt/strelka/rules/compiled/: - mask: - - close_write - - moved_to - - delete - recurse: True - auto_add: True - /opt/so/saltstack/local/pillar/: - mask: - - close_write - - moved_to - - delete - recurse: True - auto_add: True From 0d166ef732ebe483eb6f1d58840ea7da19817588 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 30 Apr 2026 09:53:00 -0400 Subject: [PATCH 006/219] remove trailing slashes --- salt/manager/files/beacons_pushstate.conf.jinja | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/manager/files/beacons_pushstate.conf.jinja b/salt/manager/files/beacons_pushstate.conf.jinja index 31a12159b..7eca37969 100644 --- a/salt/manager/files/beacons_pushstate.conf.jinja +++ b/salt/manager/files/beacons_pushstate.conf.jinja @@ -3,7 +3,7 @@ beacons: - disable_during_state_run: True - coalesce: True - files: - /opt/so/saltstack/local/salt/suricata/rules/: + /opt/so/saltstack/local/salt/suricata/rules: mask: - close_write - moved_to @@ -19,7 +19,7 @@ beacons: regex: True - '/\.#': regex: True - /opt/so/saltstack/local/salt/strelka/rules/compiled/: + /opt/so/saltstack/local/salt/strelka/rules/compiled: mask: - close_write - moved_to @@ -35,7 +35,7 @@ beacons: regex: True - '/\.#': regex: True - /opt/so/saltstack/local/pillar/: + /opt/so/saltstack/local/pillar: mask: - close_write - moved_to From 9541024eb77cd3af5f0582c03c0f419f7b0703d6 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 30 Apr 2026 15:35:24 -0400 Subject: [PATCH 007/219] fix broken things --- salt/manager/tools/sbin/so-push-drainer | 3 +-- salt/reactor/push_pillar.sls | 6 +++--- salt/reactor/push_strelka.sls | 6 +++--- salt/reactor/push_suricata.sls | 6 +++--- salt/salt/files/reactor_pushstate.conf | 12 +++++++++--- 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/salt/manager/tools/sbin/so-push-drainer b/salt/manager/tools/sbin/so-push-drainer index 3dc7bc8b3..4ce198fd0 100644 --- a/salt/manager/tools/sbin/so-push-drainer +++ b/salt/manager/tools/sbin/so-push-drainer @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/opt/saltstack/salt/bin/python3 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at @@ -35,7 +35,6 @@ import subprocess import sys import time -sys.path.append('/opt/saltstack/salt/lib/python3.10/site-packages/') import salt.client PENDING_DIR = '/opt/so/state/push_pending' diff --git a/salt/reactor/push_pillar.sls b/salt/reactor/push_pillar.sls index a31fe9be4..0e7fd40eb 100644 --- a/salt/reactor/push_pillar.sls +++ b/salt/reactor/push_pillar.sls @@ -19,7 +19,7 @@ import logging import os import time -import salt.client +from salt.client import Caller import yaml LOG = logging.getLogger(__name__) @@ -57,7 +57,7 @@ def _load_push_map(): def _push_enabled(): try: - caller = salt.client.Caller() + caller = Caller() return bool(caller.cmd('pillar.get', 'global:push:enabled', True)) except Exception: LOG.exception('push_pillar: pillar.get global:push:enabled failed, assuming enabled') @@ -132,7 +132,7 @@ def run(): LOG.info('push_pillar: push disabled, skipping') return {} - path = data.get('data', {}).get('path', '') # noqa: F821 -- data provided by reactor + path = data.get('path', '') # noqa: F821 -- data provided by reactor if not path or not path.startswith(PILLAR_ROOT): LOG.debug('push_pillar: ignoring path outside pillar root: %s', path) return {} diff --git a/salt/reactor/push_strelka.sls b/salt/reactor/push_strelka.sls index b3ed30ed7..1d6a2b044 100644 --- a/salt/reactor/push_strelka.sls +++ b/salt/reactor/push_strelka.sls @@ -14,7 +14,7 @@ import logging import os import time -import salt.client +from salt.client import Caller LOG = logging.getLogger(__name__) @@ -34,7 +34,7 @@ def _sensor_compound(): def _push_enabled(): try: - caller = salt.client.Caller() + caller = Caller() return bool(caller.cmd('pillar.get', 'global:push:enabled', True)) except Exception: LOG.exception('push_strelka: pillar.get global:push:enabled failed, assuming enabled') @@ -89,7 +89,7 @@ def run(): LOG.info('push_strelka: push disabled, skipping') return {} - path = data.get('data', {}).get('path', '') # noqa: F821 -- data provided by reactor + path = data.get('path', '') # noqa: F821 -- data provided by reactor actions = [{'state': 'strelka', 'tgt': _sensor_compound()}] _write_intent('rules_strelka', actions, path) LOG.info('push_strelka: intent updated for path=%s', path) diff --git a/salt/reactor/push_suricata.sls b/salt/reactor/push_suricata.sls index c9c6eee92..468249296 100644 --- a/salt/reactor/push_suricata.sls +++ b/salt/reactor/push_suricata.sls @@ -14,7 +14,7 @@ import logging import os import time -import salt.client +from salt.client import Caller LOG = logging.getLogger(__name__) @@ -33,7 +33,7 @@ def _sensor_compound_plus_import(): def _push_enabled(): try: - caller = salt.client.Caller() + caller = Caller() return bool(caller.cmd('pillar.get', 'global:push:enabled', True)) except Exception: LOG.exception('push_suricata: pillar.get global:push:enabled failed, assuming enabled') @@ -88,7 +88,7 @@ def run(): LOG.info('push_suricata: push disabled, skipping') return {} - path = data.get('data', {}).get('path', '') # noqa: F821 -- data provided by reactor + path = data.get('path', '') # noqa: F821 -- data provided by reactor actions = [{'state': 'suricata', 'tgt': _sensor_compound_plus_import()}] _write_intent('rules_suricata', actions, path) LOG.info('push_suricata: intent updated for path=%s', path) diff --git a/salt/salt/files/reactor_pushstate.conf b/salt/salt/files/reactor_pushstate.conf index 7d3a5a0d7..ceab284e2 100644 --- a/salt/salt/files/reactor_pushstate.conf +++ b/salt/salt/files/reactor_pushstate.conf @@ -1,7 +1,13 @@ reactor: - - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/suricata/rules/': + - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/suricata/rules': - salt://reactor/push_suricata.sls - - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/strelka/rules/compiled/': + - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/suricata/rules/*': + - salt://reactor/push_suricata.sls + - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/strelka/rules/compiled': - salt://reactor/push_strelka.sls - - 'salt/beacon/*/inotify//opt/so/saltstack/local/pillar/': + - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/strelka/rules/compiled/*': + - salt://reactor/push_strelka.sls + - 'salt/beacon/*/inotify//opt/so/saltstack/local/pillar': + - salt://reactor/push_pillar.sls + - 'salt/beacon/*/inotify//opt/so/saltstack/local/pillar/*': - salt://reactor/push_pillar.sls From 7fcace34c4b6fb35fa6219b0e0b26d1d020105e9 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 30 Apr 2026 16:09:08 -0400 Subject: [PATCH 008/219] add sensoroni to push map --- salt/reactor/pillar_push_map.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/reactor/pillar_push_map.yaml b/salt/reactor/pillar_push_map.yaml index 95fb0544e..7a1917eb2 100644 --- a/salt/reactor/pillar_push_map.yaml +++ b/salt/reactor/pillar_push_map.yaml @@ -98,6 +98,11 @@ redis: - state: redis tgt: 'G@role:so-heavynode or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-receiver or G@role:so-standalone' +# sensoroni: universal. +sensoroni: + - state: sensoroni + tgt: '*' + # soc: manager_roles exactly. soc: - state: soc From 932deab751b07aceacb500c304da575b68f5cd37 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 7 May 2026 10:51:53 -0400 Subject: [PATCH 009/219] update the push map --- salt/reactor/pillar_push_map.yaml | 113 +++++++++++++++++++++++++++++- 1 file changed, 110 insertions(+), 3 deletions(-) diff --git a/salt/reactor/pillar_push_map.yaml b/salt/reactor/pillar_push_map.yaml index 7a1917eb2..36699d3de 100644 --- a/salt/reactor/pillar_push_map.yaml +++ b/salt/reactor/pillar_push_map.yaml @@ -1,6 +1,8 @@ # One pillar directory can map to multiple (state, tgt) actions. # tgt is a raw salt compound expression. tgt_type is always "compound". # Per-action `batch` / `batch_wait` override the orch defaults (25% / 15s). +# An action with `highstate: True` triggers state.highstate instead of +# state.apply -- see salt/orch/push_batch.sls. # # Notes: # - `bpf` is a pillar-only dir (no state of its own) consumed by both @@ -8,9 +10,19 @@ # - suricata/strelka/zeek/elasticsearch/redis/kafka/logstash etc. have # their own pillar dirs AND their own state, so they map 1:1 (or 1:2 # in strelka's case, because of the split init.sls / manager.sls). -# - `data` and `node_data` pillar dirs are intentionally omitted -- -# they're pillar-only data consumed by many states; trying to handle -# them generically would amount to a highstate. +# +# Intentional omissions (these will log a "not in pillar_push_map.yaml" +# warning in push_pillar.sls and wait for the next scheduled highstate): +# - `data` and `node_data`: pillar-only data consumed by many states; +# handling them generically would amount to a fleetwide highstate. +# - `host`: soc_host describes mainint/mainip; a change is a re-IP and +# needs a coordinated procedure, not an immediate state push. +# - `hypervisor`: state changes touch libvirt and are disruptive; leave +# to the next scheduled highstate. +# - `sensor`: every field in soc_sensor.yaml is `readonly: True` or +# per-minion (`node: True`). Per-minion edits are persisted under +# pillar/minions/.sls and are handled by Branch A of push_pillar.sls +# (per-minion highstate intent), not by this app-pillar map. # # The role sets here were verified line-by-line against salt/top.sls. If # salt/top.sls changes how an app is targeted, update the corresponding @@ -28,6 +40,13 @@ firewall: - state: firewall tgt: '*' +# backup: backup.config_backup runs on eval, standalone, manager, managerhype, +# managersearch (NOT import -- the backup pillar is included on import per +# pillar/top.sls but the backup state is not run there per salt/top.sls). +backup: + - state: backup.config_backup + tgt: 'G@role:so-eval or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone' + # bpf is pillar-only (no state); consumed by both zeek and suricata as macros. # Both states run on sensor_roles + so-import per salt/top.sls. bpf: @@ -41,11 +60,22 @@ ca: - state: ca tgt: '*' +# docker: universal. The docker state is in both the all-non-managers and +# all-managers branches of salt/top.sls. +docker: + - state: docker + tgt: '*' + # elastalert: eval, standalone, manager, managerhype, managersearch (NOT import). elastalert: - state: elastalert tgt: 'G@role:so-eval or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone' +# elastic-fleet-package-registry: manager_roles exactly. +elastic-fleet-package-registry: + - state: elastic-fleet-package-registry + tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone' + # elasticsearch: 8 roles. elasticsearch: - state: elasticsearch @@ -62,11 +92,29 @@ elasticfleet: - state: elasticfleet tgt: 'G@role:so-eval or G@role:so-fleet or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone' +# global: fanout to a fleetwide highstate. The global pillar (soc_global.sls) +# carries cross-cutting settings (pipeline, url_base, imagerepo, mdengine, ...) +# that are consumed by virtually every state, so a targeted re-apply isn't +# meaningful. The drainer's batch/batch_wait throttling controls blast radius. +global: + - highstate: True + tgt: '*' + # healthcheck: eval, sensor, standalone only. healthcheck: - state: healthcheck tgt: 'G@role:so-eval or G@role:so-sensor or G@role:so-standalone' +# hydra: manager_roles exactly. +hydra: + - state: hydra + tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone' + +# idh: so-idh only. +idh: + - state: idh + tgt: 'G@role:so-idh' + # influxdb: manager_roles exactly. influxdb: - state: influxdb @@ -82,22 +130,61 @@ kibana: - state: kibana tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone' +# kratos: manager_roles exactly. +kratos: + - state: kratos + tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone' + +# logrotate: universal (top-of-file '*' branch in salt/top.sls). +logrotate: + - state: logrotate + tgt: '*' + # logstash: 8 roles, no eval/import. logstash: - state: logstash tgt: 'G@role:so-fleet or G@role:so-heavynode or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode or G@role:so-standalone' +# manager: manager_roles exactly. The manager state is also referenced under +# *_sensor / *_heavynode top.sls blocks via `sensor`, but the standalone +# `manager` state itself runs only on manager_roles. +manager: + - state: manager + tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone' + # nginx: 10 specific roles. NOT receiver, idh, hypervisor, desktop. nginx: - state: nginx tgt: 'G@role:so-eval or G@role:so-fleet or G@role:so-heavynode or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-searchnode or G@role:so-sensor or G@role:so-standalone' +# ntp: universal (top-of-file '*' branch in salt/top.sls). +ntp: + - state: ntp + tgt: '*' + +# patch: universal. soc_patch carries the OS update schedule, applied via +# patch.os.schedule on every node (it's in both the all-non-managers and +# all-managers branches of salt/top.sls). +patch: + - state: patch.os.schedule + tgt: '*' + +# postgres: manager_roles exactly. +postgres: + - state: postgres + tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone' + # redis: 6 roles. standalone, manager, managerhype, managersearch, heavynode, receiver. # (NOT eval, NOT import, NOT searchnode.) redis: - state: redis tgt: 'G@role:so-heavynode or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-receiver or G@role:so-standalone' +# registry: manager_roles exactly. +registry: + - state: registry + tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone' + # sensoroni: universal. sensoroni: - state: sensoroni @@ -108,6 +195,14 @@ soc: - state: soc tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone' +# stig: broad. Runs on standalone, manager, managerhype, managersearch, +# searchnode, sensor, receiver, fleet, hypervisor, desktop. +# NOT eval, NOT import, NOT heavynode, NOT idh (the *_idh block in +# salt/top.sls intentionally omits stig). +stig: + - state: stig + tgt: 'G@role:so-desktop or G@role:so-fleet or G@role:so-hypervisor or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode or G@role:so-sensor or G@role:so-standalone' + # strelka: sensor-side only on pillar change (sensor_roles). strelka.manager is # intentionally NOT fired on pillar changes -- YARA rule and strelka config # pillar changes are consumed by the sensor-side strelka backend, and re-running @@ -127,6 +222,18 @@ telegraf: - state: telegraf tgt: '*' +# versionlock: universal (top-of-file '*' branch in salt/top.sls). +versionlock: + - state: versionlock + tgt: '*' + +# vm: libvirt-driver hypervisors only. Matched by the salt-cloud:driver:libvirt +# grain (compound supports nested grain matching via G@::). +# pillar/vm/soc_vm.sls write path is referenced at salt/_runners/setup_hypervisor.py:856. +vm: + - state: vm + tgt: 'G@salt-cloud:driver:libvirt' + # zeek: sensor_roles + so-import (5 roles). zeek: - state: zeek From 778cc055eaa7a5948d51b0e440f959ebb0228672 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 7 May 2026 17:01:20 -0400 Subject: [PATCH 010/219] wait for salt-minion service to be ready before finishing state run --- salt/docker_clean/init.sls | 2 +- salt/salt/lasthighstate.sls | 2 +- salt/salt/master.sls | 3 +-- salt/salt/minion/init.sls | 37 +++++++++++++++++++++++++++++++++++-- 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/salt/docker_clean/init.sls b/salt/docker_clean/init.sls index ee60f5591..86dc19a8f 100644 --- a/salt/docker_clean/init.sls +++ b/salt/docker_clean/init.sls @@ -9,7 +9,7 @@ prune_images: cmd.run: - name: so-docker-prune - - order: last + - order: 9000 {% else %} diff --git a/salt/salt/lasthighstate.sls b/salt/salt/lasthighstate.sls index 606bd1082..88d060483 100644 --- a/salt/salt/lasthighstate.sls +++ b/salt/salt/lasthighstate.sls @@ -1,4 +1,4 @@ lasthighstate: file.touch: - name: /opt/so/log/salt/lasthighstate - - order: last \ No newline at end of file + - order: 9001 diff --git a/salt/salt/master.sls b/salt/salt/master.sls index b33d3631d..3833232fd 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -71,7 +71,6 @@ reactor_pushstate_config: - source: salt://salt/files/reactor_pushstate.conf - watch_in: - service: salt_master_service - - order: last {% else %} reactor_pushstate_config: file.absent: @@ -95,7 +94,7 @@ salt_master_service: - file: checkmine_engine - file: pillarWatch_engine - file: engines_config - - order: last + - order: 9002 {% else %} diff --git a/salt/salt/minion/init.sls b/salt/salt/minion/init.sls index eb7018aed..42f98de2a 100644 --- a/salt/salt/minion/init.sls +++ b/salt/salt/minion/init.sls @@ -88,13 +88,17 @@ enable_startup_states: {% endif %} -# this has to be outside the if statement above since there are _in calls to this state +# this has to be outside the if statement above since there are _in calls to this state. +# uses watch (not listen) so the restart fires in-state and its result lands on this state's +# running entry; that is what lets wait_for_salt_minion_ready below detect any restart +# uniformly via onchanges, regardless of whether the trigger came from these files or from +# external watch_in's (e.g. beacons, master/pyinotify). salt_minion_service: service.running: - name: salt-minion - enable: True - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}" - - listen: + - watch: - file: mine_functions {% if INSTALLEDSALTVERSION|string == SALTVERSION|string %} - file: set_log_levels @@ -103,3 +107,32 @@ salt_minion_service: - file: signing_policy {% endif %} - order: last + +# block until the just-restarted salt-minion is back and can execute modules locally, so +# follow-on jobs and the next highstate iteration do not race the restart. onchanges + +# require on salt_minion_service catches every restart trigger uniformly because watch +# mod_watch results replace the service state's running entry. initial sleep gives the +# systemctl restart (--no-block by default for salt-minion on >=3006.15) time to begin +# tearing down the old process before we probe for readiness. +wait_for_salt_minion_ready: + cmd.run: + - name: | + sleep 3 + timeout=120 + elapsed=3 + while [ $elapsed -lt $timeout ]; do + if systemctl is-active --quiet salt-minion \ + && salt-call --local --timeout=5 --out=quiet test.ping >/dev/null 2>&1; then + echo "salt-minion ready after ${elapsed}s" + exit 0 + fi + sleep 1 + elapsed=$((elapsed+1)) + done + echo "salt-minion did not become ready within ${timeout}s" >&2 + exit 1 + - onchanges: + - service: salt_minion_service + - require: + - service: salt_minion_service + - order: last From 66c0a662fc71a9738d16ce584c263b5d05daa427 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 8 May 2026 09:26:42 -0400 Subject: [PATCH 011/219] convert wait to script --- salt/common/tools/sbin/so-salt-minion-wait | 35 ++++++++++++++++++++++ salt/salt/minion/init.sls | 21 ++----------- 2 files changed, 38 insertions(+), 18 deletions(-) create mode 100644 salt/common/tools/sbin/so-salt-minion-wait diff --git a/salt/common/tools/sbin/so-salt-minion-wait b/salt/common/tools/sbin/so-salt-minion-wait new file mode 100644 index 000000000..a30c67e80 --- /dev/null +++ b/salt/common/tools/sbin/so-salt-minion-wait @@ -0,0 +1,35 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Block until the local salt-minion service is back up and can execute modules locally. +# Invoked from the wait_for_salt_minion_ready state in salt/minion/init.sls after +# salt_minion_service fires its watch-driven mod_watch (a non-blocking systemctl restart), +# so follow-on jobs and the next highstate iteration do not race the in-flight restart. + +. /usr/sbin/so-common + +# Initial sleep gives the systemctl restart (--no-block by default for salt-minion on +# >=3006.15) time to begin tearing down the old process before we probe for readiness. +INITIAL_SLEEP=3 +TIMEOUT=120 +PING_TIMEOUT=5 + +sleep "$INITIAL_SLEEP" + +elapsed="$INITIAL_SLEEP" +while [ "$elapsed" -lt "$TIMEOUT" ]; do + if systemctl is-active --quiet salt-minion \ + && salt-call --local --timeout="$PING_TIMEOUT" --out=quiet test.ping >/dev/null 2>&1; then + echo "salt-minion ready after ${elapsed}s" + exit 0 + fi + sleep 1 + elapsed=$((elapsed + 1)) +done + +echo "salt-minion did not become ready within ${TIMEOUT}s" >&2 +exit 1 diff --git a/salt/salt/minion/init.sls b/salt/salt/minion/init.sls index 42f98de2a..01c24e698 100644 --- a/salt/salt/minion/init.sls +++ b/salt/salt/minion/init.sls @@ -111,26 +111,11 @@ salt_minion_service: # block until the just-restarted salt-minion is back and can execute modules locally, so # follow-on jobs and the next highstate iteration do not race the restart. onchanges + # require on salt_minion_service catches every restart trigger uniformly because watch -# mod_watch results replace the service state's running entry. initial sleep gives the -# systemctl restart (--no-block by default for salt-minion on >=3006.15) time to begin -# tearing down the old process before we probe for readiness. +# mod_watch results replace the service state's running entry. wait logic lives in +# /usr/sbin/so-salt-minion-wait (deployed by common_sbin from common/tools/sbin/). wait_for_salt_minion_ready: cmd.run: - - name: | - sleep 3 - timeout=120 - elapsed=3 - while [ $elapsed -lt $timeout ]; do - if systemctl is-active --quiet salt-minion \ - && salt-call --local --timeout=5 --out=quiet test.ping >/dev/null 2>&1; then - echo "salt-minion ready after ${elapsed}s" - exit 0 - fi - sleep 1 - elapsed=$((elapsed+1)) - done - echo "salt-minion did not become ready within ${timeout}s" >&2 - exit 1 + - name: /usr/sbin/so-salt-minion-wait - onchanges: - service: salt_minion_service - require: From 7d4d6a075619a699833cb6693d13f0f9732d0018 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 8 May 2026 10:13:15 -0400 Subject: [PATCH 012/219] prune images if so-docker-prune exists --- salt/docker_clean/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/docker_clean/init.sls b/salt/docker_clean/init.sls index 86dc19a8f..65f43b1ef 100644 --- a/salt/docker_clean/init.sls +++ b/salt/docker_clean/init.sls @@ -9,6 +9,7 @@ prune_images: cmd.run: - name: so-docker-prune + - onlyif: command -v /usr/sbin/so-docker-prune >/dev/null 2>&1 - order: 9000 {% else %} From 61ca60a94cd5050067f8fd82548f9376c1efa7dd Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 13 May 2026 17:28:07 -0400 Subject: [PATCH 013/219] prep for soc db config --- salt/common/tools/sbin/so-log-check | 1 + salt/kratos/soc_kratos.yaml | 2 +- salt/postgres/config.sls | 6 +++--- salt/postgres/enabled.sls | 4 ++-- salt/postgres/files/{init-users.sh => init-db.sh} | 5 +++++ salt/postgres/telegraf_users.sls | 2 +- salt/soc/soc_soc.yaml | 1 + 7 files changed, 14 insertions(+), 7 deletions(-) rename salt/postgres/files/{init-users.sh => init-db.sh} (89%) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index a3d9c51d0..94fdd7229 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -165,6 +165,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading component template" # false positive (elasticsearch index or template names contain 'error') EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading composable template" # false positive (elasticsearch composable template names contain 'error') EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error while parsing document for index \[.ds-logs-kratos-so-.*object mapping for \[file\]" # false positive (mapping error occuring BEFORE kratos index has rolled over in 2.4.210) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|No such container" # false positive (telegraf trying to run stats on an old container) fi if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then diff --git a/salt/kratos/soc_kratos.yaml b/salt/kratos/soc_kratos.yaml index 4cfe2c1c3..267c4bc50 100644 --- a/salt/kratos/soc_kratos.yaml +++ b/salt/kratos/soc_kratos.yaml @@ -103,7 +103,7 @@ kratos: config: session: lifespan: - description: Defines the length of a login session. + description: Defines the length of a login session before it will timeout, and require a new login. global: True helpLink: kratos whoami: diff --git a/salt/postgres/config.sls b/salt/postgres/config.sls index 11ca52649..e458e8455 100644 --- a/salt/postgres/config.sls +++ b/salt/postgres/config.sls @@ -46,10 +46,10 @@ postgresinitdir: - require: - file: postgresconfdir -postgresinitusers: +postgresinitdb: file.managed: - - name: /opt/so/conf/postgres/init/init-users.sh - - source: salt://postgres/files/init-users.sh + - name: /opt/so/conf/postgres/init/init-db.sh + - source: salt://postgres/files/init-db.sh - user: 939 - group: 939 - mode: 755 diff --git a/salt/postgres/enabled.sls b/salt/postgres/enabled.sls index b3abb621e..79ef6f997 100644 --- a/salt/postgres/enabled.sls +++ b/salt/postgres/enabled.sls @@ -31,7 +31,7 @@ so-postgres: - POSTGRES_DB=securityonion # Passwords are delivered via mounted 0600 secret files, not plaintext env vars. # The upstream postgres image resolves POSTGRES_PASSWORD_FILE; entrypoint.sh and - # init-users.sh resolve SO_POSTGRES_PASS_FILE the same way. + # init-db.sh resolve SO_POSTGRES_PASS_FILE the same way. - POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password - SO_POSTGRES_USER={{ SO_POSTGRES_USER }} - SO_POSTGRES_PASS_FILE=/run/secrets/so_postgres_pass @@ -46,7 +46,7 @@ so-postgres: - /opt/so/conf/postgres/postgresql.conf:/conf/postgresql.conf:ro - /opt/so/conf/postgres/pg_hba.conf:/conf/pg_hba.conf:ro - /opt/so/conf/postgres/secrets:/run/secrets:ro - - /opt/so/conf/postgres/init/init-users.sh:/docker-entrypoint-initdb.d/init-users.sh:ro + - /opt/so/conf/postgres/init/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh:ro - /etc/pki/postgres.crt:/conf/postgres.crt:ro - /etc/pki/postgres.key:/conf/postgres.key:ro - /etc/pki/tls/certs/intca.crt:/conf/ca.crt:ro diff --git a/salt/postgres/files/init-users.sh b/salt/postgres/files/init-db.sh similarity index 89% rename from salt/postgres/files/init-users.sh rename to salt/postgres/files/init-db.sh index e28b11f0f..03e6d08dd 100644 --- a/salt/postgres/files/init-users.sh +++ b/salt/postgres/files/init-db.sh @@ -32,3 +32,8 @@ EOSQL if ! psql -U "$POSTGRES_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1; then psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE DATABASE so_telegraf" fi + +# Bootstrap the SOC database. +if ! psql -U "$POSTGRES_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='so_soc'" | grep -q 1; then + psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE DATABASE so_soc" +fi diff --git a/salt/postgres/telegraf_users.sls b/salt/postgres/telegraf_users.sls index 62490ea52..1ac7c80ed 100644 --- a/salt/postgres/telegraf_users.sls +++ b/salt/postgres/telegraf_users.sls @@ -39,7 +39,7 @@ postgres_wait_ready: - require: - docker_container: so-postgres -# Ensure the shared Telegraf database exists. init-users.sh only runs on a +# Ensure the shared Telegraf database exists. init-db.sh only runs on a # fresh data dir, so hosts upgraded onto an existing /nsm/postgres volume # would otherwise never get so_telegraf. postgres_create_telegraf_db: diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 6a2f79629..647bdd778 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -818,6 +818,7 @@ soc: description: List of available external tools visible in the SOC UI. Each tool is defined in JSON object notation, and must include the "name" key and "link" key, where the link is the tool's URL. global: True advanced: True + multiline: True forcedType: "[]{}" exportNodeId: description: The node ID on which export jobs will be executed. From 907f699721c1376e85ec2788fb2c994d61056117 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 14 May 2026 11:03:08 -0400 Subject: [PATCH 014/219] state rename --- salt/postgres/enabled.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/postgres/enabled.sls b/salt/postgres/enabled.sls index 79ef6f997..20d256ae8 100644 --- a/salt/postgres/enabled.sls +++ b/salt/postgres/enabled.sls @@ -70,7 +70,7 @@ so-postgres: - watch: - file: postgresconf - file: postgreshba - - file: postgresinitusers + - file: postgresinitdb - file: postgres_super_secret - file: postgres_app_secret - x509: postgres_crt @@ -78,7 +78,7 @@ so-postgres: - require: - file: postgresconf - file: postgreshba - - file: postgresinitusers + - file: postgresinitdb - file: postgres_super_secret - file: postgres_app_secret - x509: postgres_crt From fabecb82885d80efe96edf239f8b463f0103b8ef Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 14 May 2026 13:57:40 -0400 Subject: [PATCH 015/219] remove highstate from startup_states. highstate on system start --- salt/manager/sync_es_users.sls | 8 +++-- salt/salt/minion/boot_highstate.sls | 32 +++++++++++++++++++ salt/salt/minion/init.sls | 31 +++++++++++++++--- .../service/so-boot-highstate.service.jinja | 14 ++++++++ salt/setup/virt/setSalt.sls | 5 --- setup/so-functions | 15 +++++---- setup/so-setup | 2 +- 7 files changed, 88 insertions(+), 19 deletions(-) create mode 100644 salt/salt/minion/boot_highstate.sls create mode 100644 salt/salt/service/so-boot-highstate.service.jinja diff --git a/salt/manager/sync_es_users.sls b/salt/manager/sync_es_users.sls index 29b090e18..f452ff5fe 100644 --- a/salt/manager/sync_es_users.sls +++ b/salt/manager/sync_es_users.sls @@ -31,11 +31,13 @@ sync_es_users: - http: wait_for_kratos - file: so-user.lock # require so-user.lock file to be missing -# we dont want this added too early in setup, so we add the onlyif to verify 'startup_states: highstate' -# is in the minion config. That line is added before the final highstate during setup +# we dont want this added too early in setup, so the onlyif gates on the +# /opt/so/conf/setup-complete marker. The marker is written by +# mark_setup_complete in setup/so-functions just before the final setup +# highstate (and by an upgrade-path state for systems set up under the old gate). so-user_sync: cron.present: - user: root - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log' - identifier: so-user_sync - - onlyif: "grep -x 'startup_states: highstate' /etc/salt/minion" + - onlyif: "test -e /opt/so/conf/setup-complete" diff --git a/salt/salt/minion/boot_highstate.sls b/salt/salt/minion/boot_highstate.sls new file mode 100644 index 000000000..13bfbda65 --- /dev/null +++ b/salt/salt/minion/boot_highstate.sls @@ -0,0 +1,32 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Manages /etc/systemd/system/so-boot-highstate.service, a Type=oneshot +# RemainAfterExit=yes unit that runs `salt-call state.highstate` exactly once +# per system boot. Replaces the legacy `startup_states: highstate` minion +# config, which fired on every salt-minion service restart (causing a redundant +# highstate whenever a highstate itself restarted salt-minion). + +include: + - systemd.reload + +so_boot_highstate_unit_file: + file.managed: + - name: /etc/systemd/system/so-boot-highstate.service + - source: salt://salt/service/so-boot-highstate.service.jinja + - template: jinja + - onchanges_in: + - module: systemd_reload + +# Only enable once setup is complete. Until then the gate file is missing and +# the unit's own ConditionPathExists would no-op it anyway -- this just keeps +# `systemctl is-enabled` honest for the sync_es_users gate. +so_boot_highstate_service: + service.enabled: + - name: so-boot-highstate.service + - onlyif: test -e /opt/so/conf/setup-complete + - require: + - file: so_boot_highstate_unit_file + - module: systemd_reload diff --git a/salt/salt/minion/init.sls b/salt/salt/minion/init.sls index eb7018aed..0d0eed22c 100644 --- a/salt/salt/minion/init.sls +++ b/salt/salt/minion/init.sls @@ -17,6 +17,7 @@ include: - repo.client - salt.mine_functions - salt.minion.service_file + - salt.minion.boot_highstate {% if GLOBALS.is_manager %} - ca.signing_policy {% endif %} @@ -80,11 +81,33 @@ set_log_levels: - "log_level: info" - "log_level_logfile: info" -enable_startup_states: - file.uncomment: +# startup_states: highstate caused a full highstate to run on every +# salt-minion service start, including the restart triggered when a highstate +# itself modified the minion config (beacons, mine, unit file). Replaced by +# so-boot-highstate.service (managed in salt.minion.boot_highstate), which +# runs once per system boot only. Strip the line from /etc/salt/minion on +# upgrade; both the commented and uncommented forms historically existed. +remove_startup_states: + file.line: - name: /etc/salt/minion - - regex: '^startup_states: highstate$' - - unless: pgrep so-setup + - match: 'startup_states: highstate' + - mode: delete + +# Upgrade-path bridge: systems that already passed setup under the old gate +# (`grep -x 'startup_states: highstate' /etc/salt/minion`) get a setup-complete +# marker so so-boot-highstate.service can be enabled and the so-user_sync cron +# in sync_es_users.sls keeps installing. Setup-in-progress systems instead get +# the marker from `mark_setup_complete` in setup/so-functions at the right +# moment. `replace: false` means we never overwrite a marker once written. +mark_setup_complete_for_upgrades: + file.managed: + - name: /opt/so/conf/setup-complete + - replace: false + - makedirs: True + - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion" + - require_in: + - file: remove_startup_states + - service: so_boot_highstate_service {% endif %} diff --git a/salt/salt/service/so-boot-highstate.service.jinja b/salt/salt/service/so-boot-highstate.service.jinja new file mode 100644 index 000000000..f3ec950c3 --- /dev/null +++ b/salt/salt/service/so-boot-highstate.service.jinja @@ -0,0 +1,14 @@ +[Unit] +Description=Security Onion boot-time highstate (runs once per boot) +After=salt-minion.service network-online.target +Wants=network-online.target +Requires=salt-minion.service +ConditionPathExists=/opt/so/conf/setup-complete + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/bin/salt-call state.highstate -l info queue=True + +[Install] +WantedBy=multi-user.target diff --git a/salt/setup/virt/setSalt.sls b/salt/setup/virt/setSalt.sls index 69c8795de..59ab9e1e3 100644 --- a/salt/setup/virt/setSalt.sls +++ b/salt/setup/virt/setSalt.sls @@ -8,11 +8,6 @@ set_role_grain: - name: role - value: so-{{ grains.id.split("_") | last }} -set_highstate: - file.append: - - name: /etc/salt/minion - - text: 'startup_states: highstate' - enable_salt_minion: service.enabled: - name: salt-minion diff --git a/setup/so-functions b/setup/so-functions index c94b8eee7..da8e31d73 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -539,16 +539,19 @@ configure_minion() { " x509_v2: true"\ "log_level: info"\ "log_level_logfile: info"\ - "log_file: /opt/so/log/salt/minion"\ - "#startup_states: highstate" >> "$minion_config" + "log_file: /opt/so/log/salt/minion" >> "$minion_config" } -checkin_at_boot() { - local minion_config=/etc/salt/minion +mark_setup_complete() { + # Writes the setup-complete marker. Salt's so-boot-highstate.service + # (boot-time oneshot) and the so-user_sync cron gate in + # salt/manager/sync_es_users.sls both key off this file. + local marker=/opt/so/conf/setup-complete - info "Enabling checkin at boot" - sed -i 's/#startup_states: highstate/startup_states: highstate/' "$minion_config" + info "Marking setup as complete" + mkdir -p "$(dirname "$marker")" + touch "$marker" } check_requirements() { diff --git a/setup/so-setup b/setup/so-setup index 6c77e781c..c11d287eb 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -792,7 +792,7 @@ if ! [[ -f $install_opt_file ]]; then error "Failed to run so-elastic-fleet-setup" fail_setup fi - checkin_at_boot + mark_setup_complete set_initial_firewall_access initialize_elasticsearch_indices "so-case so-casehistory so-assistant-session so-assistant-chat" # run a final highstate before enabling scheduled highstates. From 89a28d2cfeed5729fd31f4e113d6448537ffcd38 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 21 May 2026 15:45:58 -0400 Subject: [PATCH 016/219] Bump version from 3.1.0 to 3.2.0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index fd2a01863..944880fa1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.0 +3.2.0 From aa7897874034e9e4383c06780a93d9bc2312bf40 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 21 May 2026 15:57:57 -0400 Subject: [PATCH 017/219] Add 3.2.0 option to discussion template --- .github/DISCUSSION_TEMPLATE/3-0.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/DISCUSSION_TEMPLATE/3-0.yml b/.github/DISCUSSION_TEMPLATE/3-0.yml index 3fb9e5b30..8f74145c4 100644 --- a/.github/DISCUSSION_TEMPLATE/3-0.yml +++ b/.github/DISCUSSION_TEMPLATE/3-0.yml @@ -11,6 +11,7 @@ body: - - 3.0.0 - 3.1.0 + - 3.2.0 - Other (please provide detail below) validations: required: true From 79987f3659ab554b975e31461ed20a21d009383d Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 27 May 2026 13:55:30 -0400 Subject: [PATCH 018/219] bootstrap so-soc db in postgres during soup --- salt/manager/tools/sbin/soup | 35 ++++++++++++++++++++++- salt/postgres/telegraf_users.sls | 18 ++---------- salt/postgres/tools/sbin/so-postgres-wait | 32 +++++++++++++++++++++ 3 files changed, 68 insertions(+), 17 deletions(-) create mode 100644 salt/postgres/tools/sbin/so-postgres-wait diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 3bec13716..05f58b9a5 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -370,8 +370,9 @@ preupgrade_changes() { # This function is to add any new pillar items if needed. echo "Checking to see if changes are needed." - [[ "$INSTALLEDVERSION" =~ ^2\.4\.21[0-9]+$ ]] && up_to_3.0.0 + [[ "$INSTALLEDVERSION" =~ ^2\.4\.21[0-9]+$ ]] && up_to_3.0.0 [[ "$INSTALLEDVERSION" == "3.0.0" ]] && up_to_3.1.0 + [[ "$INSTALLEDVERSION" == "3.1.0" ]] && up_to_3.2.0 true } @@ -381,6 +382,7 @@ postupgrade_changes() { [[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0 [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0 + [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0 true } @@ -720,6 +722,37 @@ post_to_3.1.0() { ### 3.1.0 End ### +### 3.2.0 Scripts ### + +bootstrap_so_soc_database() { + # init-db.sh is mounted into so-postgres at /docker-entrypoint-initdb.d/init-db.sh + # and runs automatically only on a fresh data directory. Hosts upgrading from + # 3.1.0 already have /nsm/postgres populated, so the so_soc bootstrap block + # added in 3.2 never fires. Re-run the script explicitly; it's idempotent. + echo "Bootstrapping so_soc database via init-db.sh." + if ! /usr/sbin/so-postgres-wait; then + FINAL_MESSAGE_QUEUE+=("WARNING: so-postgres was not ready during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: docker exec so-postgres bash /docker-entrypoint-initdb.d/init-db.sh") + return 0 + fi + if ! docker exec so-postgres bash /docker-entrypoint-initdb.d/init-db.sh; then + FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: docker exec so-postgres bash /docker-entrypoint-initdb.d/init-db.sh") + return 0 + fi + echo "so_soc bootstrap complete." +} + +up_to_3.2.0() { + INSTALLEDVERSION=3.2.0 +} + +post_to_3.2.0() { + bootstrap_so_soc_database + + POSTVERSION=3.2.0 +} + +### 3.2.0 End ### + repo_sync() { echo "Sync the local repo." diff --git a/salt/postgres/telegraf_users.sls b/salt/postgres/telegraf_users.sls index 28d9d6247..5e3566a95 100644 --- a/salt/postgres/telegraf_users.sls +++ b/salt/postgres/telegraf_users.sls @@ -18,26 +18,12 @@ include: {% set TG_OUT = TELEGRAFMERGED.output | upper %} {% if TG_OUT in ['POSTGRES', 'BOTH'] %} -# docker_container.running returns as soon as the container starts, but on -# first-init docker-entrypoint.sh starts a temporary postgres with -# `listen_addresses=''` to run /docker-entrypoint-initdb.d scripts, then -# shuts it down before exec'ing the real CMD. A default pg_isready check -# (Unix socket) passes during that ephemeral phase and races the shutdown -# with "the database system is shutting down". Checking TCP readiness on -# 127.0.0.1 only succeeds after the final postgres binds the port. postgres_wait_ready: cmd.run: - - name: | - for i in $(seq 1 60); do - if docker exec so-postgres pg_isready -h 127.0.0.1 -U postgres -q 2>/dev/null; then - exit 0 - fi - sleep 2 - done - echo "so-postgres did not accept TCP connections within 120s" >&2 - exit 1 + - name: /usr/sbin/so-postgres-wait - require: - docker_container: so-postgres + - file: postgres_sbin # Ensure the shared Telegraf database exists. init-db.sh only runs on a # fresh data dir, so hosts upgraded onto an existing /nsm/postgres volume diff --git a/salt/postgres/tools/sbin/so-postgres-wait b/salt/postgres/tools/sbin/so-postgres-wait new file mode 100644 index 000000000..7c4c8ce92 --- /dev/null +++ b/salt/postgres/tools/sbin/so-postgres-wait @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Wait for the so-postgres container to accept TCP connections. +# +# docker_container.running returns as soon as the container starts, but on +# first-init docker-entrypoint.sh starts a temporary postgres with +# `listen_addresses=''` to run /docker-entrypoint-initdb.d scripts, then +# shuts it down before exec'ing the real CMD. A default pg_isready check +# (Unix socket) passes during that ephemeral phase and races the shutdown +# with "the database system is shutting down". Checking TCP readiness on +# 127.0.0.1 only succeeds after the final postgres binds the port. +# +# Usage: so-postgres-wait [iterations] [sleep_seconds] +# Default: 60 iterations, 2s sleep (~120s total). + +ITERATIONS=${1:-60} +SLEEP_SECONDS=${2:-2} + +for i in $(seq 1 "$ITERATIONS"); do + if docker exec so-postgres pg_isready -h 127.0.0.1 -U postgres -q 2>/dev/null; then + exit 0 + fi + sleep "$SLEEP_SECONDS" +done + +echo "so-postgres did not accept TCP connections within $((ITERATIONS * SLEEP_SECONDS))s" >&2 +exit 1 From 93ffce98d71d332e7743056b0d36f3aacbbab24d Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 27 May 2026 15:07:25 -0400 Subject: [PATCH 019/219] add onionconfig and postgres modules to soc config --- salt/soc/defaults.yaml | 6 ++++++ salt/soc/merged.map.jinja | 7 +++++++ salt/soc/soc_soc.yaml | 20 ++++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index cc80758fc..62b451bec 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1519,6 +1519,12 @@ soc: serviceAccountJSON: "" serviceAccountLocation: "" healthTimeoutSeconds: 5 + onionconfig: + saltstackDir: /opt/so/saltstack + bypassEnabled: false + postgres: + host: + password: salt: queueDir: /opt/sensoroni/queue timeoutMs: 45000 diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index 349937983..b34efb11d 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -16,6 +16,13 @@ {% do SOCMERGED.config.server.update({'additionalCA': MANAGERMERGED.additionalCA}) %} {% do SOCMERGED.config.server.update({'insecureSkipVerify': MANAGERMERGED.insecureSkipVerify}) %} +{% if not SOCMERGED.config.server.modules.postgres.host %} +{% do SOCMERGED.config.server.modules.postgres.update({'host': GLOBALS.manager}) %} +{% endif %} +{% if not SOCMERGED.config.server.modules.postgres.password %} +{% do SOCMERGED.config.server.modules.postgres.update({'password': salt['pillar.get']('secrets:postgres_pass', '')}) %} +{% endif %} + {# if SOCMERGED.config.server.modules.cases == httpcase details come from the soc pillar #} {% if SOCMERGED.config.server.modules.cases != 'soc' %} {% do SOCMERGED.config.server.modules.elastic.update({'casesEnabled': false}) %} diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 647bdd778..3cb244eed 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -453,6 +453,26 @@ soc: description: Duration (in milliseconds) that must elapse after a grid node fails to check-in before the node will be marked offline (fault). global: True advanced: True + onionconfig: + saltstackDir: + description: Root directory containing the SaltStack tree that SOC reads and writes configuration from. Should not be changed under normal circumstances. + global: True + advanced: True + bypassEnabled: + description: When enabled, errors encountered while reading the SaltStack pillar tree (missing files, unreadable directories, etc.) are logged but do not prevent SOC from starting or serving settings. Intended for advanced troubleshooting and recovery scenarios when the pillar tree is partially unreadable. + global: True + advanced: True + forcedType: bool + postgres: + host: + description: Hostname or IP address of the PostgreSQL server used by SOC. Defaults to the manager hostname. + global: True + advanced: True + password: + description: Password used by SOC to authenticate to the PostgreSQL server. Defaults to the postgres superuser password seeded in the secrets pillar. + global: True + sensitive: True + advanced: True salt: longRelayTimeoutMs: description: Duration (in milliseconds) to wait for a response from the Salt API when executing tasks known for being long running before giving up and showing an error on the SOC UI. From bb8ae91d91936d84dbf1e61617e8bba59c66a9f8 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 27 May 2026 16:39:52 -0400 Subject: [PATCH 020/219] fix so-soc postgres bootstrap --- salt/manager/tools/sbin/soup | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 05f58b9a5..c31891f1d 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -730,12 +730,17 @@ bootstrap_so_soc_database() { # 3.1.0 already have /nsm/postgres populated, so the so_soc bootstrap block # added in 3.2 never fires. Re-run the script explicitly; it's idempotent. echo "Bootstrapping so_soc database via init-db.sh." + # The postgres image has no USER directive, so `docker exec` defaults to + # root, and the container env intentionally omits POSTGRES_USER (the upstream + # entrypoint defaults it transiently during first-init only). Recreate both + # so psql inside init-db.sh resolves the connect user correctly. + local exec_cmd="docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh" if ! /usr/sbin/so-postgres-wait; then - FINAL_MESSAGE_QUEUE+=("WARNING: so-postgres was not ready during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: docker exec so-postgres bash /docker-entrypoint-initdb.d/init-db.sh") + FINAL_MESSAGE_QUEUE+=("WARNING: so-postgres was not ready during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd") return 0 fi - if ! docker exec so-postgres bash /docker-entrypoint-initdb.d/init-db.sh; then - FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: docker exec so-postgres bash /docker-entrypoint-initdb.d/init-db.sh") + if ! $exec_cmd; then + FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd") return 0 fi echo "so_soc bootstrap complete." From 68d783e7607703a3024286d43130f2fc24278e9f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 28 May 2026 10:24:47 -0400 Subject: [PATCH 021/219] Remove outdated HOTFIX version number --- HOTFIX | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HOTFIX b/HOTFIX index 70406bf9d..8b1378917 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1 +1 @@ -20260528 + From 86edc5aaba451d571d2cb68f703745ca0d0d5843 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 28 May 2026 22:57:59 -0400 Subject: [PATCH 022/219] version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 944880fa1..03e153fda 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.2.0 +3.0.0-kilo From f54939b444313ab9a68839e010bfa78c5602562f Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 29 May 2026 14:38:59 -0400 Subject: [PATCH 023/219] Replace inotify pillar watch with postgres audit_settings beacon The active-push feature detected pillar/settings changes via an inotify beacon on the manager watching /opt/so/saltstack/local/pillar. Replace that pillar watch with a custom salt beacon (pillar_db) that polls the SOC so_soc.audit_settings table on a monotonic id watermark, so changes made through SOC drive immediate pushes from the database instead of the files. The suricata/strelka rule inotify watches (and pyinotify) are kept unchanged, since rule-file edits are not recorded in audit_settings. - salt/_beacons/pillar_db.py: new beacon. Polls audit_settings via `docker exec so-postgres psql` (unix-socket trust auth), tracks the last processed id in /opt/so/state/pillar_db_watch.id, seeds to MAX(id) on first run (no history replay), and emits one event per new row. - salt/reactor/push_pillar.sls: consume setting_id/node_id from the beacon event instead of a file path. App = first dotted segment of setting_id, looked up in pillar_push_map.yaml. Empty node_id -> grid-wide actions as is; populated node_id -> the app's state(s) retargeted to that one node. - salt/manager/files/beacons_pushstate.conf.jinja: drop the pillar inotify block, add the pillar_db beacon (interval = push.drain_interval); keep the suricata/strelka inotify watches. - salt/salt/files/reactor_pushstate.conf: map salt/beacon/*/pillar_db/ audit_settings to push_pillar.sls; remove the pillar inotify reactor lines; keep suricata/strelka. The intent -> so-push-drainer -> orch.push_batch pipeline is unchanged. Verified end-to-end on a standalone: a grid-wide telegraf.output change re-applied telegraf fleetwide (container replaced), and a per-host ntp.config.servers change applied ntp to only that node. --- salt/_beacons/pillar_db.py | 142 ++++++++++++++++++ .../files/beacons_pushstate.conf.jinja | 20 +-- salt/reactor/push_pillar.sls | 104 +++++++------ salt/salt/files/reactor_pushstate.conf | 4 +- 4 files changed, 202 insertions(+), 68 deletions(-) create mode 100644 salt/_beacons/pillar_db.py diff --git a/salt/_beacons/pillar_db.py b/salt/_beacons/pillar_db.py new file mode 100644 index 000000000..9022d9b87 --- /dev/null +++ b/salt/_beacons/pillar_db.py @@ -0,0 +1,142 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Custom salt beacon that watches the SOC audit_settings table in postgres for +# new settings changes and emits a beacon event per new row. This replaces the +# inotify watch on /opt/so/saltstack/local/pillar -- instead of monitoring pillar +# files on disk, we monitor the so_soc.audit_settings table that SOC writes to. +# +# Detection is poll-based with a monotonic `id` watermark persisted to +# WATERMARK_FILE: each pass selects rows with id greater than the last id seen, +# which makes it self-healing (a missed poll simply catches up on the next one). +# +# Each emitted event carries setting_id and node_id; the push_pillar reactor maps +# setting_id -> app via pillar_push_map.yaml and writes a push intent, after which +# the existing so-push-drainer / orch.push_batch pipeline takes over unchanged. + +import logging +import os +import subprocess + +log = logging.getLogger(__name__) + +WATERMARK_FILE = '/opt/so/state/pillar_db_watch.id' +CONTAINER = 'so-postgres' +DATABASE = 'so_soc' + +# Unaligned, tuples-only psql output with a field separator that cannot appear in +# an id/setting_id/node_id, so we can split each row reliably. +FIELD_SEP = '\x1f' + + +def __virtual__(): + return True + + +def validate(config): + return True, 'valid' + + +def _read_watermark(): + # Returns the last processed id, or None if the watermark has not been seeded. + try: + with open(WATERMARK_FILE, 'r') as f: + return int((f.read() or '').strip()) + except (IOError, ValueError): + return None + + +def _write_watermark(value): + try: + os.makedirs(os.path.dirname(WATERMARK_FILE), exist_ok=True) + tmp = WATERMARK_FILE + '.tmp' + with open(tmp, 'w') as f: + f.write(str(int(value))) + os.rename(tmp, WATERMARK_FILE) + except OSError: + log.exception('pillar_db beacon: failed to persist watermark to %s', WATERMARK_FILE) + + +def _query(sql): + # Run a query against so_soc inside the so-postgres container over the unix + # socket (trust auth, no password). Returns stdout on success, or None on any + # failure so the caller can no-op and retry on the next interval. + cmd = [ + 'docker', 'exec', CONTAINER, + 'psql', '-U', 'postgres', '-d', DATABASE, + '-tA', '-F', FIELD_SEP, '-c', sql, + ] + try: + result = subprocess.run(cmd, capture_output=True, text=True, timeout=30) + except subprocess.TimeoutExpired: + log.warning('pillar_db beacon: psql timed out') + return None + except Exception: + log.exception('pillar_db beacon: failed to exec psql') + return None + if result.returncode != 0: + log.warning('pillar_db beacon: psql failed (rc=%s): %s', + result.returncode, (result.stderr or '').strip()) + return None + return result.stdout + + +def beacon(config): + retval = [] + + watermark = _read_watermark() + + # First run / missing watermark: seed to the current MAX(id) and emit nothing + # so we never replay the entire settings history into a fleetwide push. + if watermark is None: + seed = _query('SELECT COALESCE(MAX(id), 0) FROM audit_settings;') + if seed is None: + return retval # postgres not ready yet; retry next interval + try: + _write_watermark(int((seed or '0').strip() or 0)) + except ValueError: + log.warning('pillar_db beacon: could not parse MAX(id) seed: %r', seed) + return retval + + rows = _query( + "SELECT id, setting_id, COALESCE(node_id, '') FROM audit_settings " + "WHERE id > %d ORDER BY id;" % watermark + ) + if rows is None: + return retval + + max_id = watermark + for line in rows.splitlines(): + # Do NOT str.strip() the whole line: Python treats the \x1f field + # separator (and \x1c-\x1e) as whitespace, so stripping would eat an + # empty trailing node_id field and make the row look malformed. + if not line.strip(): + continue + parts = line.split(FIELD_SEP) + if len(parts) < 3: + log.warning('pillar_db beacon: skipping malformed row: %r', line) + continue + try: + row_id = int(parts[0]) + except ValueError: + log.warning('pillar_db beacon: skipping row with non-int id: %r', line) + continue + setting_id = parts[1] + node_id = parts[2] + retval.append({ + 'tag': 'audit_settings', + 'id': row_id, + 'setting_id': setting_id, + 'node_id': node_id, + }) + if row_id > max_id: + max_id = row_id + + if max_id > watermark: + _write_watermark(max_id) + log.info('pillar_db beacon: emitted %d change(s), watermark %d -> %d', + len(retval), watermark, max_id) + + return retval diff --git a/salt/manager/files/beacons_pushstate.conf.jinja b/salt/manager/files/beacons_pushstate.conf.jinja index 7eca37969..ce9259359 100644 --- a/salt/manager/files/beacons_pushstate.conf.jinja +++ b/salt/manager/files/beacons_pushstate.conf.jinja @@ -1,4 +1,8 @@ +{% from 'global/map.jinja' import GLOBALMERGED %} beacons: + pillar_db: + - interval: {{ GLOBALMERGED.push.drain_interval }} + - disable_during_state_run: True inotify: - disable_during_state_run: True - coalesce: True @@ -35,19 +39,3 @@ beacons: regex: True - '/\.#': regex: True - /opt/so/saltstack/local/pillar: - mask: - - close_write - - moved_to - - delete - recurse: True - auto_add: True - exclude: - - '\.sw[a-z]$': - regex: True - - '~$': - regex: True - - '/4913$': - regex: True - - '/\.#': - regex: True diff --git a/salt/reactor/push_pillar.sls b/salt/reactor/push_pillar.sls index 0e7fd40eb..06ee474df 100644 --- a/salt/reactor/push_pillar.sls +++ b/salt/reactor/push_pillar.sls @@ -1,17 +1,21 @@ #!py -# Reactor invoked by the inotify beacon on pillar file changes under -# /opt/so/saltstack/local/pillar/. +# Reactor invoked by the pillar_db beacon when SOC records settings changes in +# the so_soc.audit_settings table (see salt/_beacons/pillar_db.py). The beacon +# emits one event per new row carrying setting_id and node_id. # -# Two branches: -# A) per-minion override under pillar/minions/.sls or adv_.sls -# -> write an intent that runs state.highstate on just that minion. -# B) shared app pillar (pillar//...) -> look up in -# pillar_push_map.yaml and write an intent with the entry's actions. +# Two branches, keyed on node_id: +# A) node_id populated -> the change is scoped to that one minion. Look up the +# app in pillar_push_map.yaml and write an intent that runs the app's mapped +# state(s) targeted to just that node. +# B) node_id empty -> grid-wide app change. Look up the app in +# pillar_push_map.yaml and write an intent with the entry's actions as-is. +# +# The app name is the first dotted segment of setting_id (e.g. "telegraf.output" +# -> "telegraf"), which matches the pillar_push_map.yaml keys 1:1. # # Reactors never dispatch directly. The so-push-drainer schedule picks up # ready intents, dedupes across pending files, and dispatches orch.push_batch. -# See plan /home/mreeves/.claude/plans/goofy-marinating-hummingbird.md. import fcntl import json @@ -28,9 +32,6 @@ PENDING_DIR = '/opt/so/state/push_pending' LOCK_FILE = os.path.join(PENDING_DIR, '.lock') MAX_PATHS = 20 -PILLAR_ROOT = '/opt/so/saltstack/local/pillar/' -MINIONS_PREFIX = PILLAR_ROOT + 'minions/' - # The pillar_push_map.yaml is shipped via salt:// but the reactor runs on the # master, which mounts the default saltstack tree at this path. PUSH_MAP_PATH = '/opt/so/saltstack/default/salt/reactor/pillar_push_map.yaml' @@ -107,24 +108,25 @@ def _write_intent(key, actions, path): os.close(lock_fd) -def _minion_id_from_path(path): - # path is e.g. /opt/so/saltstack/local/pillar/minions/sensor1.sls - # or /opt/so/saltstack/local/pillar/minions/adv_sensor1.sls - filename = os.path.basename(path) - if not filename.endswith('.sls'): +def _app_from_setting(setting_id): + # setting_id is e.g. 'telegraf.output' -> 'telegraf', 'ntp.config.servers' -> 'ntp' + if not setting_id: return None - stem = filename[:-4] - if stem.startswith('adv_'): - stem = stem[4:] - return stem or None + return setting_id.split('.', 1)[0] or None -def _app_from_path(path): - # path is e.g. /opt/so/saltstack/local/pillar/zeek/soc_zeek.sls -> 'zeek' - remainder = path[len(PILLAR_ROOT):] - if '/' not in remainder: - return None - return remainder.split('/', 1)[0] or None +def _node_actions(entry, node_id): + # Copy the app's mapped actions but retarget each one to the single node. + # Preserves the state/highstate selection and any batch/batch_wait overrides. + actions = [] + for action in entry: + if not isinstance(action, dict): + continue + node_action = dict(action) + node_action['tgt'] = node_id + node_action['tgt_type'] = 'glob' + actions.append(node_action) + return actions def run(): @@ -132,39 +134,43 @@ def run(): LOG.info('push_pillar: push disabled, skipping') return {} - path = data.get('path', '') # noqa: F821 -- data provided by reactor - if not path or not path.startswith(PILLAR_ROOT): - LOG.debug('push_pillar: ignoring path outside pillar root: %s', path) - return {} + # The pillar_db beacon nests its payload under data['data']; fall back to the + # top level so the reactor is robust to either shape. + event = data.get('data', data) # noqa: F821 -- data provided by reactor + setting_id = event.get('setting_id', '') + node_id = (event.get('node_id') or '').strip() - # Branch A: per-minion override - if path.startswith(MINIONS_PREFIX): - minion_id = _minion_id_from_path(path) - if not minion_id: - LOG.debug('push_pillar: ignoring non-sls path under minions/: %s', path) - return {} - actions = [{'highstate': True, 'tgt': minion_id, 'tgt_type': 'glob'}] - _write_intent('minion_{}'.format(minion_id), actions, path) - LOG.info('push_pillar: per-minion intent updated for %s (path=%s)', minion_id, path) - return {} - - # Branch B: shared app pillar -> allowlist lookup - app = _app_from_path(path) + app = _app_from_setting(setting_id) if not app: - LOG.debug('push_pillar: ignoring path with no app segment: %s', path) + LOG.debug('push_pillar: ignoring event with no app segment: setting_id=%s', setting_id) return {} push_map = _load_push_map() entry = push_map.get(app) if not entry: LOG.warning( - 'push_pillar: pillar dir "%s" is not in pillar_push_map.yaml; ' - 'change will be picked up at the next scheduled highstate (path=%s)', - app, path, + 'push_pillar: app "%s" is not in pillar_push_map.yaml; change will be ' + 'picked up at the next scheduled highstate (setting_id=%s)', + app, setting_id, ) return {} + # Branch A: per-node change -> retarget the app's states to just that node. + if node_id: + actions = _node_actions(entry, node_id) + if not actions: + LOG.warning('push_pillar: no usable actions for app "%s" (setting_id=%s)', app, setting_id) + return {} + _write_intent( + 'node_{}_{}'.format(node_id, app), actions, + 'audit:{}@{}'.format(setting_id, node_id), + ) + LOG.info('push_pillar: per-node intent updated for %s on %s (setting_id=%s)', + app, node_id, setting_id) + return {} + + # Branch B: grid-wide app change -> use the map entry's actions as-is. actions = list(entry) # copy to avoid mutating the cache - _write_intent('pillar_{}'.format(app), actions, path) - LOG.info('push_pillar: app intent updated for %s (path=%s)', app, path) + _write_intent('pillar_{}'.format(app), actions, 'audit:{}'.format(setting_id)) + LOG.info('push_pillar: app intent updated for %s (setting_id=%s)', app, setting_id) return {} diff --git a/salt/salt/files/reactor_pushstate.conf b/salt/salt/files/reactor_pushstate.conf index ceab284e2..991c4d516 100644 --- a/salt/salt/files/reactor_pushstate.conf +++ b/salt/salt/files/reactor_pushstate.conf @@ -7,7 +7,5 @@ reactor: - salt://reactor/push_strelka.sls - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/strelka/rules/compiled/*': - salt://reactor/push_strelka.sls - - 'salt/beacon/*/inotify//opt/so/saltstack/local/pillar': - - salt://reactor/push_pillar.sls - - 'salt/beacon/*/inotify//opt/so/saltstack/local/pillar/*': + - 'salt/beacon/*/pillar_db/audit_settings': - salt://reactor/push_pillar.sls From 68a82a425b155ea0dca567278990bc90635504c5 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 30 May 2026 08:12:50 -0400 Subject: [PATCH 024/219] fix version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 03e153fda..944880fa1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.0.0-kilo +3.2.0 From 79da9f9f2c011dbfb868cc02071b703666b3afb1 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 1 Jun 2026 11:26:52 -0500 Subject: [PATCH 025/219] check if there is a version or hotfix to upgrade to before verifiying elasticsearch compatibility --- salt/manager/tools/sbin/soup | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index ba76d2a3e..eb7818cf1 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1606,11 +1606,12 @@ main() { echo "Verifying we have the latest soup script." verify_latest_update_script + echo "Let's see if we need to update Security Onion." + upgrade_check + echo "Verifying Elasticsearch version compatibility across the grid before upgrading." verify_es_version_compatibility - echo "Let's see if we need to update Security Onion." - upgrade_check upgrade_space echo "Checking for Salt Master and Minion updates." From 3c533cccbce84fab75b6c96ef69933816c3bcd77 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 1 Jun 2026 11:28:59 -0500 Subject: [PATCH 026/219] and after free space check --- salt/manager/tools/sbin/soup | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index eb7818cf1..a62a39b40 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1608,12 +1608,11 @@ main() { echo "Let's see if we need to update Security Onion." upgrade_check + upgrade_space echo "Verifying Elasticsearch version compatibility across the grid before upgrading." verify_es_version_compatibility - upgrade_space - echo "Checking for Salt Master and Minion updates." upgrade_check_salt set -e From f2996fb888c5db9bd9740ae0cc5a922330b8ea6b Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 1 Jun 2026 11:52:35 -0500 Subject: [PATCH 027/219] use so-config-backup script in soup --- salt/backup/tools/sbin/so-config-backup.jinja | 6 ++++-- salt/manager/tools/sbin/soup | 14 ++++---------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/salt/backup/tools/sbin/so-config-backup.jinja b/salt/backup/tools/sbin/so-config-backup.jinja index 7f65bbba3..8d214e665 100755 --- a/salt/backup/tools/sbin/so-config-backup.jinja +++ b/salt/backup/tools/sbin/so-config-backup.jinja @@ -25,9 +25,11 @@ if [ ! -f $BACKUPFILE ]; then # Create empty backup file tar -cf $BACKUPFILE -T /dev/null - # Loop through all paths defined in global.sls, and append them to backup file + # Loop through all paths defined in global.sls, and append them to backup file if they exist {%- for LOCATION in BACKUPLOCATIONS %} - tar -rf $BACKUPFILE "${EXCLUSIONS[@]}" {{ LOCATION }} + if [[ -d {{ LOCATION }} || -f {{ LOCATION }} ]]; then + tar -rf $BACKUPFILE "${EXCLUSIONS[@]}" {{ LOCATION }} + fi {%- endfor %} fi diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 135c51276..7874bf7b2 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -188,13 +188,6 @@ airgap_update_dockers() { fi } -backup_old_states_pillars() { - - tar czf /nsm/backup/$(echo $INSTALLEDVERSION)_$(date +%Y%m%d-%H%M%S)_soup_default_states_pillars.tar.gz /opt/so/saltstack/default/ - tar czf /nsm/backup/$(echo $INSTALLEDVERSION)_$(date +%Y%m%d-%H%M%S)_soup_local_states_pillars.tar.gz /opt/so/saltstack/local/ - -} - update_registry() { docker stop so-dockerregistry docker rm so-dockerregistry @@ -1670,7 +1663,8 @@ main() { echo "Applying $HOTFIXVERSION hotfix" # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars if [[ ! "$MINION_ROLE" == "import" ]]; then - backup_old_states_pillars + echo "Running so-config-backup script." + /sbin/so-config-backup fi copy_new_files create_local_directories "/opt/so/saltstack/default" @@ -1726,8 +1720,8 @@ main() { # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars if [[ ! "$MINION_ROLE" == "import" ]]; then echo "" - echo "Creating snapshots of default and local Salt states and pillars and saving to /nsm/backup/" - backup_old_states_pillars + echo "Running so-config-backup script." + /sbin/so-config-backup fi echo "" From 8c17ae0f66d74f3fce326602edcd85bf74562897 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 1 Jun 2026 14:48:54 -0400 Subject: [PATCH 028/219] move so-salt-minion-wait --- salt/salt/init.sls | 8 ++++++++ salt/{common => salt}/tools/sbin/so-salt-minion-wait | 0 2 files changed, 8 insertions(+) rename salt/{common => salt}/tools/sbin/so-salt-minion-wait (100%) diff --git a/salt/salt/init.sls b/salt/salt/init.sls index cea67f46a..d39c984eb 100644 --- a/salt/salt/init.sls +++ b/salt/salt/init.sls @@ -5,3 +5,11 @@ salt_bootstrap: - source: salt://salt/scripts/bootstrap-salt.sh - mode: 755 - show_changes: False + +salt_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://salt/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 diff --git a/salt/common/tools/sbin/so-salt-minion-wait b/salt/salt/tools/sbin/so-salt-minion-wait similarity index 100% rename from salt/common/tools/sbin/so-salt-minion-wait rename to salt/salt/tools/sbin/so-salt-minion-wait From f9c2579261b0785024b0ba09a33329e1a96642e2 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 1 Jun 2026 12:04:33 -0500 Subject: [PATCH 029/219] remove logstash pipeline rename from hotfix moving to up_to_3.2.0 --- salt/manager/tools/sbin/soup | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 135c51276..ede301648 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -769,6 +769,8 @@ bootstrap_so_soc_database() { } up_to_3.2.0() { + fix_logstash_0013_lumberjack_pipeline_name + INSTALLEDVERSION=3.2.0 } @@ -1566,13 +1568,7 @@ EOF # Keeping this block in case we need to do a hotfix that requires salt update apply_hotfix() { - if [[ "$INSTALLEDVERSION" == "3.1.0" ]] ; then - # Do not remove this fix_logstash_0013_lumberjack_pipeline_name in future hotfixes without first validating older - # installs referencing "so/0013_input_lumberjack_fleet.conf" via pillar are upgradable - fix_logstash_0013_lumberjack_pipeline_name - else - echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)" - fi + echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)" } failed_soup_restore_items() { From 559465b40783ca3baa7265b1ac4dcd71e0ea32c9 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 1 Jun 2026 12:05:09 -0500 Subject: [PATCH 030/219] run elastic agent gen installers script in post_to_3.2.0 --- salt/manager/tools/sbin/soup | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index ede301648..1416f2ba3 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -777,6 +777,10 @@ up_to_3.2.0() { post_to_3.2.0() { bootstrap_so_soc_database + # Including agent regen script here since it was missed in post_to_3.1.0 + echo "Regenerating Elastic Agent Installers" + /sbin/so-elastic-agent-gen-installers + POSTVERSION=3.2.0 } From 7ca23132554557daf79bbf1ec47ad4a949e3c870 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 3 Jun 2026 09:05:23 -0400 Subject: [PATCH 031/219] move to securityonion db --- salt/manager/tools/sbin/soup | 24 ------------------------ salt/postgres/files/init-db.sh | 7 +------ salt/soc/defaults.yaml | 5 +++-- salt/soc/merged.map.jinja | 3 ++- salt/soc/soc_soc.yaml | 14 +++++++++++++- 5 files changed, 19 insertions(+), 34 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 135c51276..cd5f47e35 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -746,35 +746,11 @@ post_to_3.1.0() { ### 3.2.0 Scripts ### -bootstrap_so_soc_database() { - # init-db.sh is mounted into so-postgres at /docker-entrypoint-initdb.d/init-db.sh - # and runs automatically only on a fresh data directory. Hosts upgrading from - # 3.1.0 already have /nsm/postgres populated, so the so_soc bootstrap block - # added in 3.2 never fires. Re-run the script explicitly; it's idempotent. - echo "Bootstrapping so_soc database via init-db.sh." - # The postgres image has no USER directive, so `docker exec` defaults to - # root, and the container env intentionally omits POSTGRES_USER (the upstream - # entrypoint defaults it transiently during first-init only). Recreate both - # so psql inside init-db.sh resolves the connect user correctly. - local exec_cmd="docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh" - if ! /usr/sbin/so-postgres-wait; then - FINAL_MESSAGE_QUEUE+=("WARNING: so-postgres was not ready during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd") - return 0 - fi - if ! $exec_cmd; then - FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd") - return 0 - fi - echo "so_soc bootstrap complete." -} - up_to_3.2.0() { INSTALLEDVERSION=3.2.0 } post_to_3.2.0() { - bootstrap_so_soc_database - POSTVERSION=3.2.0 } diff --git a/salt/postgres/files/init-db.sh b/salt/postgres/files/init-db.sh index 03e6d08dd..2187585da 100644 --- a/salt/postgres/files/init-db.sh +++ b/salt/postgres/files/init-db.sh @@ -31,9 +31,4 @@ EOSQL # only ensures the shared database exists on first initialization. if ! psql -U "$POSTGRES_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1; then psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE DATABASE so_telegraf" -fi - -# Bootstrap the SOC database. -if ! psql -U "$POSTGRES_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='so_soc'" | grep -q 1; then - psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE DATABASE so_soc" -fi +fi \ No newline at end of file diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 62b451bec..05cad494e 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1523,8 +1523,9 @@ soc: saltstackDir: /opt/so/saltstack bypassEnabled: false postgres: - host: - password: + database: securityonion + host: "" + password: "" salt: queueDir: /opt/sensoroni/queue timeoutMs: 45000 diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index b34efb11d..cfc0fafbd 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -20,7 +20,8 @@ {% do SOCMERGED.config.server.modules.postgres.update({'host': GLOBALS.manager}) %} {% endif %} {% if not SOCMERGED.config.server.modules.postgres.password %} -{% do SOCMERGED.config.server.modules.postgres.update({'password': salt['pillar.get']('secrets:postgres_pass', '')}) %} +{% do SOCMERGED.config.server.modules.postgres.update({'password': salt['pillar.get']('postgres:auth:users:so_postgres_user:pass', '')}) %} +{% do SOCMERGED.config.server.modules.postgres.update({'user': salt['pillar.get']('postgres:auth:users:so_postgres_user:user', 'so_postgres')}) %} {% endif %} {# if SOCMERGED.config.server.modules.cases == httpcase details come from the soc pillar #} diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 3cb244eed..ad34c3bbf 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -468,8 +468,20 @@ soc: description: Hostname or IP address of the PostgreSQL server used by SOC. Defaults to the manager hostname. global: True advanced: True + port: + description: Port of the PostgreSQL server used by SOC. + global: True + advanced: True + user: + description: Username used by SOC to authenticate to the PostgreSQL server. + global: True + advanced: True + database: + description: Database used by SOC to authenticate to the PostgreSQL server. + global: True + advanced: True password: - description: Password used by SOC to authenticate to the PostgreSQL server. Defaults to the postgres superuser password seeded in the secrets pillar. + description: Password used by SOC to authenticate to the PostgreSQL server. global: True sensitive: True advanced: True From 61e72c89e422754978c7abf1b8fbdf68fcd5ef76 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 3 Jun 2026 09:49:53 -0400 Subject: [PATCH 032/219] postgres updates --- salt/postgres/files/init-db.sh | 1 + salt/soc/defaults.yaml | 5 ++++- salt/soc/soc_soc.yaml | 8 ++++++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/salt/postgres/files/init-db.sh b/salt/postgres/files/init-db.sh index 2187585da..d12bc4c9b 100644 --- a/salt/postgres/files/init-db.sh +++ b/salt/postgres/files/init-db.sh @@ -17,6 +17,7 @@ psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-E END IF; END \$\$; + GRANT ALL ON SCHEMA public TO "$SO_POSTGRES_USER"; GRANT ALL PRIVILEGES ON DATABASE "$POSTGRES_DB" TO "$SO_POSTGRES_USER"; -- Lock the SOC database down at the connect layer; PUBLIC gets CONNECT -- by default, which would let per-minion telegraf roles open sessions diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 05cad494e..c9399eab4 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1523,8 +1523,11 @@ soc: saltstackDir: /opt/so/saltstack bypassEnabled: false postgres: - database: securityonion host: "" + port: 5432 + sslMode: "allow" + database: securityonion + user: "" password: "" salt: queueDir: /opt/sensoroni/queue diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index ad34c3bbf..b2ac6d175 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -472,14 +472,18 @@ soc: description: Port of the PostgreSQL server used by SOC. global: True advanced: True - user: - description: Username used by SOC to authenticate to the PostgreSQL server. + sslMode: + description: "Use encrypted connections to the PostgreSQL server. Must be one of the following values: disable, allow, prefer, require, verify-ca, verify-full. Defaults to allow." global: True advanced: True database: description: Database used by SOC to authenticate to the PostgreSQL server. global: True advanced: True + user: + description: Username used by SOC to authenticate to the PostgreSQL server. + global: True + advanced: True password: description: Password used by SOC to authenticate to the PostgreSQL server. global: True From a767c79641a62b38c995d2e8ece2d62baf1d0b38 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 3 Jun 2026 10:39:37 -0400 Subject: [PATCH 033/219] restore soup db init --- salt/manager/tools/sbin/soup | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 82fb19434..d50187c9c 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -739,6 +739,28 @@ post_to_3.1.0() { ### 3.2.0 Scripts ### +bootstrap_so_soc_database() { + # init-db.sh is mounted into so-postgres at /docker-entrypoint-initdb.d/init-db.sh + # and runs automatically only on a fresh data directory. Hosts upgrading from + # 3.1.0 already have /nsm/postgres populated, so the so_soc bootstrap block + # added in 3.2 never fires. Re-run the script explicitly; it's idempotent. + echo "Bootstrapping so_soc database via init-db.sh." + # The postgres image has no USER directive, so `docker exec` defaults to + # root, and the container env intentionally omits POSTGRES_USER (the upstream + # entrypoint defaults it transiently during first-init only). Recreate both + # so psql inside init-db.sh resolves the connect user correctly. + local exec_cmd="docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh" + if ! /usr/sbin/so-postgres-wait; then + FINAL_MESSAGE_QUEUE+=("WARNING: so-postgres was not ready during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd") + return 0 + fi + if ! $exec_cmd; then + FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd") + return 0 + fi + echo "so_soc bootstrap complete." +} + up_to_3.2.0() { fix_logstash_0013_lumberjack_pipeline_name @@ -746,6 +768,8 @@ up_to_3.2.0() { } post_to_3.2.0() { + bootstrap_so_soc_database + # Including agent regen script here since it was missed in post_to_3.1.0 echo "Regenerating Elastic Agent Installers" /sbin/so-elastic-agent-gen-installers From 1d3d98f759132e59679bfb25980ce36c625c8007 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 3 Jun 2026 12:24:41 -0400 Subject: [PATCH 034/219] kilo --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 944880fa1..03e153fda 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.2.0 +3.0.0-kilo From 2d653b6f1bd7111f1e4f02f9463742f3bce7a77a Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 3 Jun 2026 15:46:58 -0400 Subject: [PATCH 035/219] does not need to be jinja template --- salt/salt/minion/boot_highstate.sls | 3 +-- ...-boot-highstate.service.jinja => so-boot-highstate.service} | 0 2 files changed, 1 insertion(+), 2 deletions(-) rename salt/salt/service/{so-boot-highstate.service.jinja => so-boot-highstate.service} (100%) diff --git a/salt/salt/minion/boot_highstate.sls b/salt/salt/minion/boot_highstate.sls index 13bfbda65..e489210f6 100644 --- a/salt/salt/minion/boot_highstate.sls +++ b/salt/salt/minion/boot_highstate.sls @@ -15,8 +15,7 @@ include: so_boot_highstate_unit_file: file.managed: - name: /etc/systemd/system/so-boot-highstate.service - - source: salt://salt/service/so-boot-highstate.service.jinja - - template: jinja + - source: salt://salt/service/so-boot-highstate.service - onchanges_in: - module: systemd_reload diff --git a/salt/salt/service/so-boot-highstate.service.jinja b/salt/salt/service/so-boot-highstate.service similarity index 100% rename from salt/salt/service/so-boot-highstate.service.jinja rename to salt/salt/service/so-boot-highstate.service From ca85c5d90045b333d73aa94349d82d4fadea2c89 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 3 Jun 2026 17:26:08 -0400 Subject: [PATCH 036/219] fix version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 03e153fda..944880fa1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.0.0-kilo +3.2.0 From 13f8be40b59e04d2de171b7ea93185dceddd5a32 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 4 Jun 2026 08:46:35 -0400 Subject: [PATCH 037/219] so-boot-highstate: wait for docker before running highstate Add docker.service to After= and Wants= so the boot-time highstate starts after docker is up. Uses Wants (soft) so highstate still runs if docker fails to start. --- salt/salt/service/so-boot-highstate.service | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/service/so-boot-highstate.service b/salt/salt/service/so-boot-highstate.service index f3ec950c3..a770122d6 100644 --- a/salt/salt/service/so-boot-highstate.service +++ b/salt/salt/service/so-boot-highstate.service @@ -1,7 +1,7 @@ [Unit] Description=Security Onion boot-time highstate (runs once per boot) -After=salt-minion.service network-online.target -Wants=network-online.target +After=salt-minion.service network-online.target docker.service +Wants=network-online.target docker.service Requires=salt-minion.service ConditionPathExists=/opt/so/conf/setup-complete From cb3631da818d1be7cb3a8c1e495751c2af0a8575 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 4 Jun 2026 15:07:27 -0400 Subject: [PATCH 038/219] Move setup-complete marker from /opt/so/conf to /opt/so/state The setup-complete marker is a runtime-state file, not config, so move it to /opt/so/state/setup-complete. Updates both writers (mark_setup_complete in setup/so-functions and the upgrade-path state in minion/init.sls) and the three readers (so-boot-highstate.service ConditionPathExists, boot_highstate.sls enable gate, and the so-user_sync cron gate). --- salt/manager/sync_es_users.sls | 4 ++-- salt/salt/minion/boot_highstate.sls | 2 +- salt/salt/minion/init.sls | 4 ++-- salt/salt/service/so-boot-highstate.service | 2 +- setup/so-functions | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/manager/sync_es_users.sls b/salt/manager/sync_es_users.sls index f452ff5fe..8fc9c6bb4 100644 --- a/salt/manager/sync_es_users.sls +++ b/salt/manager/sync_es_users.sls @@ -32,7 +32,7 @@ sync_es_users: - file: so-user.lock # require so-user.lock file to be missing # we dont want this added too early in setup, so the onlyif gates on the -# /opt/so/conf/setup-complete marker. The marker is written by +# /opt/so/state/setup-complete marker. The marker is written by # mark_setup_complete in setup/so-functions just before the final setup # highstate (and by an upgrade-path state for systems set up under the old gate). so-user_sync: @@ -40,4 +40,4 @@ so-user_sync: - user: root - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log' - identifier: so-user_sync - - onlyif: "test -e /opt/so/conf/setup-complete" + - onlyif: "test -e /opt/so/state/setup-complete" diff --git a/salt/salt/minion/boot_highstate.sls b/salt/salt/minion/boot_highstate.sls index e489210f6..eb2596dad 100644 --- a/salt/salt/minion/boot_highstate.sls +++ b/salt/salt/minion/boot_highstate.sls @@ -25,7 +25,7 @@ so_boot_highstate_unit_file: so_boot_highstate_service: service.enabled: - name: so-boot-highstate.service - - onlyif: test -e /opt/so/conf/setup-complete + - onlyif: test -e /opt/so/state/setup-complete - require: - file: so_boot_highstate_unit_file - module: systemd_reload diff --git a/salt/salt/minion/init.sls b/salt/salt/minion/init.sls index 0d0eed22c..59dd0289c 100644 --- a/salt/salt/minion/init.sls +++ b/salt/salt/minion/init.sls @@ -94,14 +94,14 @@ remove_startup_states: - mode: delete # Upgrade-path bridge: systems that already passed setup under the old gate -# (`grep -x 'startup_states: highstate' /etc/salt/minion`) get a setup-complete +# (`grep -x 'startup_states: highstate' /etc/salt/minion`) get a /opt/so/state/setup-complete # marker so so-boot-highstate.service can be enabled and the so-user_sync cron # in sync_es_users.sls keeps installing. Setup-in-progress systems instead get # the marker from `mark_setup_complete` in setup/so-functions at the right # moment. `replace: false` means we never overwrite a marker once written. mark_setup_complete_for_upgrades: file.managed: - - name: /opt/so/conf/setup-complete + - name: /opt/so/state/setup-complete - replace: false - makedirs: True - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion" diff --git a/salt/salt/service/so-boot-highstate.service b/salt/salt/service/so-boot-highstate.service index a770122d6..cc8c6a1c6 100644 --- a/salt/salt/service/so-boot-highstate.service +++ b/salt/salt/service/so-boot-highstate.service @@ -3,7 +3,7 @@ Description=Security Onion boot-time highstate (runs once per boot) After=salt-minion.service network-online.target docker.service Wants=network-online.target docker.service Requires=salt-minion.service -ConditionPathExists=/opt/so/conf/setup-complete +ConditionPathExists=/opt/so/state/setup-complete [Service] Type=oneshot diff --git a/setup/so-functions b/setup/so-functions index da8e31d73..5ce9a8fdc 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -547,7 +547,7 @@ mark_setup_complete() { # Writes the setup-complete marker. Salt's so-boot-highstate.service # (boot-time oneshot) and the so-user_sync cron gate in # salt/manager/sync_es_users.sls both key off this file. - local marker=/opt/so/conf/setup-complete + local marker=/opt/so/state/setup-complete info "Marking setup as complete" mkdir -p "$(dirname "$marker")" From ac907ba45fe6f146b7ee753b61c8d872557e929c Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 5 Jun 2026 16:42:08 -0500 Subject: [PATCH 039/219] fix elasticsearch template generation issue --- salt/elasticsearch/cluster.sls | 19 ++++++++++++++++++- salt/elasticsearch/template.map.jinja | 27 +++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/salt/elasticsearch/cluster.sls b/salt/elasticsearch/cluster.sls index e25aed36a..d20ee45ca 100644 --- a/salt/elasticsearch/cluster.sls +++ b/salt/elasticsearch/cluster.sls @@ -9,9 +9,12 @@ {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} {% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %} {% if GLOBALS.role != 'so-heavynode' %} -{% from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %} +{% from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS, ADDON_INDICES %} {% endif %} +include: + - elasticsearch.enabled + escomponenttemplates: file.recurse: - name: /opt/so/conf/elasticsearch/templates/component @@ -35,6 +38,20 @@ so_index_template_dir: {%- endfor %} {%- endif %} +{% if GLOBALS.role != "so-heavynode" %} +# Clean up legacy and non-SO managed templates from the elasticsearch/templates/addon-index/ directory +addon_index_template_dir: + file.directory: + - name: /opt/so/conf/elasticsearch/templates/addon-index + - clean: True + {%- if ADDON_INDICES %} + - require: + {%- for index in ADDON_INDICES %} + - file: addon_index_template_{{index}} + {%- endfor %} + {%- endif %} +{% endif %} + # Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml) # These index templates are for the core SO datasets and are always required {% for index, settings in ES_INDEX_SETTINGS.items() %} diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index e66057775..ed1b49abe 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -61,15 +61,25 @@ {% if ALL_ADDON_SETTINGS_ORIG.keys() | length > 0 %} {% for index in ALL_ADDON_SETTINGS_ORIG.keys() %} {% do ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ALL_ADDON_SETTINGS_ORIG[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %} +{# Explicitly excluding addon indices from ES_INDEX_SETTINGS_ORIG + When manager.soc_managed_annotations runs, new entries are added to the salt/elasticsearch/defaults.yaml file to support 'revert to default' functionality. + Subsequent map renders will then incorrectly include 'integration X' in 'ES_INDEX_SETTINGS_ORIG' due to being in the defaults.yaml file. #} +{% if index in ES_INDEX_SETTINGS_ORIG.keys() %} +{% do ES_INDEX_SETTINGS_ORIG.pop(index) %} +{% endif %} {% endfor %} {% endif %} {% set ES_INDEX_SETTINGS = {} %} -{% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS) %} +{% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS, EXCLUDE_INDICES=[]) %} {% do GLOBAL_OVERRIDES.update(salt['defaults.merge'](GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} {% for index, settings in GLOBAL_OVERRIDES.items() %} +{% if index in EXCLUDE_INDICES %} +{% continue %} +{% endif %} + {# prevent this action from being performed on custom defined indices. #} {# the custom defined index is not present in either of the dictionaries and fails to reder. #} {% if index in DEFINED_SETTINGS and index in GLOBAL_OVERRIDES %} @@ -150,10 +160,19 @@ {% endfor %} {% endmacro %} -{{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS) }} -{{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS) }} +{# Exclude addon integrations from final ES_INDEX_SETTINGS #} +{{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS, ALL_ADDON_SETTINGS_ORIG.keys() | list ) }} + +{# Exclude SO managed indices, otherwise ALL_ADDON_SETTINGS will include pillar values + of core integrations without merging defaults, resulting in an overlapping, but bad index template being generated. #} +{{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS, ES_INDEX_SETTINGS_ORIG.keys() | list ) }} {% set SO_MANAGED_INDICES = [] %} {% for index, settings in ES_INDEX_SETTINGS.items() %} {% do SO_MANAGED_INDICES.append(index) %} -{% endfor %} \ No newline at end of file +{% endfor %} + +{% set ADDON_INDICES = [] %} +{% for index, settings in ALL_ADDON_SETTINGS.items() %} +{% do ADDON_INDICES.append(index) %} +{% endfor %} From 9580976ba2a1317d1b99ef4091a964248170d8bb Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 8 Jun 2026 11:05:13 -0400 Subject: [PATCH 040/219] Add manager boot-time grid mine.update oneshot before highstate so-boot-mine-update.service is a manager-only Type=oneshot unit that runs once per boot after salt-master/salt-minion start and before so-boot-highstate.service. It pushes mine.update to all reachable minions so mine-backed pillars (node IPs, ES/Redis/Logstash discovery) are fresh before the boot highstate renders them. The helper waits for the responsive minion set to settle (plateau) rather than for every accepted key to report up, so an intentionally powered-off minion doesn't block the update; MAX_WAIT remains as a backstop. --- salt/manager/tools/sbin/so-boot-mine-update | 42 +++++++++++++++++++ salt/salt/master.sls | 1 + salt/salt/master/boot_mine_update.sls | 29 +++++++++++++ salt/salt/service/so-boot-mine-update.service | 15 +++++++ 4 files changed, 87 insertions(+) create mode 100755 salt/manager/tools/sbin/so-boot-mine-update create mode 100644 salt/salt/master/boot_mine_update.sls create mode 100644 salt/salt/service/so-boot-mine-update.service diff --git a/salt/manager/tools/sbin/so-boot-mine-update b/salt/manager/tools/sbin/so-boot-mine-update new file mode 100755 index 000000000..f497d891f --- /dev/null +++ b/salt/manager/tools/sbin/so-boot-mine-update @@ -0,0 +1,42 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Runs once per boot on managers (via so-boot-mine-update.service), before +# so-boot-highstate.service. Waits for the responsive minion set to settle, then +# pushes mine.update to all minions so mine-backed pillars (node IPs, ES/Redis/ +# Logstash discovery) are fresh before the boot highstate renders them. + +MAX_WAIT=${MINE_UPDATE_MAX_WAIT:-180} # hard backstop only +INTERVAL=10 +STABLE_CHECKS=3 # up-count must hold steady this many polls +elapsed=0 +prev=-1 +stable=0 +up=0 + +# Wait for the *reachable* minion set to settle rather than for every accepted +# key to report up: an operator may accept a minion's key and then intentionally +# power off that host, so requiring up >= accepted would never be satisfied and +# we'd always burn the full MAX_WAIT. Once the responsive count stops growing we +# stop waiting and run mine.update against whoever is up. +while [ "$elapsed" -lt "$MAX_WAIT" ]; do + up=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null \ + | python3 -c 'import sys,json; print(len(json.load(sys.stdin)))' 2>/dev/null) + up=${up:-0} + if [ "$up" -gt 0 ] && [ "$up" -eq "$prev" ]; then + stable=$((stable + 1)) + [ "$stable" -ge "$STABLE_CHECKS" ] && break + else + stable=0 + fi + prev=$up + sleep "$INTERVAL" + elapsed=$((elapsed + INTERVAL)) +done + +echo "so-boot-mine-update: ${up} minions up (settled after ${elapsed}s); running mine.update" +/usr/bin/salt '*' mine.update --out=txt diff --git a/salt/salt/master.sls b/salt/salt/master.sls index 895150cd7..c62bd20f3 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -14,6 +14,7 @@ include: - salt.minion + - salt.master.boot_mine_update {% if 'vrt' in salt['pillar.get']('features', []) %} - salt.cloud - salt.cloud.reactor_config_hypervisor diff --git a/salt/salt/master/boot_mine_update.sls b/salt/salt/master/boot_mine_update.sls new file mode 100644 index 000000000..9f96c0ddf --- /dev/null +++ b/salt/salt/master/boot_mine_update.sls @@ -0,0 +1,29 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Manages /etc/systemd/system/so-boot-mine-update.service, a manager-only +# Type=oneshot unit that pushes `salt '*' mine.update` once per boot, ordered +# before so-boot-highstate.service so mine-backed pillars (node IPs, ES/Redis/ +# Logstash discovery) are fresh before the boot highstate renders them. + +include: + - systemd.reload + +so_boot_mine_update_unit_file: + file.managed: + - name: /etc/systemd/system/so-boot-mine-update.service + - source: salt://salt/service/so-boot-mine-update.service + - onchanges_in: + - module: systemd_reload + +# Only enable once setup is complete. Until then the gate file is missing and +# the unit's own ConditionPathExists would no-op it anyway. +so_boot_mine_update_service: + service.enabled: + - name: so-boot-mine-update.service + - onlyif: test -e /opt/so/state/setup-complete + - require: + - file: so_boot_mine_update_unit_file + - module: systemd_reload diff --git a/salt/salt/service/so-boot-mine-update.service b/salt/salt/service/so-boot-mine-update.service new file mode 100644 index 000000000..c5c6cdf7b --- /dev/null +++ b/salt/salt/service/so-boot-mine-update.service @@ -0,0 +1,15 @@ +[Unit] +Description=Security Onion boot-time grid mine.update (managers, runs once per boot before highstate) +After=salt-master.service salt-minion.service network-online.target +Wants=network-online.target +Requires=salt-master.service salt-minion.service +Before=so-boot-highstate.service +ConditionPathExists=/opt/so/state/setup-complete + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/sbin/so-boot-mine-update + +[Install] +WantedBy=multi-user.target From 6ad345730b5aa6f7959e00fa077844f3aa947316 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 8 Jun 2026 15:02:57 -0500 Subject: [PATCH 041/219] respect elasticfleet enable_auto_configuration setting for so-elastic-fleet-urls-update --- salt/elasticfleet/manager.sls | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/elasticfleet/manager.sls b/salt/elasticfleet/manager.sls index 1728f2010..6cb672bef 100644 --- a/salt/elasticfleet/manager.sls +++ b/salt/elasticfleet/manager.sls @@ -11,14 +11,15 @@ include: - elasticfleet.config # If enabled, automatically update Fleet Logstash Outputs -{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval'] %} +{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration %} +{% if grains.role not in ['so-import', 'so-eval']%} so-elastic-fleet-auto-configure-logstash-outputs: cmd.run: - name: /usr/sbin/so-elastic-fleet-outputs-update - retry: attempts: 4 interval: 30 -{% endif %} +{% endif %} # If enabled, automatically update Fleet Server URLs & ES Connection so-elastic-fleet-auto-configure-server-urls: @@ -27,6 +28,7 @@ so-elastic-fleet-auto-configure-server-urls: - retry: attempts: 4 interval: 30 +{% endif %} # Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs so-elastic-fleet-auto-configure-elasticsearch-urls: From eb82f9ea9d07aec323fc1dccfadbd4a197a11278 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 8 Jun 2026 16:53:35 -0400 Subject: [PATCH 042/219] kilo version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 944880fa1..03e153fda 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.2.0 +3.0.0-kilo From e536ffa36387c63a97e820cf3606e44ae94da228 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 9 Jun 2026 09:35:24 -0400 Subject: [PATCH 043/219] so-boot-mine-update: render node_data after mine.update before highstate After the boot-time mine.update, have the manager actually render the node_data pillar and log whether it came back populated. node_data: False makes salt/top.sls apply the bootstrap recovery branch instead of the manager's real config, so surfacing this in the journal makes the condition visible before so-boot-highstate runs. Best-effort and non-blocking: always exits 0 so highstate proceeds regardless. --- salt/manager/tools/sbin/so-boot-mine-update | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/salt/manager/tools/sbin/so-boot-mine-update b/salt/manager/tools/sbin/so-boot-mine-update index f497d891f..292b24ecc 100755 --- a/salt/manager/tools/sbin/so-boot-mine-update +++ b/salt/manager/tools/sbin/so-boot-mine-update @@ -40,3 +40,20 @@ done echo "so-boot-mine-update: ${up} minions up (settled after ${elapsed}s); running mine.update" /usr/bin/salt '*' mine.update --out=txt + +# Best-effort: confirm the manager can render node_data (non-False) now that the +# mine is updated. node_data: False makes salt/top.sls fall back to the bootstrap +# recovery branch instead of the manager's real config, so we surface that in the +# journal here. We never block highstate -- if still empty, the recovery branch +# and later highstates self-heal. +/usr/bin/salt-call saltutil.refresh_pillar >/dev/null 2>&1 +sleep 2 +status=$(/usr/bin/salt-call --out=json pillar.get node_data 2>/dev/null \ + | python3 -c 'import sys,json; d=json.load(sys.stdin).get("local"); print("rendered" if d else "empty")' 2>/dev/null) +status=${status:-empty} +if [ "$status" = "rendered" ]; then + echo "so-boot-mine-update: node_data renders; highstate will apply manager config" +else + echo "so-boot-mine-update: WARNING node_data still empty after mine.update; highstate may hit the bootstrap recovery branch" +fi +exit 0 From 8c306eb37dc04209e896347a3140092bcf2cb340 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 9 Jun 2026 09:49:19 -0400 Subject: [PATCH 044/219] so-boot-mine-update: log the rendered node_data content Dump the actual rendered node_data pillar (pretty-printed JSON) to the journal instead of just a rendered/empty verdict, so the boot-time render attempt is fully inspectable. Empty renders print false/null and still emit the WARNING. --- salt/manager/tools/sbin/so-boot-mine-update | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/manager/tools/sbin/so-boot-mine-update b/salt/manager/tools/sbin/so-boot-mine-update index 292b24ecc..38dd63191 100755 --- a/salt/manager/tools/sbin/so-boot-mine-update +++ b/salt/manager/tools/sbin/so-boot-mine-update @@ -48,10 +48,11 @@ echo "so-boot-mine-update: ${up} minions up (settled after ${elapsed}s); running # and later highstates self-heal. /usr/bin/salt-call saltutil.refresh_pillar >/dev/null 2>&1 sleep 2 -status=$(/usr/bin/salt-call --out=json pillar.get node_data 2>/dev/null \ - | python3 -c 'import sys,json; d=json.load(sys.stdin).get("local"); print("rendered" if d else "empty")' 2>/dev/null) -status=${status:-empty} -if [ "$status" = "rendered" ]; then +rendered=$(/usr/bin/salt-call --out=json pillar.get node_data 2>/dev/null \ + | python3 -c 'import sys,json; d=json.load(sys.stdin).get("local"); print(json.dumps(d, indent=2, sort_keys=True))' 2>/dev/null) +echo "so-boot-mine-update: node_data rendered as:" +echo "${rendered:-null}" +if [ -n "$rendered" ] && [ "$rendered" != "null" ] && [ "$rendered" != "false" ]; then echo "so-boot-mine-update: node_data renders; highstate will apply manager config" else echo "so-boot-mine-update: WARNING node_data still empty after mine.update; highstate may hit the bootstrap recovery branch" From 27c77023255caf5e2eede75f0ad03cd1350d9eeb Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 9 Jun 2026 10:10:32 -0400 Subject: [PATCH 045/219] so-boot-mine-update: wait for a complete mine before highstate Mine-backed pillars (node_data, elasticsearch:nodes, redis:nodes, logstash:nodes, hypervisor:nodes) include a node only if it returned an IP from the mine, and the configs they build are rebuilt fresh every highstate. After a manager reboot with a flushed mine, the first boot highstate could run before an up node re-reported network.ip_addrs, dropping it from e.g. so-elasticsearch ExtraHosts and forcing a container recreate. After the initial broad mine.update, poll until every currently-up minion actually has network.ip_addrs in the mine, re-pushing mine.update to stragglers, before releasing the boot highstate. Shares the existing MINE_UPDATE_MAX_WAIT backstop so a slow/down node never blocks boot, and still logs the rendered node_data for inspection. --- salt/manager/tools/sbin/so-boot-mine-update | 49 +++++++++++++++------ 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/salt/manager/tools/sbin/so-boot-mine-update b/salt/manager/tools/sbin/so-boot-mine-update index 38dd63191..85da4866a 100755 --- a/salt/manager/tools/sbin/so-boot-mine-update +++ b/salt/manager/tools/sbin/so-boot-mine-update @@ -6,9 +6,11 @@ # Elastic License 2.0. # Runs once per boot on managers (via so-boot-mine-update.service), before -# so-boot-highstate.service. Waits for the responsive minion set to settle, then -# pushes mine.update to all minions so mine-backed pillars (node IPs, ES/Redis/ -# Logstash discovery) are fresh before the boot highstate renders them. +# so-boot-highstate.service. Waits for the responsive minion set to settle, pushes +# mine.update, then waits until every up minion has actually reported to the mine +# so mine-backed pillars (node IPs, ES/Redis/Logstash discovery) are complete +# before the boot highstate renders them -- otherwise a not-yet-reported node gets +# dropped from those pillars and torn out of the configs they build. MAX_WAIT=${MINE_UPDATE_MAX_WAIT:-180} # hard backstop only INTERVAL=10 @@ -41,20 +43,39 @@ done echo "so-boot-mine-update: ${up} minions up (settled after ${elapsed}s); running mine.update" /usr/bin/salt '*' mine.update --out=txt -# Best-effort: confirm the manager can render node_data (non-False) now that the -# mine is updated. node_data: False makes salt/top.sls fall back to the bootstrap -# recovery branch instead of the manager's real config, so we surface that in the -# journal here. We never block highstate -- if still empty, the recovery branch -# and later highstates self-heal. +# A node that is up but has not yet re-reported network.ip_addrs to the mine is +# silently dropped from mine-backed pillars (elasticsearch:nodes, node_data, ...) +# when highstate recompiles them -- which e.g. removes it from so-elasticsearch +# ExtraHosts and forces a container recreate. After the broad mine.update above, +# wait until every up minion actually has network.ip_addrs in the mine, re-pushing +# mine.update to stragglers, before releasing the boot highstate. Bounded by the +# same MAX_WAIT backstop so a slow/down node never blocks boot indefinitely. +missing="" +while [ "$elapsed" -lt "$MAX_WAIT" ]; do + up_json=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null) + mine_json=$(/usr/bin/salt-run mine.get '*' network.ip_addrs tgt_type=glob --out=json 2>/dev/null) + missing=$(printf '%s' "$up_json" | python3 -c ' +import sys, json +up = set(json.load(sys.stdin) or []) +mine = {k for k, v in (json.loads(sys.argv[1]) or {}).items() if v} +print("\n".join(sorted(up - mine))) +' "$mine_json" 2>/dev/null) + if [ -z "$missing" ]; then + echo "so-boot-mine-update: mine complete for all up minions after ${elapsed}s" + break + fi + echo "so-boot-mine-update: mine missing up minion(s): $(echo $missing); re-running mine.update" + for m in $missing; do /usr/bin/salt "$m" mine.update --out=txt; done + sleep "$INTERVAL" + elapsed=$((elapsed + INTERVAL)) +done +[ -n "$missing" ] && echo "so-boot-mine-update: WARNING ${MAX_WAIT}s backstop hit; up minion(s) still absent from mine: $(echo $missing); highstate may drop them from configs" + +# Log what node_data renders so the boot-time pillar state is inspectable. /usr/bin/salt-call saltutil.refresh_pillar >/dev/null 2>&1 sleep 2 rendered=$(/usr/bin/salt-call --out=json pillar.get node_data 2>/dev/null \ - | python3 -c 'import sys,json; d=json.load(sys.stdin).get("local"); print(json.dumps(d, indent=2, sort_keys=True))' 2>/dev/null) + | python3 -c 'import sys,json; print(json.dumps(json.load(sys.stdin).get("local"), indent=2, sort_keys=True))' 2>/dev/null) echo "so-boot-mine-update: node_data rendered as:" echo "${rendered:-null}" -if [ -n "$rendered" ] && [ "$rendered" != "null" ] && [ "$rendered" != "false" ]; then - echo "so-boot-mine-update: node_data renders; highstate will apply manager config" -else - echo "so-boot-mine-update: WARNING node_data still empty after mine.update; highstate may hit the bootstrap recovery branch" -fi exit 0 From 9f5a9616a5e5e942fd10fb0ac5682e45684dbbfd Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 9 Jun 2026 12:51:58 -0500 Subject: [PATCH 046/219] use pipe exit status for update_docker_containers --- setup/so-functions | 2 ++ setup/so-setup | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 5ce9a8fdc..2d5181dc1 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -980,6 +980,8 @@ docker_seed_registry() { docker_seed_update_percent=25 update_docker_containers 'netinstall' '' 'docker_seed_update' '/dev/stdout' 2>&1 | tee -a "$setup_log" + # Use pipe exit status of 'update_docker_containers' for return code + return ${PIPESTATUS[0]} fi } diff --git a/setup/so-setup b/setup/so-setup index c11d287eb..72cd555d6 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -767,7 +767,10 @@ if ! [[ -f $install_opt_file ]]; then title "Applying the registry state" logCmd "salt-call state.apply -l info registry" title "Seeding the docker registry" - docker_seed_registry + if ! docker_seed_registry; then + error "Failed to seed the docker registry" + fail_setup + fi title "Applying the manager state" logCmd "salt-call state.apply -l info manager" logCmd "salt-call state.apply influxdb -l info" From f088a27159afea926531729e2a45b399172d160b Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 9 Jun 2026 13:52:19 -0400 Subject: [PATCH 047/219] so-boot-mine-update: warm master pillar cache before highstate A complete mine is not enough: elasticsearch:nodes, redis:nodes, logstash:nodes (tgt_type=pillar) and hypervisor:nodes (tgt_type=compound) resolve their target against the master's per-minion data cache (grains+pillar in data.p), which is populated only when a minion's pillar is recompiled -- separately from the mine. After a reboot a node can be in the mine (so node_data/glob sees it) yet absent from that cache, so it fails the elasticsearch:enabled:true pillar match and is dropped from elasticsearch:nodes -> so-elasticsearch ExtraHosts -> container recreate. After the mine-completeness wait, run salt '*' saltutil.refresh_pillar wait=True to synchronously cache every up node's pillar (the same lever deploy_newnode.sls uses), then verify with salt-run cache.pillar and retry stragglers, bounded by MINE_UPDATE_MAX_WAIT. Also log elasticsearch:nodes alongside node_data for inspection. --- salt/manager/tools/sbin/so-boot-mine-update | 54 +++++++++++++++++---- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/salt/manager/tools/sbin/so-boot-mine-update b/salt/manager/tools/sbin/so-boot-mine-update index 85da4866a..79cd67844 100755 --- a/salt/manager/tools/sbin/so-boot-mine-update +++ b/salt/manager/tools/sbin/so-boot-mine-update @@ -7,10 +7,12 @@ # Runs once per boot on managers (via so-boot-mine-update.service), before # so-boot-highstate.service. Waits for the responsive minion set to settle, pushes -# mine.update, then waits until every up minion has actually reported to the mine -# so mine-backed pillars (node IPs, ES/Redis/Logstash discovery) are complete -# before the boot highstate renders them -- otherwise a not-yet-reported node gets -# dropped from those pillars and torn out of the configs they build. +# mine.update, waits until every up minion has actually reported to the mine, then +# warms the master's per-minion pillar cache so the mine-backed node pillars (node +# IPs, ES/Redis/Logstash/hypervisor discovery -- some glob- and some pillar/grain- +# targeted) are complete before the boot highstate renders them. Otherwise a node +# that is up but not yet fully reported gets dropped from those pillars and torn +# out of the configs they build (e.g. so-elasticsearch ExtraHosts -> container recreate). MAX_WAIT=${MINE_UPDATE_MAX_WAIT:-180} # hard backstop only INTERVAL=10 @@ -71,11 +73,45 @@ print("\n".join(sorted(up - mine))) done [ -n "$missing" ] && echo "so-boot-mine-update: WARNING ${MAX_WAIT}s backstop hit; up minion(s) still absent from mine: $(echo $missing); highstate may drop them from configs" -# Log what node_data renders so the boot-time pillar state is inspectable. +# The pillar/compound-targeted node pillars (elasticsearch:nodes, redis:nodes, +# logstash:nodes, hypervisor:nodes) resolve their target against the master's +# per-minion data cache (grains+pillar in .../minions//data.p), populated only +# when a minion's pillar is (re)compiled -- separately from the mine. A freshly +# booted node can be in the mine (glob/node_data sees it) yet absent from that +# cache, so it is dropped from those pillars and from the configs they build (e.g. +# so-elasticsearch ExtraHosts). Force a synchronous pillar refresh so the master +# caches every up node's pillar; refresh_pillar wait=True returns only once the +# pillar is recompiled (and thus cached for matching). Retry stragglers <= MAX_WAIT. +echo "so-boot-mine-update: warming master pillar cache for pillar/grain-targeted node pillars" +/usr/bin/salt '*' saltutil.refresh_pillar wait=True --out=txt +missing="" +while [ "$elapsed" -lt "$MAX_WAIT" ]; do + up_json=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null) + cached_json=$(/usr/bin/salt-run cache.pillar tgt='*' --out=json 2>/dev/null) + missing=$(printf '%s' "$up_json" | python3 -c ' +import sys, json +up = set(json.load(sys.stdin) or []) +cached = {k for k, v in (json.loads(sys.argv[1]) or {}).items() if v} +print("\n".join(sorted(up - cached))) +' "$cached_json" 2>/dev/null) + if [ -z "$missing" ]; then + echo "so-boot-mine-update: pillar cache warm for all up minions after ${elapsed}s" + break + fi + echo "so-boot-mine-update: pillar not yet cached for: $(echo $missing); refreshing" + for m in $missing; do /usr/bin/salt "$m" saltutil.refresh_pillar wait=True --out=txt; done + sleep "$INTERVAL" + elapsed=$((elapsed + INTERVAL)) +done +[ -n "$missing" ] && echo "so-boot-mine-update: WARNING ${MAX_WAIT}s backstop hit; pillar not cached for: $(echo $missing); pillar-targeted pillars may drop them" + +# Log what the mine-backed pillars render so the boot-time state is inspectable. /usr/bin/salt-call saltutil.refresh_pillar >/dev/null 2>&1 sleep 2 -rendered=$(/usr/bin/salt-call --out=json pillar.get node_data 2>/dev/null \ - | python3 -c 'import sys,json; print(json.dumps(json.load(sys.stdin).get("local"), indent=2, sort_keys=True))' 2>/dev/null) -echo "so-boot-mine-update: node_data rendered as:" -echo "${rendered:-null}" +for key in node_data elasticsearch:nodes; do + rendered=$(/usr/bin/salt-call --out=json pillar.get "$key" 2>/dev/null \ + | python3 -c 'import sys,json; print(json.dumps(json.load(sys.stdin).get("local"), indent=2, sort_keys=True))' 2>/dev/null) + echo "so-boot-mine-update: ${key} rendered as:" + echo "${rendered:-null}" +done exit 0 From 9aa9ea3255f3b53bbea6293d743f170d9b908ec2 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 8 Jun 2026 15:32:03 -0500 Subject: [PATCH 048/219] Iniitial DLM support --- salt/common/tools/sbin/so-common | 5 + salt/elasticfleet/content-defaults.map.jinja | 1 - salt/elasticfleet/input-defaults.map.jinja | 2 - salt/elasticsearch/cluster.sls | 12 + salt/elasticsearch/defaults.yaml | 142 +++++- salt/elasticsearch/soc_elasticsearch.yaml | 453 +++++++++++++++--- salt/elasticsearch/template.map.jinja | 12 + .../sbin/so-elasticsearch-templates-load | 8 - .../sbin_jinja/so-elasticsearch-dlm-apply | 135 ++++++ salt/manager/managed_soc_annotations.sls | 59 ++- salt/manager/tools/sbin/soup | 50 ++ 11 files changed, 771 insertions(+), 108 deletions(-) create mode 100644 salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index aca8496f5..812c1bb10 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -142,6 +142,11 @@ check_elastic_license() { fi } +check_elasticsearch_responsive() { + retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" || + fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot." +} + check_salt_master_status() { local count=0 local attempts="${1:- 10}" diff --git a/salt/elasticfleet/content-defaults.map.jinja b/salt/elasticfleet/content-defaults.map.jinja index f4237d6d1..7f40f7a21 100644 --- a/salt/elasticfleet/content-defaults.map.jinja +++ b/salt/elasticfleet/content-defaults.map.jinja @@ -9,7 +9,6 @@ {% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %} {% set ADDON_CONTENT_INTEGRATION_DEFAULTS = {} %} -{% set DEBUG_STUFF = {} %} {% for pkg in ADDON_CONTENT_PACKAGE_COMPONENTS %} {% if pkg.name in CORE_ESFLEET_PACKAGES %} diff --git a/salt/elasticfleet/input-defaults.map.jinja b/salt/elasticfleet/input-defaults.map.jinja index a02844330..6b94d0581 100644 --- a/salt/elasticfleet/input-defaults.map.jinja +++ b/salt/elasticfleet/input-defaults.map.jinja @@ -9,7 +9,6 @@ {% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %} {% set ADDON_INPUT_INTEGRATION_DEFAULTS = {} %} -{% set DEBUG_STUFF = {} %} {% for pkg in ADDON_INPUT_PACKAGE_COMPONENTS %} {% if pkg.name in CORE_ESFLEET_PACKAGES %} @@ -116,7 +115,6 @@ {% do ADDON_INPUT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %} -{% do DEBUG_STUFF.update({integration_key: "Generating defaults for "+ pkg.name })%} {% endfor %} {% endif %} {% endif %} diff --git a/salt/elasticsearch/cluster.sls b/salt/elasticsearch/cluster.sls index d20ee45ca..efa50285e 100644 --- a/salt/elasticsearch/cluster.sls +++ b/salt/elasticsearch/cluster.sls @@ -133,6 +133,18 @@ so-elasticsearch-templates: - docker_container: so-elasticsearch - file: elasticsearch_sbin_jinja +so-elasticsearch-dlm-apply: + cmd.run: + - name: /usr/sbin/so-elasticsearch-dlm-apply + - cwd: /opt/so + - require: + - docker_container: so-elasticsearch + - file: elasticsearch_sbin_jinja + - cmd: so-elasticsearch-templates + - retry: + attempts: 3 + interval: 10 + so-elasticsearch-pipelines: cmd.run: - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }} diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 52964b9cf..7b49074e8 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -2,6 +2,7 @@ elasticsearch: enabled: false version: 9.3.3 index_clean: true + data_retention_method: DLM vm: max_map_count: 1048576 config: @@ -18,9 +19,18 @@ elasticsearch: flood_stage: 90% high: 85% low: 80% + # don't want to set retention here since it will make ES restart with every update + + # potentially case where we could unintentially fall back to retention 7d and cause data loss + # data_streams: + # lifecycle: + # retention: + # default: 7d indices: id_field_data: enabled: false + # index: + # lifecycle: + # prefer_ilm: true logger: org: elasticsearch: @@ -63,6 +73,9 @@ elasticsearch: verification_mode: none index_settings: global_overrides: + # Tie this into cluster setting for data_streams.lifecycle.retention.default + data_stream_lifecycle: + data_retention: 7d index_template: template: settings: @@ -143,6 +156,8 @@ elasticsearch: order: desc so-common: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - agent-mappings @@ -304,6 +319,8 @@ elasticsearch: number_of_shards: 1 so-assistant-chat: index_sorting: false + data_stream_lifecycle: + data_retention: "" index_template: composed_of: - assistant-chat-mappings @@ -344,6 +361,8 @@ elasticsearch: min_age: 0ms so-assistant-session: index_sorting: false + data_stream_lifecycle: + data_retention: "" index_template: composed_of: - assistant-session-mappings @@ -497,6 +516,8 @@ elasticsearch: min_age: 30d so-idh: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - agent-mappings @@ -605,6 +626,8 @@ elasticsearch: min_age: 30d so-import: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - agent-mappings @@ -787,6 +810,8 @@ elasticsearch: min_age: 0ms so-kismet: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - kismet-mappings @@ -836,6 +861,8 @@ elasticsearch: min_age: 30d so-kratos: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - agent-mappings @@ -904,6 +931,8 @@ elasticsearch: min_age: 30d so-hydra: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - agent-mappings @@ -1049,6 +1078,8 @@ elasticsearch: min_age: 0ms so-logs: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - so-data-streams-mappings @@ -1129,6 +1160,8 @@ elasticsearch: min_age: 30d so-logs-detections_x_alerts: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - so-data-streams-mappings @@ -1192,6 +1225,8 @@ elasticsearch: min_age: 30d so-logs-elastic_agent: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - event-mappings @@ -1307,6 +1342,8 @@ elasticsearch: min_age: 30d so-elastic-agent-monitor: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - event-mappings @@ -1369,6 +1406,8 @@ elasticsearch: min_age: 30d so-logs-elastic_agent_x_apm_server: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - logs-elastic_agent.apm_server@package @@ -1433,6 +1472,8 @@ elasticsearch: min_age: 30d so-logs-elastic_agent_x_auditbeat: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - logs-elastic_agent.auditbeat@package @@ -1497,6 +1538,8 @@ elasticsearch: min_age: 30d so-logs-elastic_agent_x_cloudbeat: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - logs-elastic_agent.cloudbeat@package @@ -1561,6 +1604,8 @@ elasticsearch: min_age: 30d so-logs-elastic_agent_x_endpoint_security: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - event-mappings @@ -1620,6 +1665,8 @@ elasticsearch: min_age: 30d so-logs-elastic_agent_x_filebeat: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - event-mappings @@ -1679,6 +1726,8 @@ elasticsearch: min_age: 30d so-logs-elastic_agent_x_fleet_server: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - event-mappings @@ -1735,6 +1784,8 @@ elasticsearch: min_age: 30d so-logs-elastic_agent_x_heartbeat: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - logs-elastic_agent.heartbeat@package @@ -1799,6 +1850,8 @@ elasticsearch: min_age: 30d so-logs-elastic_agent_x_metricbeat: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - event-mappings @@ -1858,6 +1911,8 @@ elasticsearch: min_age: 30d so-logs-elastic_agent_x_osquerybeat: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - event-mappings @@ -1917,6 +1972,8 @@ elasticsearch: min_age: 30d so-logs-elastic_agent_x_packetbeat: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - logs-elastic_agent.packetbeat@package @@ -1981,6 +2038,8 @@ elasticsearch: min_age: 30d so-logs-elasticsearch_x_server: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - logs-elasticsearch.server@package @@ -2045,6 +2104,8 @@ elasticsearch: min_age: 30d so-logs-endpoint_x_actions: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - .logs-endpoint.actions@package @@ -2104,6 +2165,8 @@ elasticsearch: min_age: 30d so-logs-endpoint_x_action_x_responses: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - .logs-endpoint.action.responses@package @@ -2163,6 +2226,8 @@ elasticsearch: min_age: 30d so-logs-endpoint_x_alerts: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - logs-endpoint.alerts@package @@ -2222,6 +2287,8 @@ elasticsearch: min_age: 30d so-logs-endpoint_x_diagnostic_x_collection: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - .logs-endpoint.diagnostic.collection@package @@ -2297,6 +2364,8 @@ elasticsearch: min_age: 30d so-logs-endpoint_x_events_x_api: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - logs-endpoint.events.api@package @@ -2356,6 +2425,8 @@ elasticsearch: min_age: 30d so-logs-endpoint_x_events_x_file: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - logs-endpoint.events.file@package @@ -2415,6 +2486,8 @@ elasticsearch: min_age: 30d so-logs-endpoint_x_events_x_library: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - logs-endpoint.events.library@package @@ -2474,6 +2547,8 @@ elasticsearch: min_age: 30d so-logs-endpoint_x_events_x_network: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - logs-endpoint.events.network@package @@ -2533,6 +2608,8 @@ elasticsearch: min_age: 30d so-logs-endpoint_x_events_x_process: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - logs-endpoint.events.process@package @@ -2592,6 +2669,8 @@ elasticsearch: min_age: 30d so-logs-endpoint_x_events_x_registry: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - logs-endpoint.events.registry@package @@ -2651,6 +2730,8 @@ elasticsearch: min_age: 30d so-logs-endpoint_x_events_x_security: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - logs-endpoint.events.security@package @@ -2710,6 +2791,8 @@ elasticsearch: min_age: 30d so-logs-endpoint_x_heartbeat: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - .logs-endpoint.heartbeat@package @@ -2769,6 +2852,8 @@ elasticsearch: min_age: 30d so-logs-http_endpoint_x_generic: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - logs-http_endpoint.generic@package @@ -2817,6 +2902,8 @@ elasticsearch: min_age: 30d so-logs-httpjson_x_generic: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - logs-httpjson.generic@package @@ -2882,6 +2969,8 @@ elasticsearch: number_of_replicas: 0 so-logs-osquery-manager_x_action_x_responses: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: _meta: managed: true @@ -2953,6 +3042,8 @@ elasticsearch: number_of_replicas: 0 so-logs-osquery-manager_x_result: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: _meta: managed: true @@ -3005,6 +3096,8 @@ elasticsearch: min_age: 30d so-logs-soc: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - agent-mappings @@ -3113,6 +3206,8 @@ elasticsearch: min_age: 30d so-logs-system_x_application: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - event-mappings @@ -3162,6 +3257,8 @@ elasticsearch: min_age: 30d so-logs-system_x_auth: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - event-mappings @@ -3211,6 +3308,8 @@ elasticsearch: min_age: 30d so-logs-system_x_security: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - event-mappings @@ -3260,6 +3359,8 @@ elasticsearch: min_age: 30d so-logs-system_x_syslog: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - event-mappings @@ -3309,6 +3410,8 @@ elasticsearch: min_age: 30d so-logs-system_x_system: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - event-mappings @@ -3358,6 +3461,8 @@ elasticsearch: min_age: 30d so-logs-windows_x_forwarded: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - logs-windows.forwarded@package @@ -3405,6 +3510,8 @@ elasticsearch: min_age: 30d so-logs-windows_x_powershell: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - logs-windows.powershell@package @@ -3452,6 +3559,8 @@ elasticsearch: min_age: 30d so-logs-windows_x_powershell_operational: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - logs-windows.powershell_operational@package @@ -3499,6 +3608,8 @@ elasticsearch: min_age: 30d so-logs-windows_x_sysmon_operational: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - logs-windows.sysmon_operational@package @@ -3546,6 +3657,8 @@ elasticsearch: min_age: 30d so-logs-winlog_x_winlog: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - logs-winlog.winlog@package @@ -3594,6 +3707,8 @@ elasticsearch: min_age: 30d so-logstash: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - agent-mappings @@ -3709,6 +3824,8 @@ elasticsearch: min_age: 30d so-metrics-endpoint_x_metadata: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - metrics-endpoint.metadata@package @@ -3756,6 +3873,8 @@ elasticsearch: min_age: 30d so-metrics-endpoint_x_metrics: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - metrics-endpoint.metrics@package @@ -3803,6 +3922,8 @@ elasticsearch: min_age: 30d so-metrics-endpoint_x_policy: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - metrics-endpoint.policy@package @@ -3850,6 +3971,8 @@ elasticsearch: min_age: 30d so-metrics-fleet_server_x_agent_status: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - metrics@tsdb-settings @@ -3874,6 +3997,8 @@ elasticsearch: number_of_replicas: 0 so-metrics-fleet_server_x_agent_versions: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - metrics@tsdb-settings @@ -3898,6 +4023,8 @@ elasticsearch: number_of_replicas: 0 so-redis: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - agent-mappings @@ -3958,13 +4085,10 @@ elasticsearch: - vulnerability-mappings - common-settings - common-dynamic-mappings - - logs-redis.log@package - - logs-redis.log@custom data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-redis.log@custom + ignore_missing_component_templates: [] index_patterns: - logs-redis.log* priority: 501 @@ -4016,6 +4140,8 @@ elasticsearch: min_age: 30d so-strelka: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - agent-mappings @@ -4133,6 +4259,8 @@ elasticsearch: min_age: 30d so-suricata: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - agent-mappings @@ -4249,6 +4377,8 @@ elasticsearch: min_age: 30d so-suricata_x_alerts: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - agent-mappings @@ -4365,6 +4495,8 @@ elasticsearch: min_age: 30d so-syslog: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - agent-mappings @@ -4481,6 +4613,8 @@ elasticsearch: min_age: 30d so-zeek: index_sorting: false + data_stream_lifecycle: + data_retention: 7d index_template: composed_of: - agent-mappings diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index b96c58dbe..251a0c1d8 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -4,6 +4,13 @@ elasticsearch: forcedType: bool advanced: True helpLink: elasticsearch + data_retention_method: + description: Method for data retention. Options are ILM or DLM. For single node deployments and most distributed grid users, DLM will be the recommended option for simplified management. Those with more complex use cases may prefer ILM. The latter allows for more granular control, but requires more management overhead. + options: + - ILM + - DLM + forcedType: string + global: True version: description: "This specifies the version of the following containers: so-elastic-fleet-package-registry, so-elastic-agent, so-elastic-fleet, so-kibana, so-logstash and so-elasticsearch. Modifying this value in the Elasticsearch defaults.yaml will result in catastrophic grid failure." readonly: True @@ -13,7 +20,7 @@ elasticsearch: description: Specify the memory heap size in (m)egabytes for Elasticsearch. helpLink: elasticsearch index_clean: - description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, indices will only be deleted by the age defined in the ILM settings. This setting only applies to EVAL, STANDALONE, and HEAVY NODE installations. Other installations can only use ILM settings. + description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, data is retained by the configured lifecycle settings. This setting only applies to EVAL, STANDALONE, and HEAVY NODE installations. Other installations use lifecycle settings only. forcedType: bool helpLink: elasticsearch vm: @@ -139,6 +146,22 @@ elasticsearch: custom010: *pipelines index_settings: global_overrides: + data_stream_lifecycle: + data_retention: + description: | + The retention period for all data streams. Retention does not define the period that the data will be removed, but the minimum time period they will be kept. + + Use a number followed by a time unit, such as 7d. Leave blank for indefinite retention where supported. + + Configured retention period also affects the frequency of rolling over data streams. + - If retention is less than or equal to 1 day, max_age will be 1 hour + - If retention is less than or equal to 14 days, max_age will be 1 day + - If retention is less than or equal to 90 days, max_age will be 7 days + - If retention is greater than 90 days, max_age will be 30 days + global: True + forcedType: string + regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$ + regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d. index_template: template: settings: @@ -311,13 +334,29 @@ elasticsearch: forcedType: string global: True helpLink: elasticsearch - so-logs: &indexSettings + so-logs: &dataStreamSettings index_sorting: description: Sorts the index by event time, at the cost of additional processing resource consumption. forcedType: bool global: True advanced: True helpLink: elasticsearch + data_stream_lifecycle: + data_retention: + description: | + The retention period for this data stream. Retention does not define the period that the data will be removed, but the minimum time period it will be kept. + + Use a number followed by a time unit, such as 7d. Leave blank for indefinite retention where supported. + + Configured retention period also affects the frequency of rolling over this data stream. + - If retention is less than or equal to 1 day, max_age will be 1 hour + - If retention is less than or equal to 14 days, max_age will be 1 day + - If retention is less than or equal to 90 days, max_age will be 7 days + - If retention is greater than 90 days, max_age will be 30 days + global: True + forcedType: string + regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$ + regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d. index_template: index_patterns: description: Patterns for matching multiple indices or tables. @@ -335,6 +374,14 @@ elasticsearch: global: True advanced: True helpLink: elasticsearch + auto_expand_replicas: + description: Automatically expand the number of replicas based on the number of data nodes in the cluster. This can help ensure high availability as the cluster scales up or down. + forcedType: string + regex: "^(0-[1-9]|1-[2-9]|2-[3-9]|3-[4-9]|4-[5-9]|5-[6-9]|6-[7-9]|7-[89]|8-9|[0-9]-all|false)$" + regexFailureMessage: Must be in the format of "x-y" where x is minimum number of replicas and y is maximum number of replicas, or "0-all" to specify a minimum of 0 and no maximum, or "false" to disable automatic replica expansion. + global: True + advanced: True + helpLink: elasticsearch mapping: total_fields: limit: @@ -596,65 +643,349 @@ elasticsearch: global: True advanced: True helpLink: elasticsearch - so-logs-system_x_auth: *indexSettings - so-logs-system_x_syslog: *indexSettings - so-logs-system_x_system: *indexSettings - so-logs-system_x_application: *indexSettings - so-logs-system_x_security: *indexSettings - so-logs-windows_x_forwarded: *indexSettings - so-logs-windows_x_powershell: *indexSettings - so-logs-windows_x_powershell_operational: *indexSettings - so-logs-windows_x_sysmon_operational: *indexSettings - so-logs-winlog_x_winlog: *indexSettings - so-logs-detections_x_alerts: *indexSettings - so-logs-http_endpoint_x_generic: *indexSettings - so-logs-httpjson_x_generic: *indexSettings - so-logs-osquery-manager-actions: *indexSettings - so-logs-osquery-manager-action_x_responses: *indexSettings - so-logs-osquery-manager_x_action_x_responses: *indexSettings - so-logs-osquery-manager_x_result: *indexSettings - so-logs-elastic_agent_x_apm_server: *indexSettings - so-logs-elastic_agent_x_auditbeat: *indexSettings - so-logs-elastic_agent_x_cloudbeat: *indexSettings - so-logs-elastic_agent_x_endpoint_security: *indexSettings - so-logs-endpoint_x_alerts: *indexSettings - so-logs-endpoint_x_events_x_api: *indexSettings - so-logs-endpoint_x_events_x_file: *indexSettings - so-logs-endpoint_x_events_x_library: *indexSettings - so-logs-endpoint_x_events_x_network: *indexSettings - so-logs-endpoint_x_events_x_process: *indexSettings - so-logs-endpoint_x_events_x_registry: *indexSettings - so-logs-endpoint_x_events_x_security: *indexSettings - so-logs-elastic_agent_x_filebeat: *indexSettings - so-logs-elastic_agent_x_fleet_server: *indexSettings - so-logs-elastic_agent_x_heartbeat: *indexSettings - so-logs-elastic_agent: *indexSettings - so-logs-elastic_agent_x_metricbeat: *indexSettings - so-logs-elastic_agent_x_osquerybeat: *indexSettings - so-logs-elastic_agent_x_packetbeat: *indexSettings - so-logs-elasticsearch_x_server: *indexSettings - so-metrics-endpoint_x_metadata: *indexSettings - so-metrics-endpoint_x_metrics: *indexSettings - so-metrics-endpoint_x_policy: *indexSettings - so-metrics-nginx_x_stubstatus: *indexSettings - so-metrics-vsphere_x_datastore: *indexSettings - so-metrics-vsphere_x_host: *indexSettings - so-metrics-vsphere_x_virtualmachine: *indexSettings - so-case: *indexSettings - so-common: *indexSettings - so-endgame: *indexSettings - so-idh: *indexSettings - so-suricata: *indexSettings - so-suricata_x_alerts: *indexSettings - so-import: *indexSettings - so-kratos: *indexSettings - so-hydra: *indexSettings - so-kismet: *indexSettings - so-logstash: *indexSettings - so-redis: *indexSettings - so-strelka: *indexSettings - so-syslog: *indexSettings - so-zeek: *indexSettings + so-logs-system_x_auth: *dataStreamSettings + so-logs-system_x_syslog: *dataStreamSettings + so-logs-system_x_system: *dataStreamSettings + so-logs-system_x_application: *dataStreamSettings + so-logs-system_x_security: *dataStreamSettings + so-logs-windows_x_forwarded: *dataStreamSettings + so-logs-windows_x_powershell: *dataStreamSettings + so-logs-windows_x_powershell_operational: *dataStreamSettings + so-logs-windows_x_sysmon_operational: *dataStreamSettings + so-logs-winlog_x_winlog: *dataStreamSettings + so-logs-detections_x_alerts: *dataStreamSettings + so-logs-http_endpoint_x_generic: *dataStreamSettings + so-logs-httpjson_x_generic: *dataStreamSettings + so-logs-osquery-manager-actions: *dataStreamSettings + so-logs-osquery-manager-action_x_responses: *dataStreamSettings + so-logs-osquery-manager_x_action_x_responses: *dataStreamSettings + so-logs-osquery-manager_x_result: *dataStreamSettings + so-logs-elastic_agent_x_apm_server: *dataStreamSettings + so-logs-elastic_agent_x_auditbeat: *dataStreamSettings + so-logs-elastic_agent_x_cloudbeat: *dataStreamSettings + so-logs-elastic_agent_x_endpoint_security: *dataStreamSettings + so-logs-endpoint_x_alerts: *dataStreamSettings + so-logs-endpoint_x_events_x_api: *dataStreamSettings + so-logs-endpoint_x_events_x_file: *dataStreamSettings + so-logs-endpoint_x_events_x_library: *dataStreamSettings + so-logs-endpoint_x_events_x_network: *dataStreamSettings + so-logs-endpoint_x_events_x_process: *dataStreamSettings + so-logs-endpoint_x_events_x_registry: *dataStreamSettings + so-logs-endpoint_x_events_x_security: *dataStreamSettings + so-logs-elastic_agent_x_filebeat: *dataStreamSettings + so-logs-elastic_agent_x_fleet_server: *dataStreamSettings + so-logs-elastic_agent_x_heartbeat: *dataStreamSettings + so-logs-elastic_agent: *dataStreamSettings + so-logs-elastic_agent_x_metricbeat: *dataStreamSettings + so-logs-elastic_agent_x_osquerybeat: *dataStreamSettings + so-logs-elastic_agent_x_packetbeat: *dataStreamSettings + so-logs-elasticsearch_x_server: *dataStreamSettings + so-metrics-endpoint_x_metadata: *dataStreamSettings + so-metrics-endpoint_x_metrics: *dataStreamSettings + so-metrics-endpoint_x_policy: *dataStreamSettings + so-metrics-nginx_x_stubstatus: *dataStreamSettings + so-metrics-vsphere_x_datastore: *dataStreamSettings + so-metrics-vsphere_x_host: *dataStreamSettings + so-metrics-vsphere_x_virtualmachine: *dataStreamSettings + so-common: *dataStreamSettings + so-endgame: *dataStreamSettings + so-idh: *dataStreamSettings + so-suricata: *dataStreamSettings + so-suricata_x_alerts: *dataStreamSettings + so-import: *dataStreamSettings + so-kratos: *dataStreamSettings + so-hydra: *dataStreamSettings + so-kismet: *dataStreamSettings + so-logstash: *dataStreamSettings + so-redis: *dataStreamSettings + so-strelka: *dataStreamSettings + so-syslog: *dataStreamSettings + so-zeek: *dataStreamSettings + # Managed SOC integration annotations are inserted below this line. Referencing '*dataStreamSettings' + so-case: &indexSettings + index_sorting: + description: Sorts the index by event time, at the cost of additional processing resource consumption. + forcedType: bool + global: True + advanced: True + helpLink: elasticsearch + index_template: + index_patterns: + description: Patterns for matching multiple indices or tables. + forcedType: "[]string" + multiline: True + global: True + advanced: True + helpLink: elasticsearch + template: + settings: + index: + number_of_replicas: + description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs. + forcedType: int + global: True + advanced: True + helpLink: elasticsearch + auto_expand_replicas: + description: Automatically expand the number of replicas based on the number of data nodes in the cluster. This can help ensure high availability as the cluster scales up or down. + forcedType: string + regex: "^(0-[1-9]|1-[2-9]|2-[3-9]|3-[4-9]|4-[5-9]|5-[6-9]|6-[7-9]|7-[89]|8-9|[0-9]-all|false)$" + regexFailureMessage: Must be in the format of "x-y" where x is minimum number of replicas and y is maximum number of replicas, or "0-all" to specify a minimum of 0 and no maximum, or "false" to disable automatic replica expansion. + global: True + advanced: True + helpLink: elasticsearch + mapping: + total_fields: + limit: + description: Max number of fields that can exist on a single index. Larger values will consume more resources. + global: True + advanced: True + helpLink: elasticsearch + refresh_interval: + description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation. + global: True + advanced: True + helpLink: elasticsearch + number_of_shards: + description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs. + global: True + advanced: True + helpLink: elasticsearch + sort: + field: + description: The field to sort by. Must set index_sorting to True. + global: True + advanced: True + helpLink: elasticsearch + order: + description: The order to sort by. Must set index_sorting to True. + global: True + advanced: True + helpLink: elasticsearch + mappings: + _meta: + package: + name: + description: Meta settings for the mapping. + global: True + advanced: True + helpLink: elasticsearch + managed_by: + description: Meta settings for the mapping. + global: True + advanced: True + helpLink: elasticsearch + managed: + description: Meta settings for the mapping. + forcedType: bool + global: True + advanced: True + helpLink: elasticsearch + composed_of: + description: The index template is composed of these component templates. + forcedType: "[]string" + global: True + advanced: True + helpLink: elasticsearch + priority: + description: The priority of the index template. + forcedType: int + global: True + advanced: True + helpLink: elasticsearch + policy: + phases: + hot: + min_age: + description: Minimum age of index. This determines when the index should be moved to the hot tier. + global: True + advanced: True + helpLink: elasticsearch + actions: + set_priority: + priority: + description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. + forcedType: int + global: True + advanced: True + helpLink: elasticsearch + rollover: + max_age: + description: Maximum age of index. Once an index reaches this limit, it will be rolled over into a new index. + global: True + advanced: True + helpLink: elasticsearch + max_primary_shard_size: + description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index. + global: True + advanced: True + helpLink: elasticsearch + shrink: + method: + description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size. + options: + - COUNT + - SIZE + global: True + advanced: True + forcedType: string + number_of_shards: + title: shard count + description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'. + global: True + forcedType: int + advanced: True + max_primary_shard_size: + title: max shard size + description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'. + regex: ^[0-9]+(?:gb|tb|pb)$ + global: True + forcedType: string + advanced: True + allow_write_after_shrink: + description: Allow writes after shrink. + global: True + forcedType: bool + default: False + advanced: True + forcemerge: + max_num_segments: + description: Reduce the number of segments in each index shard and clean up deleted documents. + global: True + forcedType: int + advanced: True + index_codec: + title: compression + description: Use higher compression for stored fields at the cost of slower performance. + forcedType: bool + global: True + default: False + advanced: True + warm: + min_age: + description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier. + regex: ^[0-9]{1,5}d$ + forcedType: string + global: True + advanced: True + helpLink: elasticsearch + actions: + set_priority: + priority: + description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. + forcedType: int + global: True + advanced: True + helpLink: elasticsearch + rollover: + max_age: + description: Maximum age of index. Once an index reaches this limit, it will be rolled over into a new index. + global: True + advanced: True + helpLink: elasticsearch + max_primary_shard_size: + description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index. + global: True + advanced: True + helpLink: elasticsearch + shrink: + method: + description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size. + options: + - COUNT + - SIZE + global: True + advanced: True + number_of_shards: + title: shard count + description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'. + global: True + forcedType: int + advanced: True + max_primary_shard_size: + title: max shard size + description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'. + regex: ^[0-9]+(?:gb|tb|pb)$ + global: True + forcedType: string + advanced: True + allow_write_after_shrink: + description: Allow writes after shrink. + global: True + forcedType: bool + default: False + advanced: True + forcemerge: + max_num_segments: + description: Reduce the number of segments in each index shard and clean up deleted documents. + global: True + forcedType: int + advanced: True + index_codec: + title: compression + description: Use higher compression for stored fields at the cost of slower performance. + forcedType: bool + global: True + default: False + advanced: True + allocate: + number_of_replicas: + description: Set the number of replicas. Remains the same as the previous phase by default. + forcedType: int + global: True + advanced: True + cold: + min_age: + description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier. + regex: ^[0-9]{1,5}d$ + forcedType: string + global: True + advanced: True + helpLink: elasticsearch + actions: + set_priority: + priority: + description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. + forcedType: int + global: True + advanced: True + helpLink: elasticsearch + allocate: + number_of_replicas: + description: Set the number of replicas. Remains the same as the previous phase by default. + forcedType: int + global: True + advanced: True + delete: + min_age: + description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion. + regex: ^[0-9]{1,5}d$ + forcedType: string + global: True + advanced: True + helpLink: elasticsearch + _meta: + package: + name: + description: Meta settings for the mapping. + global: True + advanced: True + helpLink: elasticsearch + managed_by: + description: Meta settings for the mapping. + global: True + advanced: True + helpLink: elasticsearch + managed: + description: Meta settings for the mapping. + forcedType: bool + global: True + advanced: True + helpLink: elasticsearch + sos-backup: *indexSettings + so-detection: *indexSettings + so-assistant-chat: *indexSettings + so-assistant-session: *indexSettings so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings index_sorting: description: Sorts the index by event time, at the cost of additional processing resource consumption. diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index ed1b49abe..7acb3eb87 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -5,6 +5,7 @@ {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %} +{% set DATA_RETENTION_METHOD = salt['pillar.get']('elasticsearch:data_retention_method', ELASTICSEARCHDEFAULTS.elasticsearch.get('data_retention_method', 'ILM')) %} {% set PILLAR_GLOBAL_OVERRIDES = {} %} {% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings', {}) %} @@ -105,6 +106,17 @@ {% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %} {% do settings.index_template.template.settings.index.pop('sort') %} {% endif %} +{% if DATA_RETENTION_METHOD == 'DLM' and settings.index_template.data_stream is defined and settings.data_stream_lifecycle is defined %} +{% if settings.data_stream_lifecycle.data_retention is defined and settings.data_stream_lifecycle.data_retention %} +{% do settings.index_template.template.update({'lifecycle': {'data_retention': settings.data_stream_lifecycle.data_retention}}) %} +{% else %} +{% do settings.index_template.template.update({'lifecycle': {}}) %} +{% endif %} +{% if settings.index_template.template.settings.index.lifecycle is not defined %} +{% do settings.index_template.template.settings.index.update({'lifecycle': {}}) %} +{% endif %} +{% do settings.index_template.template.settings.index.lifecycle.update({'prefer_ilm': false}) %} +{% endif %} {% endif %} {# advanced ilm actions #} diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load index a0ebd66e8..4e87fd602 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load @@ -125,14 +125,6 @@ load_component_templates() { done } -check_elasticsearch_responsive() { - # Cannot load templates if Elasticsearch is not responding. - # NOTE: Slightly faster exit w/ failure than previous "retry 240 1" if there is a problem with Elasticsearch the - # script should exit sooner rather than hang at the 'so-elasticsearch-templates' salt state. - retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" || - fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot." -} - index_templates_exist() { local templates_dir="$1" diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply new file mode 100644 index 000000000..4943bdbff --- /dev/null +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply @@ -0,0 +1,135 @@ +#!/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +{%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} + +{%- set DATA_RETENTION_METHOD = salt['pillar.get']('elasticsearch:data_retention_method', ELASTICSEARCHDEFAULTS.elasticsearch.get('data_retention_method', 'ILM')) %} +{%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %} +{%- if GLOBALS.role != "so-heavynode" %} +{%- from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %} +{%- endif %} +{%- set DLM_STREAMS = [] %} +{%- for index, settings in ES_INDEX_SETTINGS.items() %} +{%- if settings.index_template is defined and settings.index_template.data_stream is defined and settings.data_stream_lifecycle is defined %} +{%- do DLM_STREAMS.append({'template': index, 'data_retention': settings.data_stream_lifecycle.get('data_retention', '')}) %} +{%- endif %} +{%- endfor %} +{%- if GLOBALS.role != "so-heavynode" %} +{%- for index, settings in ALL_ADDON_SETTINGS.items() %} +{%- if settings.index_template is defined and settings.index_template.data_stream is defined and settings.data_stream_lifecycle is defined %} +{%- do DLM_STREAMS.append({'template': index, 'data_retention': settings.data_stream_lifecycle.get('data_retention', '')}) %} +{%- endif %} +{%- endfor %} +{%- endif %} + +STREAM_CONFIG='{{ DLM_STREAMS | tojson }}' +DATA_RETENTION_METHOD="{{ DATA_RETENTION_METHOD }}" +DLM_FAILURES=0 +DLM_FAILURE_NAMES=() + +if [[ "$DATA_RETENTION_METHOD" != "DLM" && "$DATA_RETENTION_METHOD" != "ILM" ]]; then + echo "Unsupported data retention method $DATA_RETENTION_METHOD. Expected DLM or ILM." + exit 1 +fi + +set_data_stream_lifecycle() { + local data_stream="$1" + local data_retention="$2" + local body + local output + + if [[ -n "$data_retention" ]]; then + if jq -e --arg data_stream "$data_stream" --arg data_retention "$data_retention" '.data_streams[]? | select(.name == $data_stream and .lifecycle.enabled == true and .lifecycle.data_retention == $data_retention)' >/dev/null 2>&1 <<< "$data_streams"; then + echo "DLM lifecycle already set for $data_stream with data_retention $data_retention, skipping." + return 0 + fi + elif jq -e --arg data_stream "$data_stream" '.data_streams[]? | select(.name == $data_stream and .lifecycle.enabled == true and (.lifecycle.data_retention == null))' >/dev/null 2>&1 <<< "$data_streams"; then + echo "DLM lifecycle already set for $data_stream with indefinite retention, skipping." + return 0 + fi + + if [[ -n "$data_retention" ]]; then + body=$(jq -cn --arg data_retention "$data_retention" '{data_retention: $data_retention}') + else + # Setting indefinite retention + body='{}' + fi + + if ! output=$(so-elasticsearch-query "_data_stream/${data_stream}/_lifecycle" -XPUT -d "$body" --retry 3 --retry-delay 5 --fail); then + echo "Failed to set data stream lifecycle for $data_stream." + echo "$output" + return 1 + fi + + if [[ -n "$data_retention" ]]; then + echo "Set DLM lifecycle for $data_stream with data_retention $data_retention." + else + echo "Set DLM lifecycle for $data_stream with indefinite retention." + fi +} + +disable_data_stream_lifecycle() { + local data_stream="$1" + local body='{"enabled":false}' + local output + + if ! jq -e --arg data_stream "$data_stream" '.data_streams[]? | select(.name == $data_stream and .lifecycle != null and .lifecycle.enabled != false)' >/dev/null 2>&1 <<< "$data_streams"; then + # No action needed + return 0 + fi + + if ! output=$(so-elasticsearch-query "_data_stream/${data_stream}/_lifecycle" -XPUT -d "$body" --retry 3 --retry-delay 5 --fail); then + echo "Failed to disable data stream lifecycle for $data_stream." + echo "$output" + return 1 + fi + + echo "Disabled DLM lifecycle for $data_stream." +} + +process_data_stream() { + local data_stream="$1" + local data_retention="$2" + + if [[ "$DATA_RETENTION_METHOD" == "DLM" ]]; then + set_data_stream_lifecycle "$data_stream" "$data_retention" + else + disable_data_stream_lifecycle "$data_stream" + fi +} + +check_elasticsearch_responsive + +if ! data_streams=$(so-elasticsearch-query "_data_stream?format=json" --retry 3 --retry-delay 5 --fail); then + echo "Failed to retrieve data streams." + exit 1 +fi + +while read -r config; do + template=$(jq -r '.template' <<< "$config") + data_retention=$(jq -r '.data_retention // ""' <<< "$config") + + while read -r data_stream; do + [[ -z "$data_stream" ]] && continue + + if ! process_data_stream "$data_stream" "$data_retention"; then + DLM_FAILURES=$((DLM_FAILURES + 1)) + DLM_FAILURE_NAMES+=("$data_stream") + fi + done <<< "$(jq -r --arg template "$template" '.data_streams[]? | select(.template == $template) | .name' <<< "$data_streams")" +done <<< "$(jq -c '.[]' <<< "$STREAM_CONFIG")" + +if [[ $DLM_FAILURES -eq 0 ]]; then + echo "Data stream lifecycle updates completed successfully." +else + echo "Encountered $DLM_FAILURES failure(s) updating data stream lifecycle:" + for failed_data_stream in "${DLM_FAILURE_NAMES[@]}"; do + echo " - $failed_data_stream" + done + exit 1 +fi diff --git a/salt/manager/managed_soc_annotations.sls b/salt/manager/managed_soc_annotations.sls index b2fbb7334..46d727bf7 100644 --- a/salt/manager/managed_soc_annotations.sls +++ b/salt/manager/managed_soc_annotations.sls @@ -16,40 +16,35 @@ {% endif %} {% endfor %} {% endfor %} +{% set soc_annotation_lines = [] %} +{% set defaults_lines = [] %} +{% for k in matched_integration_names %} +{% do soc_annotation_lines.append(' ' ~ k ~ ': *dataStreamSettings') %} +{% do defaults_lines.append(' ' ~ k ~ ':') %} +{% set defaults_yaml = salt['slsutil.serialize']('yaml', ADDON_INTEGRATION_DEFAULTS[k], default_flow_style=False).strip() %} +{% for line in defaults_yaml.splitlines() %} +{% do defaults_lines.append(' ' ~ line) %} +{% endfor %} +{% endfor %} {% set es_soc_annotations = '/opt/so/saltstack/default/salt/elasticsearch/soc_elasticsearch.yaml' %} -{{ es_soc_annotations }}: - file.serialize: - - dataset: - {% set data = salt['file.read'](es_soc_annotations) | load_yaml %} - {% set es = data.get('elasticsearch', {}) %} - {% set index_settings = es.get('index_settings', {}) %} - {% set input = index_settings.get('so-logs', {}) %} - {% for k in matched_integration_names %} - {% do index_settings.update({k: input}) %} - {% endfor %} - {% for k in addon_integration_keys %} - {% if k not in matched_integration_names and k in index_settings %} - {% do index_settings.pop(k) %} - {% endif %} - {% endfor %} - {{ data }} +manage_soc_annotations: + file.blockreplace: + - name: {{ es_soc_annotations }} + - marker_start: ' # START managed SOC integration annotations' + - marker_end: ' # END managed SOC integration annotations' + - content: {{ soc_annotation_lines | join('\n') | tojson }} + - insert_after_match: '^ # Managed SOC integration annotations are inserted below this line\.' + - append_if_not_found: False + - show_changes: True {# Managed elasticsearch/defaults.yaml file for enabling 'Revert to default' via SOC UI for newly added config items #} {% set es_defaults = '/opt/so/saltstack/default/salt/elasticsearch/defaults.yaml' %} {{ es_defaults }}: - file.serialize: - - dataset: - {% set data = salt['file.read'](es_defaults) | load_yaml %} - {% set es = data.get('elasticsearch', {}) %} - {% set index_settings = es.get('index_settings', {}) %} - {% for k in matched_integration_names %} - {% set input = ADDON_INTEGRATION_DEFAULTS[k] %} - {% do index_settings.update({k: input})%} - {% endfor %} - {% for k in addon_integration_keys %} - {% if k not in matched_integration_names and k in index_settings %} - {% do index_settings.pop(k) %} - {% endif %} - {% endfor %} - {{ data }} -{% endif %} \ No newline at end of file + file.blockreplace: + - marker_start: ' # START managed SOC integration defaults' + - marker_end: ' # END managed SOC integration defaults' + - content: {{ defaults_lines | join('\n') | tojson }} + - insert_after_match: '^ index_settings:$' + - append_if_not_found: False + - show_changes: True +{% endif %} diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index d50187c9c..d955580fd 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -761,9 +761,57 @@ bootstrap_so_soc_database() { echo "so_soc bootstrap complete." } +# Existing grids should keep ILM unless an admin explicitly opts in to DLM. +pin_elasticsearch_data_retention_method() { + local elasticsearch_file=/opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls + mkdir -p "$(dirname "$elasticsearch_file")" + [[ -f "$elasticsearch_file" ]] || touch "$elasticsearch_file" + + if so-yaml.py get -r "$elasticsearch_file" elasticsearch.data_retention_method >/dev/null 2>&1; then + echo "elasticsearch.data_retention_method already set; leaving as-is." + return 0 + fi + + echo "Pinning existing grid to ILM data retention." + so-yaml.py add "$elasticsearch_file" elasticsearch.data_retention_method ILM + chown socore:socore "$elasticsearch_file" +} + +# Addes auto_expand_replicas setting to .kibana_streams index template +# +# In Kibana 9.3.3 the auto_expand_replicas setting was not added to the .kibana_streams index template. Causing single node deployments to be stuck in yellow state (unable to assign replica). Here we update the template in place using the so_kibana system user (system managed index template) to include the auto_expand_replicas setting +# +# Reference: https://github.com/elastic/kibana/issues/263048 +kibana_backport_streams_index_template() { + local current_template updated_template + current_template=$(so-elasticsearch-query "_index_template/.kibana_streams" --retry 3 --retry-delay 5 --fail) + + if [[ -z "$current_template" ]]; then + echo "Unable to retrieve current .kibana_streams index template, skipping backport." + return 0 + fi + + updated_template=$(jq '.index_templates[0].index_template | .template.settings += {"index.auto_expand_replicas": "0-1"} | del(.created_date_millis, .modified_date_millis)' <<< "$current_template") + + if ! kibana_user_pass=$(/usr/sbin/so-yaml.py get -r /opt/so/saltstack/local/pillar/elasticsearch/auth.sls elasticsearch.auth.users.so_kibana_user.pass); then + echo "Unable to retrieve so_kibana_user password, skipping .kibana_streams index template backport." + return 0 + fi + + if ! so-elasticsearch-query "_index_template/.kibana_streams" -XPUT -d "$updated_template" -u "so_kibana:$kibana_user_pass" --retry 3 --retry-delay 5 --fail; then + echo "Unable to automatically update .kibana_streams index template" + return 0 + fi + + ## NOTE: Should really add a check here for existing .kibana_streams index and then update its config in place + +} + up_to_3.2.0() { fix_logstash_0013_lumberjack_pipeline_name + pin_elasticsearch_data_retention_method + INSTALLEDVERSION=3.2.0 } @@ -774,6 +822,8 @@ post_to_3.2.0() { echo "Regenerating Elastic Agent Installers" /sbin/so-elastic-agent-gen-installers + kibana_backport_streams_index_template + POSTVERSION=3.2.0 } From cf456dc58ca2d3da677e1c07b33f05f6562f8eeb Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 9 Jun 2026 23:21:43 -0500 Subject: [PATCH 049/219] reuse existing index templates --- .../sbin_jinja/so-elasticsearch-dlm-apply | 103 +++++++++++++----- 1 file changed, 73 insertions(+), 30 deletions(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply index 4943bdbff..843fad625 100644 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply @@ -9,26 +9,16 @@ {%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} {%- set DATA_RETENTION_METHOD = salt['pillar.get']('elasticsearch:data_retention_method', ELASTICSEARCHDEFAULTS.elasticsearch.get('data_retention_method', 'ILM')) %} -{%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %} -{%- if GLOBALS.role != "so-heavynode" %} -{%- from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %} -{%- endif %} -{%- set DLM_STREAMS = [] %} -{%- for index, settings in ES_INDEX_SETTINGS.items() %} -{%- if settings.index_template is defined and settings.index_template.data_stream is defined and settings.data_stream_lifecycle is defined %} -{%- do DLM_STREAMS.append({'template': index, 'data_retention': settings.data_stream_lifecycle.get('data_retention', '')}) %} -{%- endif %} -{%- endfor %} -{%- if GLOBALS.role != "so-heavynode" %} -{%- for index, settings in ALL_ADDON_SETTINGS.items() %} -{%- if settings.index_template is defined and settings.index_template.data_stream is defined and settings.data_stream_lifecycle is defined %} -{%- do DLM_STREAMS.append({'template': index, 'data_retention': settings.data_stream_lifecycle.get('data_retention', '')}) %} -{%- endif %} -{%- endfor %} -{%- endif %} -STREAM_CONFIG='{{ DLM_STREAMS | tojson }}' -DATA_RETENTION_METHOD="{{ DATA_RETENTION_METHOD }}" +ELASTICSEARCH_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR:-/opt/so/conf/elasticsearch/templates}" +TEMPLATE_DIRS=( + "${ELASTICSEARCH_TEMPLATES_DIR}/index" + "${ELASTICSEARCH_TEMPLATES_DIR}/addon-index" +) +DATA_RETENTION_METHOD=$(cat <<'EOF' +{{ DATA_RETENTION_METHOD }} +EOF +) DLM_FAILURES=0 DLM_FAILURE_NAMES=() @@ -37,6 +27,44 @@ if [[ "$DATA_RETENTION_METHOD" != "DLM" && "$DATA_RETENTION_METHOD" != "ILM" ]]; exit 1 fi +validate_template_file() { + local template_file="$1" + + if ! jq -e 'type == "object" and (.data_stream == null or (.data_stream | type == "object")) and (.template.lifecycle == null or (.template.lifecycle | type == "object")) and (.template.lifecycle.data_retention == null or (.template.lifecycle.data_retention | type == "string"))' >/dev/null 2>&1 "$template_file"; then + echo "Invalid index template JSON: $template_file" + return 1 + fi +} + +is_data_stream_template() { + jq -e '.data_stream | type == "object"' >/dev/null 2>&1 "$1" +} + +has_data_stream_lifecycle() { + jq -e '.template.lifecycle | type == "object"' >/dev/null 2>&1 "$1" +} + +get_data_retention() { + jq -r '.template.lifecycle.data_retention // ""' "$1" +} + +find_template_file() { + local template="$1" + local template_dir + local template_file + + for template_dir in "${TEMPLATE_DIRS[@]}"; do + template_file="${template_dir}/${template}-template.json" + + if [[ -f "$template_file" ]]; then + echo "$template_file" + return 0 + fi + done + + return 1 +} + set_data_stream_lifecycle() { local data_stream="$1" local data_retention="$2" @@ -110,19 +138,34 @@ if ! data_streams=$(so-elasticsearch-query "_data_stream?format=json" --retry 3 exit 1 fi -while read -r config; do - template=$(jq -r '.template' <<< "$config") - data_retention=$(jq -r '.data_retention // ""' <<< "$config") +while read -r data_stream_config; do + data_stream=$(jq -r '.name' <<< "$data_stream_config") + template=$(jq -r '.template' <<< "$data_stream_config") - while read -r data_stream; do - [[ -z "$data_stream" ]] && continue + if ! template_file=$(find_template_file "$template"); then + echo "Skipping $data_stream: index template file not found for $template." + continue + fi - if ! process_data_stream "$data_stream" "$data_retention"; then - DLM_FAILURES=$((DLM_FAILURES + 1)) - DLM_FAILURE_NAMES+=("$data_stream") - fi - done <<< "$(jq -r --arg template "$template" '.data_streams[]? | select(.template == $template) | .name' <<< "$data_streams")" -done <<< "$(jq -c '.[]' <<< "$STREAM_CONFIG")" + validate_template_file "$template_file" || exit 1 + + if ! is_data_stream_template "$template_file"; then + echo "Skipping $data_stream: $template_file is not a data stream template." + continue + fi + + if [[ "$DATA_RETENTION_METHOD" == "DLM" ]] && ! has_data_stream_lifecycle "$template_file"; then + echo "Skipping $data_stream: $template_file does not define data stream lifecycle." + continue + fi + + data_retention=$(get_data_retention "$template_file") + + if ! process_data_stream "$data_stream" "$data_retention"; then + DLM_FAILURES=$((DLM_FAILURES + 1)) + DLM_FAILURE_NAMES+=("$data_stream") + fi +done < <(jq -c '.data_streams[]' <<< "$data_streams") if [[ $DLM_FAILURES -eq 0 ]]; then echo "Data stream lifecycle updates completed successfully." From 944e7737599326e7028bf4a3b075f876cfe29b32 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 10 Jun 2026 14:58:49 -0500 Subject: [PATCH 050/219] save exit until all packages have been attempted --- .../so-elastic-fleet-package-upgrade | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade index 18211a7c6..a89a0409b 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade @@ -8,18 +8,35 @@ . /usr/sbin/so-elastic-fleet-common +PKG_LOAD_FAILURES=0 +PKG_LOAD_FAILURES_NAMES=() + {%- for PACKAGE in SUPPORTED_PACKAGES %} echo "Upgrading {{ PACKAGE }} package..." if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then # exit 1 on failure to upgrade a default package, allow salt to handle retries - echo -e "\nERROR: Failed to upgrade $PACKAGE to version: $VERSION" - exit 1 + echo -e "\nERROR: Failed to upgrade {{ PACKAGE }} to version: $VERSION" + PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1)) + PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}") fi else - echo -e "\nERROR: Failed to get version information for integration $PACKAGE" + echo -e "\nERROR: Failed to get version information for integration {{ PACKAGE }}" + PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1)) + PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}") fi echo {%- endfor %} + +if [ $PKG_LOAD_FAILURES -gt 0 ]; then + echo "ERROR: Failed to upgrade $PKG_LOAD_FAILURES package(s):" + for PKG in "${PKG_LOAD_FAILURES_NAMES[@]}"; do + echo " - $PKG" + done + exit 1 +else + echo "Successfully upgraded all packages." +fi + echo /usr/sbin/so-elasticsearch-templates-load From bd5e77afc58fb6f14136d067b7ae3fa0045590ab Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 10 Jun 2026 14:59:29 -0500 Subject: [PATCH 051/219] increase delay in so-elastic-fleet-package-upgrade attempts --- salt/elasticfleet/manager.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/manager.sls b/salt/elasticfleet/manager.sls index 6cb672bef..04430d496 100644 --- a/salt/elasticfleet/manager.sls +++ b/salt/elasticfleet/manager.sls @@ -55,7 +55,7 @@ so-elastic-fleet-package-upgrade: - name: /usr/sbin/so-elastic-fleet-package-upgrade - retry: attempts: 3 - interval: 10 + interval: 30 - onchanges: - file: /opt/so/state/elastic_fleet_packages.txt From f905afbc6fda4a94b2090452010d7f8cabf2143b Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 10 Jun 2026 15:01:22 -0500 Subject: [PATCH 052/219] logging --- .../tools/sbin_jinja/so-elastic-fleet-package-upgrade | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade index a89a0409b..8ba250c00 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade @@ -15,13 +15,10 @@ PKG_LOAD_FAILURES_NAMES=() echo "Upgrading {{ PACKAGE }} package..." if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then - # exit 1 on failure to upgrade a default package, allow salt to handle retries - echo -e "\nERROR: Failed to upgrade {{ PACKAGE }} to version: $VERSION" PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1)) PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}") fi else - echo -e "\nERROR: Failed to get version information for integration {{ PACKAGE }}" PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1)) PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}") fi @@ -33,6 +30,7 @@ if [ $PKG_LOAD_FAILURES -gt 0 ]; then for PKG in "${PKG_LOAD_FAILURES_NAMES[@]}"; do echo " - $PKG" done + # exit 1 on failure to upgrade a default package, allow salt to handle retries exit 1 else echo "Successfully upgraded all packages." From 83aaa76f982c008a451c19d6160cfc5ee8936569 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 10 Jun 2026 16:34:10 -0400 Subject: [PATCH 053/219] allow full highstate on manager when locked --- salt/manager/tools/sbin/soup | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index d50187c9c..4fd474ff2 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -343,10 +343,11 @@ highstate() { masterlock() { echo "Locking Salt Master" mv -v $TOPFILE $BACKUPTOPFILE - echo "base:" > $TOPFILE - echo " $MINIONID:" >> $TOPFILE - echo " - ca" >> $TOPFILE - echo " - elasticsearch" >> $TOPFILE + # Render the real top file only for the host running soup; every other + # minion gets an empty top (no states) while the master is upgrading. + echo "{% if grains['id'] == '$MINIONID' %}" > $TOPFILE + cat $BACKUPTOPFILE >> $TOPFILE + echo "{% endif %}" >> $TOPFILE } masterunlock() { From 289ddda5e8054ef324f2d06a41922e86d5a38542 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 10 Jun 2026 17:06:22 -0500 Subject: [PATCH 054/219] kibana health check for fleet scripts --- salt/elasticfleet/enabled.sls | 11 ++++++++ salt/elasticfleet/manager.sls | 21 +++++++++++++++ .../so-elastic-fleet-integration-policy-load | 7 +++-- salt/kibana/enabled.sls | 3 +++ salt/kibana/healthcheck.sls | 27 +++++++++++++++++++ 5 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 salt/kibana/healthcheck.sls diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 166cb9719..60cf27deb 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -101,6 +101,17 @@ so-elastic-fleet: - file: trusttheca - x509: etc_elasticfleet_key - x509: etc_elasticfleet_crt + +wait_for_so-elastic-fleet: + http.wait_for_successful_query: + - name: "https://localhost:8220/api/status" + - ssl: True + - verify_ssl: False + - status: 200 + - wait_for: 300 + - request_interval: 15 + - require: + - docker_container: so-elastic-fleet {% endif %} delete_so-elastic-fleet_so-status.disabled: diff --git a/salt/elasticfleet/manager.sls b/salt/elasticfleet/manager.sls index 04430d496..a0aa83460 100644 --- a/salt/elasticfleet/manager.sls +++ b/salt/elasticfleet/manager.sls @@ -9,6 +9,7 @@ include: - elasticfleet.config + - kibana.healthcheck # If enabled, automatically update Fleet Logstash Outputs {% if ELASTICFLEETMERGED.config.server.enable_auto_configuration %} @@ -19,6 +20,8 @@ so-elastic-fleet-auto-configure-logstash-outputs: - retry: attempts: 4 interval: 30 + - require: + - http: wait_for_so-kibana {% endif %} # If enabled, automatically update Fleet Server URLs & ES Connection @@ -28,6 +31,8 @@ so-elastic-fleet-auto-configure-server-urls: - retry: attempts: 4 interval: 30 + - require: + - http: wait_for_so-kibana {% endif %} # Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs @@ -37,6 +42,8 @@ so-elastic-fleet-auto-configure-elasticsearch-urls: - retry: attempts: 4 interval: 30 + - require: + - http: wait_for_so-kibana so-elastic-fleet-auto-configure-artifact-urls: cmd.run: @@ -44,6 +51,8 @@ so-elastic-fleet-auto-configure-artifact-urls: - retry: attempts: 4 interval: 30 + - require: + - http: wait_for_so-kibana so-elastic-fleet-package-statefile: file.managed: @@ -56,6 +65,8 @@ so-elastic-fleet-package-upgrade: - retry: attempts: 3 interval: 30 + - require: + - http: wait_for_so-kibana - onchanges: - file: /opt/so/state/elastic_fleet_packages.txt @@ -65,6 +76,8 @@ so-elastic-fleet-integrations: - retry: attempts: 3 interval: 10 + - require: + - http: wait_for_so-kibana so-elastic-agent-grid-upgrade: cmd.run: @@ -72,6 +85,8 @@ so-elastic-agent-grid-upgrade: - retry: attempts: 12 interval: 5 + - require: + - http: wait_for_so-kibana so-elastic-fleet-integration-upgrade: cmd.run: @@ -79,16 +94,22 @@ so-elastic-fleet-integration-upgrade: - retry: attempts: 3 interval: 10 + - require: + - http: wait_for_so-kibana {# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #} so-elastic-fleet-addon-integrations: cmd.run: - name: /usr/sbin/so-elastic-fleet-optional-integrations-load + - require: + - http: wait_for_so-kibana {% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %} so-elastic-defend-manage-filters-file-watch: cmd.run: - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log + - require: + - http: wait_for_so-kibana - onchanges: - file: elasticdefendcustom - file: elasticdefenddisabled diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load index e548c7f86..81a3c74be 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load @@ -108,9 +108,12 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then done # Only create the state file if all policies were created/updated successfully - if [[ "$RETURN_CODE" != "1" ]]; then + if [[ $RETURN_CODE -eq 0 ]]; then touch /opt/so/state/eaintegrations.txt + else + exit 1 fi else - exit $RETURN_CODE + echo "Fleet integration policies already loaded." + exit 0 fi diff --git a/salt/kibana/enabled.sls b/salt/kibana/enabled.sls index 04f44e508..4bb5fef9c 100644 --- a/salt/kibana/enabled.sls +++ b/salt/kibana/enabled.sls @@ -10,6 +10,7 @@ include: - kibana.config + - kibana.healthcheck - kibana.sostatus # Start the kibana docker @@ -59,6 +60,8 @@ so-kibana: {% endif %} - watch: - file: kibanaconfig + - require_in: + - http: wait_for_so-kibana delete_so-kibana_so-status.disabled: file.uncomment: diff --git a/salt/kibana/healthcheck.sls b/salt/kibana/healthcheck.sls new file mode 100644 index 000000000..f209c3964 --- /dev/null +++ b/salt/kibana/healthcheck.sls @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states %} +{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} + +wait_for_so-kibana: + http.wait_for_successful_query: + - name: "https://localhost:5601/api/status" + - username: 'so_elastic' + - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}' + - ssl: True + - verify_ssl: False + - status: 200 + - wait_for: 300 + - request_interval: 15 + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} From 46655860e926ad0b6692b5ba11ca6218b2d5b9e8 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 10 Jun 2026 17:27:23 -0500 Subject: [PATCH 055/219] http --- salt/kibana/healthcheck.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/healthcheck.sls b/salt/kibana/healthcheck.sls index f209c3964..bfeb633c5 100644 --- a/salt/kibana/healthcheck.sls +++ b/salt/kibana/healthcheck.sls @@ -9,7 +9,7 @@ wait_for_so-kibana: http.wait_for_successful_query: - - name: "https://localhost:5601/api/status" + - name: "http://localhost:5601/api/status" - username: 'so_elastic' - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}' - ssl: True From 4741cc92bd8d6d84d8551ac10331f091caef2b82 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 10 Jun 2026 17:52:08 -0500 Subject: [PATCH 056/219] fleet manager start kibana if it isn't already running and wait for healthly status --- salt/elasticfleet/manager.sls | 2 +- salt/kibana/enabled.sls | 17 ++++++++++++++--- salt/kibana/healthcheck.sls | 27 --------------------------- 3 files changed, 15 insertions(+), 31 deletions(-) delete mode 100644 salt/kibana/healthcheck.sls diff --git a/salt/elasticfleet/manager.sls b/salt/elasticfleet/manager.sls index a0aa83460..c9fe91d4d 100644 --- a/salt/elasticfleet/manager.sls +++ b/salt/elasticfleet/manager.sls @@ -9,7 +9,7 @@ include: - elasticfleet.config - - kibana.healthcheck + - kibana.enabled # If enabled, automatically update Fleet Logstash Outputs {% if ELASTICFLEETMERGED.config.server.enable_auto_configuration %} diff --git a/salt/kibana/enabled.sls b/salt/kibana/enabled.sls index 4bb5fef9c..a2fb6cde9 100644 --- a/salt/kibana/enabled.sls +++ b/salt/kibana/enabled.sls @@ -6,11 +6,11 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls.split('.')[0] in allowed_states %} {% from 'docker/docker.map.jinja' import DOCKERMERGED %} +{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} {% from 'vars/globals.map.jinja' import GLOBALS %} include: - kibana.config - - kibana.healthcheck - kibana.sostatus # Start the kibana docker @@ -60,8 +60,19 @@ so-kibana: {% endif %} - watch: - file: kibanaconfig - - require_in: - - http: wait_for_so-kibana + +wait_for_so-kibana: + http.wait_for_successful_query: + - name: "http://localhost:5601/api/status" + - username: 'so_elastic' + - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}' + - ssl: True + - verify_ssl: False + - status: 200 + - wait_for: 300 + - request_interval: 15 + - require: + - docker_container: so-kibana delete_so-kibana_so-status.disabled: file.uncomment: diff --git a/salt/kibana/healthcheck.sls b/salt/kibana/healthcheck.sls deleted file mode 100644 index bfeb633c5..000000000 --- a/salt/kibana/healthcheck.sls +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} -{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} - -wait_for_so-kibana: - http.wait_for_successful_query: - - name: "http://localhost:5601/api/status" - - username: 'so_elastic' - - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}' - - ssl: True - - verify_ssl: False - - status: 200 - - wait_for: 300 - - request_interval: 15 - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} From f083db67e4b735aaad2e4a464115e003d2d767c8 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 11 Jun 2026 08:17:39 -0400 Subject: [PATCH 057/219] disable telemetry for automated tests --- setup/so-setup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup/so-setup b/setup/so-setup index 6c77e781c..f9c8c4460 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -223,6 +223,8 @@ if [ -n "$test_profile" ]; then WEBPASSWD1=0n10nus3r WEBPASSWD2=0n10nus3r NODE_DESCRIPTION="${HOSTNAME} - ${install_type} - ${MSRVIP_OFFSET}" + # opt out of telemetry for automated testing + telemetry=1 update_sudoers_for_testing fi From b8bf684077c8e06f0420807f220a530d1f433ed3 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 11 Jun 2026 08:18:38 -0400 Subject: [PATCH 058/219] ver --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 03e153fda..944880fa1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.0.0-kilo +3.2.0 From 780d9faf0dd2b346ce4bd5d06a7b665a0584b0e1 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 11 Jun 2026 12:08:32 -0400 Subject: [PATCH 059/219] Parallelize so-elasticsearch-ilm-policy-load PUTs Run the ~300 ILM policy PUTs concurrently (bounded to 10 in flight via a throttle gate) instead of one serial curl per policy. Adds a put_policy helper and waits for all background jobs before exiting. Preserves policy parity; only the scheduling changes. Drops the dead empty sid cookie arg (falls back to basic auth from curl.config as before). --- .../so-elasticsearch-ilm-policy-load | 50 +++++++++++-------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load index 7988c1905..e6a5f4eb8 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load @@ -6,6 +6,23 @@ . /usr/sbin/so-common +MAX_JOBS=10 + +put_policy() { + local desc="$1" policyname="$2" data="$3" + echo "Setting up ${desc} policy..." + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L \ + -X PUT "https://localhost:9200/_ilm/policy/${policyname}" \ + -H 'Content-Type: application/json' -d"${data}" +} + +# Block until fewer than MAX_JOBS background curls are running. +throttle() { + while (( $(jobs -rp | wc -l) >= MAX_JOBS )); do + wait -n + done +} + {%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %} {%- if GLOBALS.role != "so-heavynode" %} {%- from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %} @@ -14,35 +31,26 @@ {%- for index, settings in ES_INDEX_SETTINGS.items() %} {%- if settings.policy is defined %} {%- if index == 'so-logs-detections.alerts' %} - echo - echo "Setting up so-logs-detections.alerts-so policy..." - curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-so" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' - echo + throttle + put_policy "so-logs-detections.alerts-so" "{{ index }}-so" '{ "policy": {{ settings.policy | tojson(true) }} }' & {%- elif index == 'so-logs-soc' %} - echo - echo "Setting up so-soc-logs policy..." - curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/so-soc-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' - echo - echo - echo "Setting up {{ index }}-logs policy..." - curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' - echo + throttle + put_policy "so-soc-logs" "so-soc-logs" '{ "policy": {{ settings.policy | tojson(true) }} }' & + throttle + put_policy "{{ index }}-logs" "{{ index }}-logs" '{ "policy": {{ settings.policy | tojson(true) }} }' & {%- else %} - echo - echo "Setting up {{ index }}-logs policy..." - curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' - echo + throttle + put_policy "{{ index }}-logs" "{{ index }}-logs" '{ "policy": {{ settings.policy | tojson(true) }} }' & {%- endif %} {%- endif %} {%- endfor %} -echo {%- if GLOBALS.role != "so-heavynode" %} {%- for index, settings in ALL_ADDON_SETTINGS.items() %} {%- if settings.policy is defined %} - echo - echo "Setting up {{ index }}-logs policy..." - curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' - echo + throttle + put_policy "{{ index }}-logs" "{{ index }}-logs" '{ "policy": {{ settings.policy | tojson(true) }} }' & {%- endif %} {%- endfor %} {%- endif %} + +wait From 07d3b148b553907d17b9a01d6b8daa6333ae1ceb Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 11 Jun 2026 13:37:26 -0400 Subject: [PATCH 060/219] fix output --- .../sbin_jinja/so-elasticsearch-ilm-policy-load | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load index e6a5f4eb8..a75023cae 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load @@ -8,12 +8,19 @@ MAX_JOBS=10 +# Policies are loaded concurrently (up to MAX_JOBS at a time) for speed. Each policy's block is +# printed atomically the moment its curl returns, so output appears in COMPLETION ORDER, not the +# order policies are defined in configuration. +echo "Loading ILM policies concurrently; output below appears in completion order, not configuration order." +echo + put_policy() { - local desc="$1" policyname="$2" data="$3" - echo "Setting up ${desc} policy..." - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L \ + local desc="$1" policyname="$2" data="$3" result + result=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L \ -X PUT "https://localhost:9200/_ilm/policy/${policyname}" \ - -H 'Content-Type: application/json' -d"${data}" + -H 'Content-Type: application/json' -d"${data}") + # Single atomic write so concurrent jobs don't interleave; prints live as each curl finishes. + printf 'Setting up %s policy...\n%s\n\n' "${desc}" "${result}" } # Block until fewer than MAX_JOBS background curls are running. From f23652397c966e0c9627eee9442d097133af25d1 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 11 Jun 2026 13:57:56 -0400 Subject: [PATCH 061/219] Speed up so-elastic-fleet-optional-integrations-load decision logic Replace the per-package decision loop (which forked ~10 processes per package and rebuilt a growing JSON file on every add -> O(n^2)) with two jq passes: one prints the status messages, one builds the bulk install list. A vnum/needs() jq definition reproduces the previous version_conversion/compare_versions and excluded/subscription/installed/ upgrade/in-use logic exactly. Also fetch each agent policy once and extract non-default package names locally instead of re-fetching the policy per integration (1+K -> 1 GET per policy). Install behavior is unchanged. --- ...o-elastic-fleet-optional-integrations-load | 155 +++++++----------- 1 file changed, 57 insertions(+), 98 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load index ab38b7065..7579123cb 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load @@ -16,7 +16,6 @@ STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json -BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json INTEGRATION_PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json INPUT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_input_package_components.json @@ -29,29 +28,6 @@ PENDING_UPDATE=false # Requiring some level of manual Elastic Stack configuration before installation EXCLUDED_INTEGRATIONS=('apm') -version_conversion(){ - version=$1 - echo "$version" | awk -F '.' '{ printf("%d%03d%03d\n", $1, $2, $3); }' -} - -compare_versions() { - version1=$1 - version2=$2 - - # Convert versions to numbers - num1=$(version_conversion "$version1") - num2=$(version_conversion "$version2") - - # Compare using bc - if (( $(echo "$num1 < $num2" | bc -l) )); then - echo "less" - elif (( $(echo "$num1 > $num2" | bc -l) )); then - echo "greater" - else - echo "equal" - fi -} - IFS=$'\n' agent_policies=$(elastic_fleet_agent_policy_ids) if [ $? -ne 0 ]; then @@ -63,23 +39,23 @@ default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.l in_use_integrations=() +# Fetch each agent policy once; its package_policies[] already contain both the integration name +# and the .package.name, so extract all non-default package names locally in a single jq instead +# of re-fetching the same policy per integration. +default_packages_json=$(printf '%s\n' "${default_packages[@]}" | jq -R . | jq -s '.') for AGENT_POLICY in $agent_policies; do - if ! integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY"); then + if ! policy_json=$(fleet_api "agent_policies/$AGENT_POLICY"); then # skip the agent policy if we can't get required info, let salt retry. Integrations loaded by this script are non-default integrations. echo "Skipping $AGENT_POLICY.. " continue fi - for INTEGRATION in $integrations; do - if ! PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION"); then - echo "Not adding $INTEGRATION, couldn't get package name" - continue - fi - # non-default integrations that are in-use in any policy - if ! [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then - in_use_integrations+=("$PACKAGE_NAME") - fi - done + # non-default integrations that are in-use in any policy + while IFS= read -r PACKAGE_NAME; do + [ -n "$PACKAGE_NAME" ] && in_use_integrations+=("$PACKAGE_NAME") + done < <(jq -r --argjson def "$default_packages_json" \ + '.item.package_policies[].package.name | select(. as $n | ($def | index($n)) | not)' \ + <<<"$policy_json") done if [[ -f $STATE_FILE_SUCCESS ]]; then @@ -90,72 +66,55 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then rm -f $INSTALLED_PACKAGE_LIST echo $latest_package_list | jq '{packages: [.items[] | {name: .name, latest_version: .version, installed_version: .installationInfo.version, subscription: .conditions.elastic.subscription }]}' >> $INSTALLED_PACKAGE_LIST - while read -r package; do - # get package details - package_name=$(echo "$package" | jq -r '.name') - latest_version=$(echo "$package" | jq -r '.latest_version') - installed_version=$(echo "$package" | jq -r '.installed_version') - subscription=$(echo "$package" | jq -r '.subscription') - bulk_package=$(echo "$package" | jq '{name: .name, version: .latest_version}' ) + # Build the bulk install list and the per-package status messages with two jq passes + # instead of a per-package bash loop. The old loop forked ~10 processes per package + # (5 jq + awk/bc for the version compare) and re-parsed/rewrote a growing JSON file on + # every add (O(n^2)). Selection and messages below are identical to that logic. + SUB={% if SUB %}true{% else %}false{% endif %} + AUTOUP={% if AUTO_UPGRADE_INTEGRATIONS %}true{% else %}false{% endif %} + EXCLUDED_JSON=$(printf '%s\n' "${EXCLUDED_INTEGRATIONS[@]}" | jq -R 'select(length>0)' | jq -s '.') + INUSE_JSON=$(printf '%s\n' "${in_use_integrations[@]}" | jq -R 'select(length>0)' | jq -s 'unique') - if [[ ! "${EXCLUDED_INTEGRATIONS[@]}" =~ "$package_name" ]]; then - {% if not SUB %} - if [[ "$subscription" != "basic" && "$subscription" != "null" && -n "$subscription" ]]; then - # pass over integrations that require non-basic elastic license - echo "$package_name integration requires an Elastic license of $subscription or greater... skipping" - continue - else - if [[ "$installed_version" == "null" || -z "$installed_version" ]]; then - echo "$package_name is not installed... Adding to next update." - jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST + # vnum replicates the previous version_conversion (%d%03d%03d of the first three dotted + # fields); needs() replicates the excluded/subscription/installed/upgrade/in-use logic. + JQ_DECISION=' +def vnum: + [ (split(".")|.[0:3][] | gsub("[^0-9].*";"") | (if .=="" then "0" else . end) | tonumber) ] + | (.[0]//0)*1000000 + (.[1]//0)*1000 + (.[2]//0); +def needs($sub;$autoup;$excluded;$inuse): + .name as $n + | ($n | IN($excluded[]) | not) + and ( $sub or (.subscription==null or .subscription=="basic" or .subscription=="") ) + and ( (.installed_version==null or .installed_version=="") + or ( ((.latest_version|vnum) > (.installed_version|vnum)) + and ( $autoup or ($n | IN($inuse[]) | not) ) ) );' - PENDING_UPDATE=true - else - results=$(compare_versions "$latest_version" "$installed_version") - if [ $results == "greater" ]; then - {#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations #} - {%- if not AUTO_UPGRADE_INTEGRATIONS %} - if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then - {%- endif %} - echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update." - jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST + JQ_ARGS=(--argjson sub "$SUB" --argjson autoup "$AUTOUP" --argjson excluded "$EXCLUDED_JSON" --argjson inuse "$INUSE_JSON") - PENDING_UPDATE=true - {%- if not AUTO_UPGRADE_INTEGRATIONS %} - else - echo "skipping available upgrade for in use integration - $package_name." - fi - {%- endif %} - fi - fi - fi - {% else %} - if [[ "$installed_version" == "null" || -z "$installed_version" ]]; then - echo "$package_name is not installed... Adding to next update." - jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST - PENDING_UPDATE=true - else - results=$(compare_versions "$latest_version" "$installed_version") - if [ $results == "greater" ]; then - {#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations #} - {%- if not AUTO_UPGRADE_INTEGRATIONS %} - if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then - {%- endif %} - echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update." - jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST - PENDING_UPDATE=true - {%- if not AUTO_UPGRADE_INTEGRATIONS %} - else - echo "skipping available upgrade for in use integration - $package_name." - fi - {%- endif %} - fi - fi - {% endif %} - else - echo "Skipping $package_name..." - fi - done <<< "$(jq -c '.packages[]' "$INSTALLED_PACKAGE_LIST")" + # (a) Per-package status messages (parity with the previous echo output). + jq -r "${JQ_ARGS[@]}" "$JQ_DECISION"' + .packages[] + | .name as $n + | if ($n|IN($excluded[])) then "Skipping \($n)..." + elif (($sub|not) and (.subscription!=null and .subscription!="basic" and .subscription!="")) then + "\($n) integration requires an Elastic license of \(.subscription) or greater... skipping" + elif (.installed_version==null or .installed_version=="") then + "\($n) is not installed... Adding to next update." + elif ((.latest_version|vnum) > (.installed_version|vnum)) then + (if ($autoup or ($n|IN($inuse[])|not)) + then "\($n) is at version \(.installed_version) latest version is \(.latest_version)... Adding to next update." + else "skipping available upgrade for in use integration - \($n)." end) + else empty end + ' "$INSTALLED_PACKAGE_LIST" + + # (b) The bulk install list, built in a single pass. + jq "${JQ_ARGS[@]}" "$JQ_DECISION"' + {packages: [ .packages[] | select(needs($sub;$autoup;$excluded;$inuse)) | {name, version: .latest_version} ]} + ' "$INSTALLED_PACKAGE_LIST" > "$BULK_INSTALL_PACKAGE_LIST" + + if jq -e '.packages | length > 0' "$BULK_INSTALL_PACKAGE_LIST" >/dev/null; then + PENDING_UPDATE=true + fi if [ "$PENDING_UPDATE" = true ]; then # Run chunked install of packages From 6c42c419e2f6c89d3c0b57d5e909b5e532b59a2e Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 11 Jun 2026 15:42:41 -0400 Subject: [PATCH 062/219] Serialize ILM policy-load output with flock to stop interleaving A single printf per block was not actually one write() call, so concurrent jobs still occasionally interleaved their label and response lines. Hold an flock around just the printf (curl still runs in parallel) so each policy's block prints intact, keeping live completion-order streaming. --- .../sbin_jinja/so-elasticsearch-ilm-policy-load | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load index a75023cae..a884f2e2f 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load @@ -8,9 +8,13 @@ MAX_JOBS=10 +# Lock used to serialize block writes so concurrent jobs never interleave their output. +ILM_OUTPUT_LOCK=$(mktemp) +trap 'rm -f "$ILM_OUTPUT_LOCK"' EXIT + # Policies are loaded concurrently (up to MAX_JOBS at a time) for speed. Each policy's block is -# printed atomically the moment its curl returns, so output appears in COMPLETION ORDER, not the -# order policies are defined in configuration. +# printed the moment its curl returns, so output appears in COMPLETION ORDER, not the order +# policies are defined in configuration. echo "Loading ILM policies concurrently; output below appears in completion order, not configuration order." echo @@ -19,8 +23,11 @@ put_policy() { result=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L \ -X PUT "https://localhost:9200/_ilm/policy/${policyname}" \ -H 'Content-Type: application/json' -d"${data}") - # Single atomic write so concurrent jobs don't interleave; prints live as each curl finishes. - printf 'Setting up %s policy...\n%s\n\n' "${desc}" "${result}" + # curl above ran in parallel; serialize just this block write so concurrent jobs never interleave. + { + flock 200 + printf 'Setting up %s policy...\n%s\n\n' "${desc}" "${result}" + } 200>>"${ILM_OUTPUT_LOCK}" } # Block until fewer than MAX_JOBS background curls are running. From b1273573ed33712613795e2f401bdf9376a90679 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 11 Jun 2026 15:50:53 -0400 Subject: [PATCH 063/219] Fix jq $def keyword collision in optional-integrations-load The agent-policy enumeration passed --argjson def, creating a jq variable $def. 'def' is a reserved keyword in jq and the deployed jq version rejects it, so the program failed to compile and in_use_integrations was left empty (silently disabling the in-use upgrade guard). Rename the arg to $defaults. --- .../sbin_jinja/so-elastic-fleet-optional-integrations-load | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load index 7579123cb..75bbc29d8 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load @@ -53,8 +53,8 @@ for AGENT_POLICY in $agent_policies; do # non-default integrations that are in-use in any policy while IFS= read -r PACKAGE_NAME; do [ -n "$PACKAGE_NAME" ] && in_use_integrations+=("$PACKAGE_NAME") - done < <(jq -r --argjson def "$default_packages_json" \ - '.item.package_policies[].package.name | select(. as $n | ($def | index($n)) | not)' \ + done < <(jq -r --argjson defaults "$default_packages_json" \ + '.item.package_policies[].package.name | select(. as $n | ($defaults | index($n)) | not)' \ <<<"$policy_json") done From d9f6cde4e19d7d303a2d21418f4e1e5aa06d74ae Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 11 Jun 2026 15:11:29 -0500 Subject: [PATCH 064/219] remove global setting from data_retention annotation --- salt/elasticsearch/soc_elasticsearch.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 251a0c1d8..c95a9467e 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -158,7 +158,6 @@ elasticsearch: - If retention is less than or equal to 14 days, max_age will be 1 day - If retention is less than or equal to 90 days, max_age will be 7 days - If retention is greater than 90 days, max_age will be 30 days - global: True forcedType: string regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$ regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d. @@ -353,7 +352,6 @@ elasticsearch: - If retention is less than or equal to 14 days, max_age will be 1 day - If retention is less than or equal to 90 days, max_age will be 7 days - If retention is greater than 90 days, max_age will be 30 days - global: True forcedType: string regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$ regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d. From c5051604807b1125367ebb223f8b3c7b8fe1778d Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 11 Jun 2026 15:13:28 -0500 Subject: [PATCH 065/219] set default DLM retention 90d --- salt/elasticsearch/defaults.yaml | 122 +++++++++++++++---------------- 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 7b49074e8..ffb53ecbc 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -75,7 +75,7 @@ elasticsearch: global_overrides: # Tie this into cluster setting for data_streams.lifecycle.retention.default data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: template: settings: @@ -157,7 +157,7 @@ elasticsearch: so-common: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - agent-mappings @@ -517,7 +517,7 @@ elasticsearch: so-idh: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - agent-mappings @@ -627,7 +627,7 @@ elasticsearch: so-import: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - agent-mappings @@ -811,7 +811,7 @@ elasticsearch: so-kismet: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - kismet-mappings @@ -862,7 +862,7 @@ elasticsearch: so-kratos: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - agent-mappings @@ -932,7 +932,7 @@ elasticsearch: so-hydra: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - agent-mappings @@ -1079,7 +1079,7 @@ elasticsearch: so-logs: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - so-data-streams-mappings @@ -1161,7 +1161,7 @@ elasticsearch: so-logs-detections_x_alerts: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - so-data-streams-mappings @@ -1226,7 +1226,7 @@ elasticsearch: so-logs-elastic_agent: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - event-mappings @@ -1343,7 +1343,7 @@ elasticsearch: so-elastic-agent-monitor: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - event-mappings @@ -1407,7 +1407,7 @@ elasticsearch: so-logs-elastic_agent_x_apm_server: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - logs-elastic_agent.apm_server@package @@ -1473,7 +1473,7 @@ elasticsearch: so-logs-elastic_agent_x_auditbeat: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - logs-elastic_agent.auditbeat@package @@ -1539,7 +1539,7 @@ elasticsearch: so-logs-elastic_agent_x_cloudbeat: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - logs-elastic_agent.cloudbeat@package @@ -1605,7 +1605,7 @@ elasticsearch: so-logs-elastic_agent_x_endpoint_security: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - event-mappings @@ -1666,7 +1666,7 @@ elasticsearch: so-logs-elastic_agent_x_filebeat: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - event-mappings @@ -1727,7 +1727,7 @@ elasticsearch: so-logs-elastic_agent_x_fleet_server: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - event-mappings @@ -1785,7 +1785,7 @@ elasticsearch: so-logs-elastic_agent_x_heartbeat: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - logs-elastic_agent.heartbeat@package @@ -1851,7 +1851,7 @@ elasticsearch: so-logs-elastic_agent_x_metricbeat: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - event-mappings @@ -1912,7 +1912,7 @@ elasticsearch: so-logs-elastic_agent_x_osquerybeat: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - event-mappings @@ -1973,7 +1973,7 @@ elasticsearch: so-logs-elastic_agent_x_packetbeat: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - logs-elastic_agent.packetbeat@package @@ -2039,7 +2039,7 @@ elasticsearch: so-logs-elasticsearch_x_server: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - logs-elasticsearch.server@package @@ -2105,7 +2105,7 @@ elasticsearch: so-logs-endpoint_x_actions: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - .logs-endpoint.actions@package @@ -2166,7 +2166,7 @@ elasticsearch: so-logs-endpoint_x_action_x_responses: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - .logs-endpoint.action.responses@package @@ -2227,7 +2227,7 @@ elasticsearch: so-logs-endpoint_x_alerts: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - logs-endpoint.alerts@package @@ -2288,7 +2288,7 @@ elasticsearch: so-logs-endpoint_x_diagnostic_x_collection: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - .logs-endpoint.diagnostic.collection@package @@ -2365,7 +2365,7 @@ elasticsearch: so-logs-endpoint_x_events_x_api: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - logs-endpoint.events.api@package @@ -2426,7 +2426,7 @@ elasticsearch: so-logs-endpoint_x_events_x_file: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - logs-endpoint.events.file@package @@ -2487,7 +2487,7 @@ elasticsearch: so-logs-endpoint_x_events_x_library: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - logs-endpoint.events.library@package @@ -2548,7 +2548,7 @@ elasticsearch: so-logs-endpoint_x_events_x_network: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - logs-endpoint.events.network@package @@ -2609,7 +2609,7 @@ elasticsearch: so-logs-endpoint_x_events_x_process: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - logs-endpoint.events.process@package @@ -2670,7 +2670,7 @@ elasticsearch: so-logs-endpoint_x_events_x_registry: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - logs-endpoint.events.registry@package @@ -2731,7 +2731,7 @@ elasticsearch: so-logs-endpoint_x_events_x_security: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - logs-endpoint.events.security@package @@ -2792,7 +2792,7 @@ elasticsearch: so-logs-endpoint_x_heartbeat: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - .logs-endpoint.heartbeat@package @@ -2853,7 +2853,7 @@ elasticsearch: so-logs-http_endpoint_x_generic: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - logs-http_endpoint.generic@package @@ -2903,7 +2903,7 @@ elasticsearch: so-logs-httpjson_x_generic: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - logs-httpjson.generic@package @@ -2970,7 +2970,7 @@ elasticsearch: so-logs-osquery-manager_x_action_x_responses: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: _meta: managed: true @@ -3043,7 +3043,7 @@ elasticsearch: so-logs-osquery-manager_x_result: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: _meta: managed: true @@ -3097,7 +3097,7 @@ elasticsearch: so-logs-soc: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - agent-mappings @@ -3207,7 +3207,7 @@ elasticsearch: so-logs-system_x_application: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - event-mappings @@ -3258,7 +3258,7 @@ elasticsearch: so-logs-system_x_auth: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - event-mappings @@ -3309,7 +3309,7 @@ elasticsearch: so-logs-system_x_security: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - event-mappings @@ -3360,7 +3360,7 @@ elasticsearch: so-logs-system_x_syslog: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - event-mappings @@ -3411,7 +3411,7 @@ elasticsearch: so-logs-system_x_system: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - event-mappings @@ -3462,7 +3462,7 @@ elasticsearch: so-logs-windows_x_forwarded: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - logs-windows.forwarded@package @@ -3511,7 +3511,7 @@ elasticsearch: so-logs-windows_x_powershell: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - logs-windows.powershell@package @@ -3560,7 +3560,7 @@ elasticsearch: so-logs-windows_x_powershell_operational: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - logs-windows.powershell_operational@package @@ -3609,7 +3609,7 @@ elasticsearch: so-logs-windows_x_sysmon_operational: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - logs-windows.sysmon_operational@package @@ -3658,7 +3658,7 @@ elasticsearch: so-logs-winlog_x_winlog: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - logs-winlog.winlog@package @@ -3708,7 +3708,7 @@ elasticsearch: so-logstash: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - agent-mappings @@ -3825,7 +3825,7 @@ elasticsearch: so-metrics-endpoint_x_metadata: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - metrics-endpoint.metadata@package @@ -3874,7 +3874,7 @@ elasticsearch: so-metrics-endpoint_x_metrics: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - metrics-endpoint.metrics@package @@ -3923,7 +3923,7 @@ elasticsearch: so-metrics-endpoint_x_policy: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - metrics-endpoint.policy@package @@ -3972,7 +3972,7 @@ elasticsearch: so-metrics-fleet_server_x_agent_status: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - metrics@tsdb-settings @@ -3998,7 +3998,7 @@ elasticsearch: so-metrics-fleet_server_x_agent_versions: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - metrics@tsdb-settings @@ -4024,7 +4024,7 @@ elasticsearch: so-redis: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - agent-mappings @@ -4141,7 +4141,7 @@ elasticsearch: so-strelka: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - agent-mappings @@ -4260,7 +4260,7 @@ elasticsearch: so-suricata: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - agent-mappings @@ -4378,7 +4378,7 @@ elasticsearch: so-suricata_x_alerts: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - agent-mappings @@ -4496,7 +4496,7 @@ elasticsearch: so-syslog: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - agent-mappings @@ -4614,7 +4614,7 @@ elasticsearch: so-zeek: index_sorting: false data_stream_lifecycle: - data_retention: 7d + data_retention: 90d index_template: composed_of: - agent-mappings From 80c39d612cb65d8c10fabdcb76e0a4f886b37bd8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 11 Jun 2026 18:40:43 -0400 Subject: [PATCH 066/219] Pin NIC names by MAC via udev (run-once) from the common state Add so-nic-pin, which writes by-MAC persistent-net udev rules pinning each physical NIC to its current name so a kernel upgrade can't renumber the interfaces Security Onion binds by name (host:mainint, sensor:mainint, bond0). Gated by the drop file /opt/so/state/nic_names_pinned: run-once on highstate, and an admin can pre-create the marker to opt out. Wired into common/init.sls as pin_nic_names, guarded by a matching unless. --- salt/common/init.sls | 11 +++++ salt/common/tools/sbin/so-nic-pin | 76 +++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 salt/common/tools/sbin/so-nic-pin diff --git a/salt/common/init.sls b/salt/common/init.sls index 120e73e3e..9618d2c67 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -130,6 +130,17 @@ common_sbin: - so-pcap-import {% endif %} +# Pin physical NIC names by MAC (run-once) so a kernel upgrade can't renumber the +# interfaces SO binds by name. The marker keeps it a one-time setup; an admin can +# pre-create the marker to opt out. +pin_nic_names: + cmd.run: + - name: /usr/sbin/so-nic-pin + - unless: 'test -e /opt/so/state/nic_names_pinned' + - require: + - file: common_sbin + - file: statedir + common_sbin_jinja: file.recurse: - name: /usr/sbin diff --git a/salt/common/tools/sbin/so-nic-pin b/salt/common/tools/sbin/so-nic-pin new file mode 100644 index 000000000..9e2b4e13a --- /dev/null +++ b/salt/common/tools/sbin/so-nic-pin @@ -0,0 +1,76 @@ +#!/bin/bash +# +# so-nic-pin — pin physical NIC names by permanent MAC via classic by-MAC udev +# rules, so a kernel upgrade can't renumber them. +# +# Security Onion binds its management and monitor interfaces BY NAME in pillar +# (host:mainint, sensor:mainint, and bond0 is built on a specific physical NIC). +# A kernel upgrade can change the kernel/systemd-udevd predictable-naming output +# and renumber those NICs (e.g. enp1s0 -> enp2s0), which breaks the grid: the +# pillar references a name that no longer exists and bond/bridge bring-up fails. +# +# This writes /etc/udev/rules.d/70-persistent-net.rules pinning each PHYSICAL NIC +# to its CURRENT name by its PERMANENT MAC, freezing the names across future kernel +# changes. It only writes the rules file; it does NOT live-trigger a rename (the +# rules apply on the next boot/kernel, and a live rename would be disruptive). +# +# Run-once: gated by the drop file /opt/so/state/nic_names_pinned. If the marker is +# present the script does nothing, so an admin can pre-create it to opt out. Invoked +# from the common state on every highstate; the marker keeps it a one-time setup. + +NET_RULES_FILE="/etc/udev/rules.d/70-persistent-net.rules" +MARKER="/opt/so/state/nic_names_pinned" + +log() { echo -e "[so-nic-pin] $*"; } + +# Echo " " for every PHYSICAL NIC. A physical NIC is backed by a +# real device (has device/driver), which excludes bond0/sobridge/docker0/veth*/lo whose +# MACs are dynamic and must never be pinned. The PERMANENT MAC is used (ethtool -P, with +# fallbacks), not the current one: an enslaved bond member's current MAC is rewritten to +# the bond's, so matching on it would be wrong/ambiguous. +physical_nics() { + local path n mac + for path in /sys/class/net/*; do + n="${path##*/}" + [ "$n" = "lo" ] && continue + [ -e "${path}/device/driver" ] || continue # real device only + mac="$(ethtool -P "$n" 2>/dev/null | awk '/Permanent address/{print $NF}')" + case "$mac" in ""|00:00:00:00:00:00) mac="$(cat "${path}/bonding_slave/perm_hwaddr" 2>/dev/null)" ;; esac + case "$mac" in ""|00:00:00:00:00:00) mac="$(cat "${path}/address" 2>/dev/null)" ;; esac + case "$mac" in ""|00:00:00:00:00:00) continue ;; esac + echo "$n $mac" + done +} + +# Turn " " lines on stdin into classic by-MAC persistent-net udev rules. +render_net_rules() { + echo "# Generated by so-nic-pin: pin NIC names by MAC so kernel upgrades can't renumber them." + echo "# Security Onion binds its management/monitor interfaces by name; do not hand-edit." + local n mac + while read -r n mac; do + [ -n "$n" ] || continue + printf 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="%s", NAME="%s"\n' \ + "$mac" "$n" + done +} + +[ "$(id -u)" -eq 0 ] || exit 0 # salt runs us as root; bail quietly otherwise +[ -e "${MARKER}" ] && exit 0 # run-once guard (mirrors the state's unless) + +nics="$(physical_nics)" +if [ -z "${nics}" ]; then + log "no physical NICs detected — nothing to pin (will retry on next highstate)" + exit 0 # do NOT drop the marker; let it retry later +fi + +log "pinning physical NICs by permanent MAC:" +echo "${nics}" | sed 's/^/ /' + +[ -f "${NET_RULES_FILE}" ] && cp -f "${NET_RULES_FILE}" "${NET_RULES_FILE}.bak" +echo "${nics}" | render_net_rules > "${NET_RULES_FILE}" || { + log "ERROR: failed to write ${NET_RULES_FILE}" + exit 1 +} + +mkdir -p "$(dirname "${MARKER}")" && touch "${MARKER}" +log "wrote ${NET_RULES_FILE} ($(grep -c '^SUBSYSTEM' "${NET_RULES_FILE}") NIC(s) pinned); dropped ${MARKER}" From ae6a705ce16d5fe9f228be16ef2335b78ee2fba1 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 12 Jun 2026 09:38:41 -0400 Subject: [PATCH 067/219] Speed up so-elastic-fleet-integration-policy-load Fetch each agent policy once per group instead of refetching the full policy (plus a fresh Kibana session cookie) for every integration file, and dispatch the create/update writes as throttled background jobs. Adds elastic_fleet_load_integrations_dir and elastic_fleet_throttle to so-elastic-fleet-common, reusing the bounded-concurrency pattern from so-elasticsearch-ilm-policy-load. Replaces the four serial loops in the loader with one call per agent policy. --- .../tools/sbin/so-elastic-fleet-common | 64 +++++++++++++ .../so-elastic-fleet-integration-policy-load | 95 +++---------------- 2 files changed, 78 insertions(+), 81 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common index 91fa787f2..d4f40b3ed 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common @@ -30,6 +30,70 @@ fleet_api() { curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/${QUERYPATH}" "$@" --retry 3 --retry-delay 10 --fail 2>/dev/null } +# Max number of concurrent Fleet write jobs (create/update). Override via env if needed. +MAX_FLEET_JOBS=${MAX_FLEET_JOBS:-10} + +# Block until fewer than MAX_FLEET_JOBS background jobs are running. +elastic_fleet_throttle() { + while (( $(jobs -rp | wc -l) >= MAX_FLEET_JOBS )); do + wait -n + done +} + +# Load every integration JSON in a directory into a single agent policy. +# The agent policy is fetched ONCE (not per file), and the create/update writes +# are dispatched as throttled background jobs. +# $1 AGENT_POLICY - the agent policy id/name to load integrations into +# $2 DIR - directory of integration *.json files +# $3 LABEL - human-readable label for log output +# $4 SKIP_CREATE_NAME - (optional) integration name to skip when creating (still updated if present) +# Returns 1 if any integration failed to create/update. +elastic_fleet_load_integrations_dir() { + local AGENT_POLICY=$1 + local DIR=$2 + local LABEL=$3 + local SKIP_CREATE_NAME=$4 + local POLICY_JSON FAIL_FILE INTEGRATION NAME ID + + FAIL_FILE=$(mktemp) + + # Fetch the agent policy a single time; we look up integration ids locally below. + POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY") + + for INTEGRATION in "$DIR"/*.json; do + [ -e "$INTEGRATION" ] || continue + NAME=$(jq -r .name "$INTEGRATION") + ID=$(jq -r --arg n "$NAME" '.item.package_policies[]? | select(.name==$n) | .id' <<<"$POLICY_JSON") + + elastic_fleet_throttle + { + if [ -n "$ID" ]; then + printf "\n\n%s - Updating integration %s\n" "$LABEL" "$NAME" + if ! elastic_fleet_integration_update "$ID" "@$INTEGRATION"; then + flock 9; echo "update ${INTEGRATION##*/}" >&9 + fi + elif [ -n "$SKIP_CREATE_NAME" ] && [ "$NAME" == "$SKIP_CREATE_NAME" ]; then + printf "\n\n%s - Skipping creation of %s\n" "$LABEL" "$NAME" + else + printf "\n\n%s - Creating integration %s\n" "$LABEL" "$NAME" + if ! elastic_fleet_integration_create "@$INTEGRATION"; then + flock 9; echo "create ${INTEGRATION##*/}" >&9 + fi + fi + } 9>>"$FAIL_FILE" & + done + wait + + local rc=0 + if [ -s "$FAIL_FILE" ]; then + printf "\n%s: failed integrations:\n" "$LABEL" + cat "$FAIL_FILE" + rc=1 + fi + rm -f "$FAIL_FILE" + return $rc +} + elastic_fleet_integration_check() { AGENT_POLICY=$1 diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load index e548c7f86..73d569dd9 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load @@ -18,93 +18,26 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then # Third, configure Elastic Defend Integration seperately /usr/sbin/so-elastic-fleet-integration-policy-elastic-defend + # Each group fetches its agent policy once and dispatches create/update writes concurrently. + # Initial Endpoints - for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/endpoints-initial/*.json; do - printf "\n\nInitial Endpoints Policy - Loading $INTEGRATION\n" - elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION" - if [ -n "$INTEGRATION_ID" ]; then - printf "\n\nIntegration $NAME exists - Updating integration\n" - if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then - echo -e "\nFailed to update integration for ${INTEGRATION##*/}" - RETURN_CODE=1 - continue - fi - else - printf "\n\nIntegration does not exist - Creating integration\n" - if ! elastic_fleet_integration_create "@$INTEGRATION"; then - echo -e "\nFailed to create integration for ${INTEGRATION##*/}" - RETURN_CODE=1 - continue - fi - fi - done + elastic_fleet_load_integrations_dir "endpoints-initial" \ + /opt/so/conf/elastic-fleet/integrations/endpoints-initial "Initial Endpoints Policy" || RETURN_CODE=1 # Grid Nodes - General - for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_general/*.json; do - printf "\n\nGrid Nodes Policy_General - Loading $INTEGRATION\n" - elastic_fleet_integration_check "so-grid-nodes_general" "$INTEGRATION" - if [ -n "$INTEGRATION_ID" ]; then - printf "\n\nIntegration $NAME exists - Updating integration\n" - if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then - echo -e "\nFailed to update integration for ${INTEGRATION##*/}" - RETURN_CODE=1 - continue - fi - else - printf "\n\nIntegration does not exist - Creating integration\n" - if ! elastic_fleet_integration_create "@$INTEGRATION"; then - echo -e "\nFailed to create integration for ${INTEGRATION##*/}" - RETURN_CODE=1 - continue - fi - fi - done + elastic_fleet_load_integrations_dir "so-grid-nodes_general" \ + /opt/so/conf/elastic-fleet/integrations/grid-nodes_general "Grid Nodes Policy_General" || RETURN_CODE=1 # Grid Nodes - Heavy - for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy/*.json; do - printf "\n\nGrid Nodes Policy_Heavy - Loading $INTEGRATION\n" - elastic_fleet_integration_check "so-grid-nodes_heavy" "$INTEGRATION" - if [ -n "$INTEGRATION_ID" ]; then - printf "\n\nIntegration $NAME exists - Updating integration\n" - if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then - echo -e "\nFailed to update integration for ${INTEGRATION##*/}" - RETURN_CODE=1 - continue - fi - else - printf "\n\nIntegration does not exist - Creating integration\n" - if ! elastic_fleet_integration_create "@$INTEGRATION"; then - echo -e "\nFailed to create integration for ${INTEGRATION##*/}" - RETURN_CODE=1 - continue - fi - fi - done + elastic_fleet_load_integrations_dir "so-grid-nodes_heavy" \ + /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy "Grid Nodes Policy_Heavy" || RETURN_CODE=1 - # Fleet Server - Optional integrations - for INTEGRATION in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json; do - if ! [ "$INTEGRATION" == "/opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json" ]; then - FLEET_POLICY=`echo "$INTEGRATION"| cut -d'/' -f7` - printf "\n\nFleet Server Policy - Loading $INTEGRATION\n" - elastic_fleet_integration_check "$FLEET_POLICY" "$INTEGRATION" - if [ -n "$INTEGRATION_ID" ]; then - printf "\n\nIntegration $NAME exists - Updating integration\n" - if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then - echo -e "\nFailed to update integration for ${INTEGRATION##*/}" - RETURN_CODE=1 - continue - fi - else - printf "\n\nIntegration does not exist - Creating integration\n" - if [ "$NAME" != "elasticsearch-logs" ]; then - if ! elastic_fleet_integration_create "@$INTEGRATION"; then - echo -e "\nFailed to create integration for ${INTEGRATION##*/}" - RETURN_CODE=1 - continue - fi - fi - fi - fi + # Fleet Server - Optional integrations (one agent policy per FleetServer_* directory) + for FLEET_DIR in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/; do + [ -d "$FLEET_DIR" ] || continue + FLEET_POLICY=$(basename "$FLEET_DIR") + elastic_fleet_load_integrations_dir "$FLEET_POLICY" \ + "${FLEET_DIR%/}" "Fleet Server Policy" "elasticsearch-logs" || RETURN_CODE=1 done # Only create the state file if all policies were created/updated successfully From 9031c1fd220acd1a074412e1241fa81ddc15475f Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 12 Jun 2026 11:18:59 -0400 Subject: [PATCH 068/219] userid vs names --- salt/kafka/enabled.sls | 2 +- salt/kibana/enabled.sls | 2 +- salt/logstash/enabled.sls | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/kafka/enabled.sls b/salt/kafka/enabled.sls index 06fa701c6..504efea05 100644 --- a/salt/kafka/enabled.sls +++ b/salt/kafka/enabled.sls @@ -32,7 +32,7 @@ so-kafka: - networks: - sobridge: - ipv4_address: {{ DOCKERMERGED.containers['so-kafka'].ip }} - - user: kafka + - user: "960" - environment: KAFKA_HEAP_OPTS: -Xmx2G -Xms1G KAFKA_OPTS: "-javaagent:/opt/jolokia/agents/jolokia-agent-jvm-javaagent.jar=port=8778,host={{ DOCKERMERGED.containers['so-kafka'].ip }},policyLocation=file:/opt/jolokia/jolokia.xml {%- if KAFKA_EXTERNAL_ACCESS %} -Djava.security.auth.login.config=/opt/kafka/config/kafka_server_jaas.conf {% endif -%}" diff --git a/salt/kibana/enabled.sls b/salt/kibana/enabled.sls index a2fb6cde9..1257f66c6 100644 --- a/salt/kibana/enabled.sls +++ b/salt/kibana/enabled.sls @@ -18,7 +18,7 @@ so-kibana: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kibana:{{ GLOBALS.so_version }} - hostname: kibana - - user: kibana + - user: "932:0" - networks: - sobridge: - ipv4_address: {{ DOCKERMERGED.containers['so-kibana'].ip }} diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index d89304144..60fed4ced 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -33,7 +33,7 @@ so-logstash: - networks: - sobridge: - ipv4_address: {{ DOCKERMERGED.containers['so-logstash'].ip }} - - user: logstash + - user: "931:0" - extra_hosts: {% for node in LOGSTASH_NODES %} {% for hostname, ip in node.items() %} From 43f72c1f9f0c8d7c874781ae2ac54844f74b8029 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 12 Jun 2026 15:11:34 -0400 Subject: [PATCH 069/219] Parallelize so-elasticsearch-templates-load template PUTs Load component and index templates as throttled background jobs (max 10 concurrent) instead of sequential curl PUTs, matching the bounded-concurrency + flock-serialized-output pattern used by the fleet/ILM load scripts. Keeps a wait barrier between the component phase and the index phase so index templates never load before their referenced component templates. Failures are tracked via per-job marker files since counter increments can't escape background subshells. --- .../sbin/so-elasticsearch-templates-load | 143 +++++++++++++----- 1 file changed, 107 insertions(+), 36 deletions(-) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load index a0ebd66e8..f3c830f1c 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load @@ -11,10 +11,8 @@ ADDON_STATEFILE_SUCCESS=/opt/so/state/addon_estemplates.txt ELASTICSEARCH_TEMPLATES_DIR="/opt/so/conf/elasticsearch/templates" SO_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/index" ADDON_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/addon-index" -SO_LOAD_FAILURES=0 -ADDON_LOAD_FAILURES=0 -SO_LOAD_FAILURES_NAMES=() -ADDON_LOAD_FAILURES_NAMES=() +FAILED_NAMES=() +FAILED_COUNT=0 IS_HEAVYNODE="false" FORCE="false" VERBOSE="false" @@ -46,20 +44,86 @@ while [[ $# -gt 0 ]]; do shift done +# Max number of concurrent template PUT jobs. Override via env if needed. +MAX_TEMPLATE_JOBS=${MAX_TEMPLATE_JOBS:-10} + +# Block until fewer than MAX_TEMPLATE_JOBS background jobs are running. +template_throttle() { + while (( $(jobs -rp | wc -l) >= MAX_TEMPLATE_JOBS )); do + wait -n + done +} + +# Per-job failure markers and an output lock for serializing parallel job output. +# Each failed load drops one file (named after the template) into FAIL_DIR; the +# output of each job is flushed as a single block under flock so concurrent jobs +# never interleave their (chatty) retry output. +FAIL_DIR=$(mktemp -d) +OUTPUT_LOCK="${FAIL_DIR}/.output.lock" +: > "$OUTPUT_LOCK" +trap 'rm -rf "$FAIL_DIR"' EXIT + +# Record a failure: $1 = the template name/path to report later. Slashes are +# encoded so the path becomes a safe single filename. +record_failure() { + local marker="${1//\//__}" + : > "${FAIL_DIR}/fail.${marker}" +} + +# Populate FAILED_NAMES and FAILED_COUNT from the current phase's markers. +# Must run in the current shell (not a command substitution) so the array sticks. +collect_failures() { + FAILED_NAMES=() + FAILED_COUNT=0 + local f name + shopt -s nullglob + for f in "${FAIL_DIR}"/fail.*; do + name="${f##*/fail.}" + name="${name//__//}" + FAILED_NAMES+=("$name") + FAILED_COUNT=$((FAILED_COUNT + 1)) + done + shopt -u nullglob +} + +# Clear markers and names between phases so SO and addon counts stay independent. +reset_failures() { + shopt -s nullglob + rm -f "${FAIL_DIR}"/fail.* + shopt -u nullglob + FAILED_NAMES=() + FAILED_COUNT=0 +} + +# Print a block of text atomically (under the shared output lock) so the output +# of concurrent background jobs is not interleaved. +locked_echo() { + { flock 9; printf '%s\n' "$1"; } 9>>"$OUTPUT_LOCK" +} + +# Loads one template file via PUT. Intended to be dispatched as a background job. +# $1 uri - e.g. _component_template/foo or _index_template/foo +# $2 file - path to the template JSON +# $3 report_name - name/path to record if this load fails load_template() { local uri="$1" local file="$2" + local report_name="$3" + local out rc=0 block - echo "Loading template file $file" - if ! output=$(retry 3 3 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"); then - echo "$output" - - return 1 - + # Capture everything (including retry's diagnostic chatter) into one block so + # concurrent jobs never interleave; the whole block is flushed under one flock. + block="Loading template file $file"$'\n' + if ! out=$(retry 3 3 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}" 2>&1); then + block+="$out"$'\n' + rc=1 elif [[ "$VERBOSE" == "true" ]]; then - echo "$output" + block+="$out"$'\n' fi + { flock 9; printf '%s' "$block"; } 9>>"$OUTPUT_LOCK" + + (( rc != 0 )) && record_failure "$report_name" } check_required_component_template_exists() { @@ -110,6 +174,9 @@ load_component_templates() { return fi + # Dispatch loads as throttled background jobs. The barrier (wait) happens in + # the caller after all component groups have been dispatched, since index + # templates must not load until every component template is in place. for component in "$pattern"/*.json; do tmpl_name=$(basename "${component%.json}") @@ -118,10 +185,8 @@ load_component_templates() { tmpl_name="${tmpl_name%-mappings}-mappings" fi - if ! load_template "_component_template/${tmpl_name}" "$component"; then - SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1)) - SO_LOAD_FAILURES_NAMES+=("$component") - fi + template_throttle + load_template "_component_template/${tmpl_name}" "$component" "$component" & done } @@ -180,6 +245,9 @@ if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]] && index_templates_e load_component_templates "Elastic Agent" "elastic-agent" load_component_templates "Security Onion" "so" + # Barrier: every component template PUT must complete before we snapshot the + # component template list and start loading index templates that depend on them. + wait component_templates=$(so-elasticsearch-component-templates-list) echo -e "Loading Security Onion index templates...\n" for so_idx_tmpl in "${SO_TEMPLATES_DIR}"/*.json; do @@ -189,7 +257,7 @@ if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]] && index_templates_e # TODO: Better way to load only heavynode specific templates if ! check_heavynode_compatiable_index_template "$tmpl_name"; then if [[ "$VERBOSE" == "true" ]]; then - echo "Skipping over $so_idx_tmpl, template is not a heavynode specific index template." + locked_echo "Skipping over $so_idx_tmpl, template is not a heavynode specific index template." fi continue @@ -197,32 +265,34 @@ if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]] && index_templates_e fi if check_required_component_template_exists "$so_idx_tmpl"; then - if ! load_template "_index_template/$tmpl_name" "$so_idx_tmpl"; then - SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1)) - SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl") - fi + template_throttle + load_template "_index_template/$tmpl_name" "$so_idx_tmpl" "$so_idx_tmpl" & else - echo "Skipping over $so_idx_tmpl due to missing required component template(s)." - SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1)) - SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl") + locked_echo "Skipping over $so_idx_tmpl due to missing required component template(s)." + record_failure "$so_idx_tmpl" continue fi done - if [[ $SO_LOAD_FAILURES -eq 0 ]]; then + # Barrier: all SO index template PUTs must finish before tallying failures. + wait + + collect_failures + if [[ $FAILED_COUNT -eq 0 ]]; then echo "All Security Onion core templates loaded successfully." touch "$SO_STATEFILE_SUCCESS" else - echo "Encountered $SO_LOAD_FAILURES failure(s) loading templates:" - for failed_template in "${SO_LOAD_FAILURES_NAMES[@]}"; do + echo "Encountered $FAILED_COUNT failure(s) loading templates:" + for failed_template in "${FAILED_NAMES[@]}"; do echo " - $failed_template" done if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then fail "Failed to load all Security Onion core templates successfully." fi fi + reset_failures elif ! index_templates_exist "$SO_TEMPLATES_DIR"; then echo "No Security Onion core index templates found in ${SO_TEMPLATES_DIR}, skipping." elif [[ -f "$SO_STATEFILE_SUCCESS" ]]; then @@ -241,26 +311,27 @@ if should_load_addon_templates; then tmpl_name=$(basename "${addon_idx_tmpl%-template.json}") if check_required_component_template_exists "$addon_idx_tmpl"; then - if ! load_template "_index_template/${tmpl_name}" "$addon_idx_tmpl"; then - ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1)) - ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl") - fi + template_throttle + load_template "_index_template/${tmpl_name}" "$addon_idx_tmpl" "$addon_idx_tmpl" & else - echo "Skipping over $addon_idx_tmpl due to missing required component template(s)." - ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1)) - ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl") + locked_echo "Skipping over $addon_idx_tmpl due to missing required component template(s)." + record_failure "$addon_idx_tmpl" continue fi done - if [[ $ADDON_LOAD_FAILURES -eq 0 ]]; then + # Barrier: all addon index template PUTs must finish before tallying failures. + wait + + collect_failures + if [[ $FAILED_COUNT -eq 0 ]]; then echo "All addon integration templates loaded successfully." touch "$ADDON_STATEFILE_SUCCESS" else - echo "Encountered $ADDON_LOAD_FAILURES failure(s) loading addon integration templates:" - for failed_template in "${ADDON_LOAD_FAILURES_NAMES[@]}"; do + echo "Encountered $FAILED_COUNT failure(s) loading addon integration templates:" + for failed_template in "${FAILED_NAMES[@]}"; do echo " - $failed_template" done if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then From 1ee555957a971a778cd8db0c6cbbe9fbe8ab712a Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 12 Jun 2026 15:23:43 -0400 Subject: [PATCH 070/219] Speed up so-elastic-fleet-integration-upgrade Fetch each agent policy once and extract integration name/package/version/id locally via a single jq pass instead of re-fetching the identical policy JSON 1+3N times. Memoize epm/packages latest-version lookups so each package is queried once instead of per (policy, integration). Dispatch the per-integration dry-run+upgrade as throttled background jobs (MAX_FLEET_JOBS) with flock-serialized output and a FAIL_FILE marker, mirroring elastic_fleet_load_integrations_dir. Behavior preserved: same elastic-defend-endpoints/fleet_server skips, same AUTO_UPGRADE_INTEGRATIONS default-package gating (moved into jq, using $defaults to avoid the jq $def keyword collision), and exit 1 on any failure so salt retries. --- .../so-elastic-fleet-integration-upgrade | 133 ++++++++++-------- 1 file changed, 75 insertions(+), 58 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade index 1a1448c53..c0008f362 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade @@ -23,73 +23,90 @@ if [ $? -ne 0 ]; then fi default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %}) +# JSON array of the default packages, used by the jq filter below. +default_packages_json=$(printf '%s\n' "${default_packages[@]}" | jq -R . | jq -s '.') + +# Output lock (serializes concurrent job output) and failure file (one marker line per +# failed integration). Mirrors the pattern used by elastic_fleet_load_integrations_dir. +OUTPUT_LOCK=$(mktemp) +FAIL_FILE=$(mktemp) +trap 'rm -f "$OUTPUT_LOCK" "$FAIL_FILE"' EXIT + +# Cache of package name -> latest available version, so the same package is only looked up +# once instead of once per (policy, integration). +declare -A LATEST_VERSION_CACHE -ERROR=false for AGENT_POLICY in $agent_policies; do - if ! integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY"); then + # Fetch the agent policy a single time; package name/version and integration id are all + # extracted locally below instead of re-fetching the same policy per integration. + if ! POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY"); then # this script upgrades default integration packages, exit 1 and let salt handle retrying exit 1 fi - for INTEGRATION in $integrations; do - if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then - # Get package name so we know what package to look for when checking the current and latest available version - if ! PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION"); then + + # One jq pass emits name/package.name/package.version/id for every eligible integration. + # The endpoint/fleet_server skips and the default-package gate are applied here in jq. + # $defaults (not $def, a jq reserved keyword) holds the default package list. + while IFS=$'\t' read -r INTEGRATION PACKAGE_NAME PACKAGE_VERSION INTEGRATION_ID; do + [ -n "$INTEGRATION" ] || continue + + # Look up the latest available version once per package, then memoize it. + if [[ -z "${LATEST_VERSION_CACHE[$PACKAGE_NAME]+set}" ]]; then + if ! AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME"); then + echo "Error: Failed getting latest version for $PACKAGE_NAME" exit 1 fi - {%- if not AUTO_UPGRADE_INTEGRATIONS %} - if [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then - {%- endif %} - # Get currently installed version of package - attempt=0 - max_attempts=3 - while [ $attempt -lt $max_attempts ]; do - if PACKAGE_VERSION=$(elastic_fleet_integration_policy_package_version "$AGENT_POLICY" "$INTEGRATION") && AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME"); then - break - fi - attempt=$((attempt + 1)) - done - if [ $attempt -eq $max_attempts ]; then - echo "Error: Failed getting $PACKAGE_VERSION or $AVAILABLE_VERSION" - exit 1 - fi - - # Get integration ID - if ! INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION"); then - exit 1 - fi - - if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then - # Dry run of the upgrade - echo "" - echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..." - echo "Upgrading $INTEGRATION..." - echo "Starting dry run..." - if ! DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID"); then - exit 1 - fi - DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors) - - # If no errors with dry run, proceed with actual upgrade - if [[ "$DRYRUN_ERRORS" == "false" ]]; then - echo "No errors detected. Proceeding with upgrade..." - if ! elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"; then - echo "Error: Upgrade failed for $PACKAGE_NAME with integration ID '$INTEGRATION_ID'." - ERROR=true - continue - fi - else - echo "Errors detected during dry run for $PACKAGE_NAME policy upgrade..." - ERROR=true - continue - fi - fi - {%- if not AUTO_UPGRADE_INTEGRATIONS %} - fi - {%- endif %} + LATEST_VERSION_CACHE[$PACKAGE_NAME]=$AVAILABLE_VERSION fi - done + AVAILABLE_VERSION=${LATEST_VERSION_CACHE[$PACKAGE_NAME]} + + if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then + # Dry run, then (if clean) the actual upgrade, dispatched as a throttled background + # job. Each job builds its full log into one block, then flushes it under a single + # shared lock (OUTPUT_LOCK) so concurrent jobs never interleave on stdout; a failed + # job also appends a marker line to FAIL_FILE while holding that same lock. + elastic_fleet_throttle + { + block=$'\n'"Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..."$'\n' + block+="Upgrading $INTEGRATION..."$'\n'"Starting dry run..."$'\n' + fail="" + if ! DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID"); then + block+="Error: Failed to complete dry run for '$INTEGRATION_ID'."$'\n' + fail="dryrun $INTEGRATION" + elif [[ "$(jq .[].hasErrors <<<"$DRYRUN_OUTPUT")" == "false" ]]; then + block+="No errors detected. Proceeding with upgrade..."$'\n' + if ! elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"; then + block+="Error: Upgrade failed for $PACKAGE_NAME with integration ID '$INTEGRATION_ID'."$'\n' + fail="upgrade $INTEGRATION" + fi + else + block+="Errors detected during dry run for $PACKAGE_NAME policy upgrade..."$'\n' + fail="dryrun-errors $INTEGRATION" + fi + { + flock 9 + printf '%s' "$block" + [ -n "$fail" ] && printf '%s\n' "$fail" >>"$FAIL_FILE" + } 9>>"$OUTPUT_LOCK" + } & + fi + done < <(jq -r --argjson defaults "$default_packages_json" ' + .item.package_policies[] + | select(.name != "elastic-defend-endpoints") + | select(.name | startswith("fleet_server-") | not) + {%- if not AUTO_UPGRADE_INTEGRATIONS %} + | select(.package.name | IN($defaults[])) + {%- endif %} + | [.name, .package.name, .package.version, .id] | @tsv + ' <<<"$POLICY_JSON") done -if [[ "$ERROR" == "true" ]]; then + +# Barrier: wait for every dispatched dry-run/upgrade job to finish. +wait + +if [ -s "$FAIL_FILE" ]; then + printf '\nFailed integration upgrades:\n' + cat "$FAIL_FILE" exit 1 fi echo From ae1ddf38173d2e46eea36233b7797bcadea3ddf0 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 15 Jun 2026 12:33:08 -0400 Subject: [PATCH 071/219] es|ql defaults --- salt/soc/defaults.yaml | 1 + salt/soc/soc_soc.yaml | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index c9399eab4..7e8e76094 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1464,6 +1464,7 @@ soc: sigmaRulePackages: - core - emerging_threats_addon + useEsql: false elastic: hostUrl: remoteHostUrls: [] diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index b2ac6d175..19853196a 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -383,6 +383,11 @@ soc: global: True advanced: False helpLink: sigma + useEsql: + description: "(Pre-release) Use Elasticsearch Piped Query Language (ES|QL) instead of EQL (Elastic Query Language) for Elasticsearch queries. The Sigma converter will output ES|QL instead of EQL, allowing support for correlations." + global: True + advanced: True + forcedType: bool elastic: index: description: Comma-separated list of indices or index patterns (wildcard "*" supported) that SOC will search for records. From d10f21399c72bff0b16d80dbae9a7f3373383d8d Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 15 Jun 2026 13:31:23 -0500 Subject: [PATCH 072/219] remove comments --- salt/elasticsearch/defaults.yaml | 20 +++++++------------ .../sbin_jinja/so-elasticsearch-dlm-apply | 2 -- salt/manager/tools/sbin/soup | 2 -- 3 files changed, 7 insertions(+), 17 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index ffb53ecbc..f0b01b3ca 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -19,18 +19,9 @@ elasticsearch: flood_stage: 90% high: 85% low: 80% - # don't want to set retention here since it will make ES restart with every update + - # potentially case where we could unintentially fall back to retention 7d and cause data loss - # data_streams: - # lifecycle: - # retention: - # default: 7d indices: id_field_data: enabled: false - # index: - # lifecycle: - # prefer_ilm: true logger: org: elasticsearch: @@ -73,7 +64,6 @@ elasticsearch: verification_mode: none index_settings: global_overrides: - # Tie this into cluster setting for data_streams.lifecycle.retention.default data_stream_lifecycle: data_retention: 90d index_template: @@ -2110,6 +2100,7 @@ elasticsearch: composed_of: - .logs-endpoint.actions@package - .logs-endpoint.actions@custom + - endpoint@custom - event-mappings - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 @@ -2119,8 +2110,9 @@ elasticsearch: hidden: false ignore_missing_component_templates: - .logs-endpoint.actions@custom + - endpoint@custom index_patterns: - - logs-endpoint.actions-* + - .logs-endpoint.actions-* priority: 501 template: settings: @@ -2171,6 +2163,7 @@ elasticsearch: composed_of: - .logs-endpoint.action.responses@package - .logs-endpoint.action.responses@custom + - endpoint@custom - event-mappings - so-fleet_integrations.ip_mappings-1 - so-fleet_globals-1 @@ -2180,14 +2173,15 @@ elasticsearch: hidden: false ignore_missing_component_templates: - .logs-endpoint.action.responses@custom + - endpoint@custom index_patterns: - - logs-endpoint.action.responses-* + - .logs-endpoint.action.responses-* priority: 501 template: settings: index: lifecycle: - name: so-logs-endpoint.actions-logs + name: so-logs-endpoint.action.responses-logs mapping: total_fields: limit: 5000 diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply index 843fad625..af761973c 100644 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply @@ -90,7 +90,6 @@ set_data_stream_lifecycle() { if ! output=$(so-elasticsearch-query "_data_stream/${data_stream}/_lifecycle" -XPUT -d "$body" --retry 3 --retry-delay 5 --fail); then echo "Failed to set data stream lifecycle for $data_stream." - echo "$output" return 1 fi @@ -113,7 +112,6 @@ disable_data_stream_lifecycle() { if ! output=$(so-elasticsearch-query "_data_stream/${data_stream}/_lifecycle" -XPUT -d "$body" --retry 3 --retry-delay 5 --fail); then echo "Failed to disable data stream lifecycle for $data_stream." - echo "$output" return 1 fi diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index d955580fd..b0f2610b7 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -803,8 +803,6 @@ kibana_backport_streams_index_template() { return 0 fi - ## NOTE: Should really add a check here for existing .kibana_streams index and then update its config in place - } up_to_3.2.0() { From 596471e1404b8287b453e5892ba0e82fc72fea6f Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 15 Jun 2026 13:31:53 -0500 Subject: [PATCH 073/219] using new annotation config --- salt/elasticsearch/soc_elasticsearch.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index c95a9467e..46e836c9c 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -159,6 +159,8 @@ elasticsearch: - If retention is less than or equal to 90 days, max_age will be 7 days - If retention is greater than 90 days, max_age will be 30 days forcedType: string + allowedNodeTypes: + - heavynode regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$ regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d. index_template: @@ -353,6 +355,8 @@ elasticsearch: - If retention is less than or equal to 90 days, max_age will be 7 days - If retention is greater than 90 days, max_age will be 30 days forcedType: string + allowedNodeTypes: + - heavynode regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$ regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d. index_template: From 95cae4c734ec04ed06ff7ac96c7d6572e062f639 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 15 Jun 2026 13:32:45 -0500 Subject: [PATCH 074/219] remove so-elasticsearch-indices-delete cron when using DLM --- salt/elasticsearch/cluster.sls | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/cluster.sls b/salt/elasticsearch/cluster.sls index efa50285e..a8ccd3780 100644 --- a/salt/elasticsearch/cluster.sls +++ b/salt/elasticsearch/cluster.sls @@ -165,7 +165,8 @@ so-elasticsearch-roles-load: {% set ap = "absent" %} {% endif %} {% if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %} -{% if ELASTICSEARCHMERGED.index_clean %} +{# Remove so-elasticsearch-indices-delete script when using DLM #} +{% if ELASTICSEARCHMERGED.index_clean and ELASTICSEARCHMERGED.data_retention_method == "ILM" %} {% set ap = "present" %} {% else %} {% set ap = "absent" %} From 1a423a243401944cc6493f84e9510243d6bc5be6 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 15 Jun 2026 14:17:34 -0500 Subject: [PATCH 075/219] update message --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index b0f2610b7..0016cc00e 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -787,7 +787,7 @@ kibana_backport_streams_index_template() { current_template=$(so-elasticsearch-query "_index_template/.kibana_streams" --retry 3 --retry-delay 5 --fail) if [[ -z "$current_template" ]]; then - echo "Unable to retrieve current .kibana_streams index template, skipping backport." + echo "Index template .kibana_streams does not exist, skipping backport." return 0 fi From f68e3e47a1f7d0d29096ec22fc39bec3f2d2b699 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 16 Jun 2026 09:19:10 -0500 Subject: [PATCH 076/219] remove pillar merge --- salt/elasticsearch/template.map.jinja | 2 +- .../tools/sbin_jinja/so-elasticsearch-dlm-apply | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 7acb3eb87..15481ee4c 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -5,7 +5,7 @@ {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %} -{% set DATA_RETENTION_METHOD = salt['pillar.get']('elasticsearch:data_retention_method', ELASTICSEARCHDEFAULTS.elasticsearch.get('data_retention_method', 'ILM')) %} +{% set DATA_RETENTION_METHOD = salt['pillar.get']('elasticsearch:data_retention_method', ELASTICSEARCHDEFAULTS.elasticsearch.data_retention_method) %} {% set PILLAR_GLOBAL_OVERRIDES = {} %} {% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings', {}) %} diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply index af761973c..e25e07223 100644 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply @@ -6,9 +6,8 @@ . /usr/sbin/so-common -{%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} - -{%- set DATA_RETENTION_METHOD = salt['pillar.get']('elasticsearch:data_retention_method', ELASTICSEARCHDEFAULTS.elasticsearch.get('data_retention_method', 'ILM')) %} +{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} +{%- set DATA_RETENTION_METHOD = ELASTICSEARCHMERGED.data_retention_method %} ELASTICSEARCH_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR:-/opt/so/conf/elasticsearch/templates}" TEMPLATE_DIRS=( From a769d4c68094c8b19fcb455f0ece69953ba9d7fc Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 16 Jun 2026 09:32:37 -0500 Subject: [PATCH 077/219] another unneeded default --- salt/elasticsearch/template.map.jinja | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 15481ee4c..72be1ec58 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -4,8 +4,11 @@ Elastic License 2.0. #} {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} +{# ELASTICSEARCHMERGED only used here to collect data_retention_method. This file intentionally works with ELASTICSEARCHDEFAULTS #} +{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} + {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %} -{% set DATA_RETENTION_METHOD = salt['pillar.get']('elasticsearch:data_retention_method', ELASTICSEARCHDEFAULTS.elasticsearch.data_retention_method) %} +{% set DATA_RETENTION_METHOD = ELASTICSEARCHMERGED.data_retention_method %} {% set PILLAR_GLOBAL_OVERRIDES = {} %} {% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings', {}) %} From 4a6c6752236a5b3cf98f7717f8a9e4b3d7a59169 Mon Sep 17 00:00:00 2001 From: Jorge Reyes <94730068+reyesj2@users.noreply.github.com> Date: Tue, 16 Jun 2026 10:33:11 -0500 Subject: [PATCH 078/219] skip kibana backport if the template doesn't exist --- salt/manager/tools/sbin/soup | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 0016cc00e..668d7d1de 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -784,9 +784,8 @@ pin_elasticsearch_data_retention_method() { # Reference: https://github.com/elastic/kibana/issues/263048 kibana_backport_streams_index_template() { local current_template updated_template - current_template=$(so-elasticsearch-query "_index_template/.kibana_streams" --retry 3 --retry-delay 5 --fail) - if [[ -z "$current_template" ]]; then + if ! current_template=$(so-elasticsearch-query "_index_template/.kibana_streams" --retry 3 --retry-delay 5 --fail); then echo "Index template .kibana_streams does not exist, skipping backport." return 0 fi From 4456bde1c820c2c5cbda1054e1398c010fed375b Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 16 Jun 2026 10:45:53 -0500 Subject: [PATCH 079/219] check if template exists without --fail flag --- salt/manager/tools/sbin/soup | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 668d7d1de..12be85eaf 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -783,10 +783,16 @@ pin_elasticsearch_data_retention_method() { # # Reference: https://github.com/elastic/kibana/issues/263048 kibana_backport_streams_index_template() { - local current_template updated_template + local current_template updated_template current_template_exists + + current_template_exists=$(so-elasticsearch-query "_index_template/.kibana_streams" -o /dev/null -w "%{http_code}") + if [[ "$current_template_exists" == "404" ]]; then + echo "Index template .kibana_streams does not exist, skipping backport." + return 0 + fi if ! current_template=$(so-elasticsearch-query "_index_template/.kibana_streams" --retry 3 --retry-delay 5 --fail); then - echo "Index template .kibana_streams does not exist, skipping backport." + echo "Unable to retrieve .kibana_streams index template, skipping backport." return 0 fi From 3daed551df0a3e435be6f98b341630f808e2d88d Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 16 Jun 2026 11:17:04 -0500 Subject: [PATCH 080/219] use --fail flag without set -x, since elasticsearch can return a 404 on the template lookup --- salt/manager/tools/sbin/soup | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 12be85eaf..96313aea4 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -783,18 +783,14 @@ pin_elasticsearch_data_retention_method() { # # Reference: https://github.com/elastic/kibana/issues/263048 kibana_backport_streams_index_template() { - local current_template updated_template current_template_exists + local current_template updated_template - current_template_exists=$(so-elasticsearch-query "_index_template/.kibana_streams" -o /dev/null -w "%{http_code}") - if [[ "$current_template_exists" == "404" ]]; then + set +e + if ! current_template=$(so-elasticsearch-query "_index_template/.kibana_streams" --retry 3 --retry-delay 5 --fail); then echo "Index template .kibana_streams does not exist, skipping backport." return 0 fi - - if ! current_template=$(so-elasticsearch-query "_index_template/.kibana_streams" --retry 3 --retry-delay 5 --fail); then - echo "Unable to retrieve .kibana_streams index template, skipping backport." - return 0 - fi + set -e updated_template=$(jq '.index_templates[0].index_template | .template.settings += {"index.auto_expand_replicas": "0-1"} | del(.created_date_millis, .modified_date_millis)' <<< "$current_template") From 6a18f35020173a2f68fb9a1c087382e219c27042 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 16 Jun 2026 17:47:46 -0500 Subject: [PATCH 081/219] add context to soup errors and optional soup debug log with xtrace output --- salt/manager/tools/sbin/soup | 58 +++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 96313aea4..a64524bff 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -16,6 +16,7 @@ POSTVERSION=$INSTALLEDVERSION INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}') BATCHSIZE=5 SOUP_LOG=/root/soup.log +SOUP_DEBUG_LOG=/root/soup-debug.log WHATWOULDYOUSAYYAHDOHERE=soup whiptail_title='Security Onion UPdater' NOTIFYCUSTOMELASTICCONFIG=false @@ -34,6 +35,7 @@ if [[ -f /etc/salt/cloud.profiles.d/socloud.conf ]]; then fi # used to display messages to the user at the end of soup declare -a FINAL_MESSAGE_QUEUE=() +SOUP_ERR_CONTEXT= check_err() { @@ -114,11 +116,49 @@ check_err() { echo "$err_msg" fi + if [[ -n $SOUP_ERR_CONTEXT ]]; then + echo "" + printf '%s\n' "$SOUP_ERR_CONTEXT" + fi + + echo "SOUP XTRACE debug log (if enabled) at $SOUP_DEBUG_LOG. Re-run soup with SOUP_DEBUG=1 to create $SOUP_DEBUG_LOG" + exit $exit_code fi } +# Collect bash error context before passing off to check_err() +on_err() { + local exit_code=$? + # Use first error context, multiple errors can happen with command substitutions or nested functions. We just need context from the initial error. + [[ -n $SOUP_ERR_CONTEXT ]] && return $exit_code + # turn off xtrace to prevent added noise in debug log + [[ $- == *x* ]] && set +x + + local cmd=$BASH_COMMAND + local line=${BASH_LINENO[0]} + local function=${FUNCNAME[1]:-main} + local source=${BASH_SOURCE[1]##*/} + local -a err_lines=( + "ERROR on: ${cmd}" + " source: ${source}:${line} in ${function}()" + ) + local i caller_line caller_src caller_func + + for ((i=2; i<${#FUNCNAME[@]}-1; i++)); do + caller_line=${BASH_LINENO[$((i-1))]} + [[ -n $caller_line && $caller_line -gt 0 ]] || continue + caller_src=${BASH_SOURCE[$i]##*/} + caller_func=${FUNCNAME[$i]:-main} + err_lines+=(" called by: ${caller_src}:${caller_line} in ${caller_func}()") + done + + SOUP_ERR_CONTEXT=$(printf '%s\n' "${err_lines[@]}") + + return $exit_code +} + airgap_mounted() { # Let's see if the ISO is already mounted. if [[ -f /tmp/soagupdate/SecurityOnion/VERSION ]]; then @@ -1982,4 +2022,20 @@ EOF read -r input fi -main "$@" | tee -a $SOUP_LOG +set -o errtrace +trap on_err ERR + +if [[ $SOUP_DEBUG == 1 ]]; then + if [ -f $SOUP_DEBUG_LOG ]; then + current_time=$(date +%Y%m%d.%H%M%S) + mv $SOUP_DEBUG_LOG $SOUP_DEBUG_LOG.$INSTALLEDVERSION.$current_time + fi + exec {SOUP_XTRACE_FD}>>"$SOUP_DEBUG_LOG" + export SOUP_XTRACE_FD + BASH_XTRACEFD=$SOUP_XTRACE_FD + PS4='[${BASH_SOURCE##*/}:${LINENO} ${FUNCNAME[0]:-main}] | ' + set -x + export SOUP_DEBUG +fi + +main "$@" 2>&1 | tee -a $SOUP_LOG From 16149df71fd25c797e318c057c02dac1ead0ee6c Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 16 Jun 2026 18:06:27 -0500 Subject: [PATCH 082/219] formatting --- salt/manager/tools/sbin/soup | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index a64524bff..fcde61d9e 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -131,10 +131,11 @@ check_err() { # Collect bash error context before passing off to check_err() on_err() { local exit_code=$? + # turn off xtrace to prevent added noise in debug log + set +x 2>/dev/null || true + # Use first error context, multiple errors can happen with command substitutions or nested functions. We just need context from the initial error. [[ -n $SOUP_ERR_CONTEXT ]] && return $exit_code - # turn off xtrace to prevent added noise in debug log - [[ $- == *x* ]] && set +x local cmd=$BASH_COMMAND local line=${BASH_LINENO[0]} @@ -2033,7 +2034,7 @@ if [[ $SOUP_DEBUG == 1 ]]; then exec {SOUP_XTRACE_FD}>>"$SOUP_DEBUG_LOG" export SOUP_XTRACE_FD BASH_XTRACEFD=$SOUP_XTRACE_FD - PS4='[${BASH_SOURCE##*/}:${LINENO} ${FUNCNAME[0]:-main}] | ' + PS4='+ [${BASH_SOURCE##*/}:${LINENO} ${FUNCNAME[0]:-main}()] | ' set -x export SOUP_DEBUG fi From d0bea2ebcbef3234636590e78bc7318f1621ef7e Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 18 Jun 2026 11:19:36 -0400 Subject: [PATCH 083/219] Restore grouped per-integration logging and retry 409s in fleet integration loader elastic_fleet_load_integrations_dir now buffers each concurrent job's output (header + API response) to a per-job file and prints them in submission order after wait, restoring the readable serial-style output while keeping concurrent writes. Add --retry-all-errors to the integration create/update curl calls so transient 409 conflicts from concurrent writes to the same agent policy are retried (curl --retry alone does not retry 409). --- .../tools/sbin/so-elastic-fleet-common | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common index d4f40b3ed..e8ded916f 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common @@ -53,9 +53,13 @@ elastic_fleet_load_integrations_dir() { local DIR=$2 local LABEL=$3 local SKIP_CREATE_NAME=$4 - local POLICY_JSON FAIL_FILE INTEGRATION NAME ID + local POLICY_JSON FAIL_FILE OUT_DIR INTEGRATION NAME ID i FAIL_FILE=$(mktemp) + # Each job buffers its full output (header + API response) into its own file so the + # parent can print them grouped and in submission order after concurrent writes finish. + OUT_DIR=$(mktemp -d) + i=0 # Fetch the agent policy a single time; we look up integration ids locally below. POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY") @@ -67,23 +71,31 @@ elastic_fleet_load_integrations_dir() { elastic_fleet_throttle { + local RESP if [ -n "$ID" ]; then printf "\n\n%s - Updating integration %s\n" "$LABEL" "$NAME" - if ! elastic_fleet_integration_update "$ID" "@$INTEGRATION"; then + if ! RESP=$(elastic_fleet_integration_update "$ID" "@$INTEGRATION"); then flock 9; echo "update ${INTEGRATION##*/}" >&9 fi + printf '%s\n' "$RESP" elif [ -n "$SKIP_CREATE_NAME" ] && [ "$NAME" == "$SKIP_CREATE_NAME" ]; then printf "\n\n%s - Skipping creation of %s\n" "$LABEL" "$NAME" else printf "\n\n%s - Creating integration %s\n" "$LABEL" "$NAME" - if ! elastic_fleet_integration_create "@$INTEGRATION"; then + if ! RESP=$(elastic_fleet_integration_create "@$INTEGRATION"); then flock 9; echo "create ${INTEGRATION##*/}" >&9 fi + printf '%s\n' "$RESP" fi - } 9>>"$FAIL_FILE" & + } >"$OUT_DIR/$(printf '%03d' "$i")" 9>>"$FAIL_FILE" & + i=$((i+1)) done wait + # Emit per-integration output grouped and in submission order (glob sorts numerically). + cat "$OUT_DIR"/* 2>/dev/null + rm -rf "$OUT_DIR" + local rc=0 if [ -s "$FAIL_FILE" ]; then printf "\n%s: failed integrations:\n" "$LABEL" @@ -110,7 +122,9 @@ elastic_fleet_integration_create() { JSON_STRING=$1 - if ! fleet_api "package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPOST -d "$JSON_STRING"; then + # --retry-all-errors so transient 409 conflicts (concurrent writes to the same agent + # policy) are retried; curl --retry alone does not retry 409. + if ! fleet_api "package_policies" --retry-all-errors -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPOST -d "$JSON_STRING"; then return 1 fi } @@ -141,7 +155,9 @@ elastic_fleet_integration_update() { JSON_STRING=$2 - if ! fleet_api "package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPUT -d "$JSON_STRING"; then + # --retry-all-errors so transient 409 conflicts (concurrent writes to the same agent + # policy) are retried; curl --retry alone does not retry 409. + if ! fleet_api "package_policies/$UPDATE_ID" --retry-all-errors -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPUT -d "$JSON_STRING"; then return 1 fi } From 22d5c96bd50d7d4ad81e8297d820945e9e312ff0 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 18 Jun 2026 14:56:29 -0500 Subject: [PATCH 084/219] don't create stack trace when set -e is disabled --- salt/manager/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index fcde61d9e..73bb2eec2 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -131,6 +131,8 @@ check_err() { # Collect bash error context before passing off to check_err() on_err() { local exit_code=$? + # Ignore failures in blocks that explicitly disabled errexit with `set +e`. + [[ $- == *e* ]] || return $exit_code # turn off xtrace to prevent added noise in debug log set +x 2>/dev/null || true From b77103aa9fd99055d96ffba809173c2673efc75c Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 22 Jun 2026 13:01:02 -0400 Subject: [PATCH 085/219] upgrade registry --- salt/registry/enabled.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/registry/enabled.sls b/salt/registry/enabled.sls index fc5021910..a7a0f4478 100644 --- a/salt/registry/enabled.sls +++ b/salt/registry/enabled.sls @@ -16,7 +16,7 @@ include: # Install the registry container so-dockerregistry: docker_container.running: - - image: ghcr.io/security-onion-solutions/registry:3.0.0 + - image: ghcr.io/security-onion-solutions/registry:3.1.1 - hostname: so-registry - networks: - sobridge: From bcc60a4ae00507d4b4efeb9fe4b336f0d5cc97eb Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 22 Jun 2026 13:07:49 -0400 Subject: [PATCH 086/219] kilo version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 944880fa1..03e153fda 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.2.0 +3.0.0-kilo From 96fcc0ec3848060ce84ec9b2dde4ac0234704bee Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 22 Jun 2026 14:25:46 -0500 Subject: [PATCH 087/219] wip --- .../tools/sbin/so-elastic-fleet-common | 20 ++++++++--- .../so-elasticsearch-ilm-policy-load | 35 +++++++++++++++---- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common index e8ded916f..855a28510 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common @@ -36,7 +36,7 @@ MAX_FLEET_JOBS=${MAX_FLEET_JOBS:-10} # Block until fewer than MAX_FLEET_JOBS background jobs are running. elastic_fleet_throttle() { while (( $(jobs -rp | wc -l) >= MAX_FLEET_JOBS )); do - wait -n + wait -n || true done } @@ -47,7 +47,7 @@ elastic_fleet_throttle() { # $2 DIR - directory of integration *.json files # $3 LABEL - human-readable label for log output # $4 SKIP_CREATE_NAME - (optional) integration name to skip when creating (still updated if present) -# Returns 1 if any integration failed to create/update. +# Returns 1 if the policy cannot be fetched or if any integration failed to create/update. elastic_fleet_load_integrations_dir() { local AGENT_POLICY=$1 local DIR=$2 @@ -62,7 +62,19 @@ elastic_fleet_load_integrations_dir() { i=0 # Fetch the agent policy a single time; we look up integration ids locally below. - POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY") + if ! POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY"); then + echo "Error: Failed to retrieve agent policy '$AGENT_POLICY'." + rm -f "$FAIL_FILE" + rm -rf "$OUT_DIR" + return 1 + fi + + if ! jq -e '.item.package_policies' <<<"$POLICY_JSON" >/dev/null 2>&1; then + echo "Error: Invalid agent policy response for '$AGENT_POLICY'." + rm -f "$FAIL_FILE" + rm -rf "$OUT_DIR" + return 1 + fi for INTEGRATION in "$DIR"/*.json; do [ -e "$INTEGRATION" ] || continue @@ -90,7 +102,7 @@ elastic_fleet_load_integrations_dir() { } >"$OUT_DIR/$(printf '%03d' "$i")" 9>>"$FAIL_FILE" & i=$((i+1)) done - wait + wait || true # Emit per-integration output grouped and in submission order (glob sorts numerically). cat "$OUT_DIR"/* 2>/dev/null diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load index a884f2e2f..9b748ce59 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load @@ -6,11 +6,12 @@ . /usr/sbin/so-common -MAX_JOBS=10 +MAX_JOBS=${MAX_ILM_JOBS:-10} # Lock used to serialize block writes so concurrent jobs never interleave their output. ILM_OUTPUT_LOCK=$(mktemp) -trap 'rm -f "$ILM_OUTPUT_LOCK"' EXIT +ILM_FAIL_FILE=$(mktemp) +trap 'rm -f "$ILM_OUTPUT_LOCK" "$ILM_FAIL_FILE"' EXIT # Policies are loaded concurrently (up to MAX_JOBS at a time) for speed. Each policy's block is # printed the moment its curl returns, so output appears in COMPLETION ORDER, not the order @@ -19,21 +20,31 @@ echo "Loading ILM policies concurrently; output below appears in completion orde echo put_policy() { - local desc="$1" policyname="$2" data="$3" result - result=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L \ + local desc="$1" policyname="$2" data="$3" result rc=0 + if ! result=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L --fail \ -X PUT "https://localhost:9200/_ilm/policy/${policyname}" \ - -H 'Content-Type: application/json' -d"${data}") + -H 'Content-Type: application/json' -d"${data}" 2>&1); then + rc=1 + elif ! jq -e '.acknowledged == true' <<<"$result" >/dev/null 2>&1; then + rc=1 + fi + # curl above ran in parallel; serialize just this block write so concurrent jobs never interleave. { flock 200 printf 'Setting up %s policy...\n%s\n\n' "${desc}" "${result}" + if (( rc != 0 )); then + printf '%s\n' "${policyname}" >>"$ILM_FAIL_FILE" + fi } 200>>"${ILM_OUTPUT_LOCK}" + + return "$rc" } # Block until fewer than MAX_JOBS background curls are running. throttle() { while (( $(jobs -rp | wc -l) >= MAX_JOBS )); do - wait -n + wait -n || true done } @@ -67,4 +78,14 @@ throttle() { {%- endfor %} {%- endif %} -wait +wait || true + +if [[ -s "$ILM_FAIL_FILE" ]]; then + echo "ERROR: Failed to load ILM policy(s):" + while read -r POLICY; do + echo " - $POLICY" + done < "$ILM_FAIL_FILE" + exit 1 +else + echo "Successfully loaded all ILM policies." +fi From e2e3e690ca933e2bfa80daae9f4648b64d1682b9 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 22 Jun 2026 16:52:29 -0400 Subject: [PATCH 088/219] reset version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 03e153fda..944880fa1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.0.0-kilo +3.2.0 From 698a746d6da370f41ca7f18637b53d1c368a6bf1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 23 Jun 2026 13:19:56 -0400 Subject: [PATCH 089/219] Add UEK8 kernel repo support across install and grid Mirror the kernel repo to full parity with the main package repo so the grid can pull the Oracle UEK8 kernel: - setup/so-functions: securityonion_repo() emits a [securityonionkernel] section in every branch (mirrorlist on non-airgap, https://$MSRV/kernelrepo for airgap/minion, file:///nsm/kernelrepo/ for manager); repo_sync_local() and create_repo() sync and build /nsm/kernelrepo. - manager/init.sls: create /nsm/kernelrepo and deploy mirror-kernel.txt. - nginx/enabled.sls: serve /nsm/kernelrepo at https:///kernelrepo. - repo/client/oracle.sls: add so_kernel_repo, gated by onlyif test -e /opt/so/state/nic_names_pinned so the kernel repo is only assigned once NICs are pinned by MAC. - update_packages(): run so-nic-pin before the dnf update that pulls the kernel, freezing interface names and dropping the pin marker so the kernel isn't downgraded then re-upgraded on the first highstate. --- salt/manager/files/mirror-kernel.txt | 2 ++ salt/manager/files/repodownload.conf | 7 ++++- salt/manager/init.sls | 17 ++++++++++++ salt/manager/tools/sbin/so-repo-sync | 4 +++ salt/nginx/enabled.sls | 1 + salt/repo/client/oracle.sls | 16 +++++++++++ setup/so-functions | 41 ++++++++++++++++++++++++++++ 7 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 salt/manager/files/mirror-kernel.txt diff --git a/salt/manager/files/mirror-kernel.txt b/salt/manager/files/mirror-kernel.txt new file mode 100644 index 000000000..1d9ce75d2 --- /dev/null +++ b/salt/manager/files/mirror-kernel.txt @@ -0,0 +1,2 @@ +https://repo.securityonion.net/file/so-repo/prod/3/oracle/9-uek8 +https://repo-alt.securityonion.net/prod/3/oracle/9-uek8 diff --git a/salt/manager/files/repodownload.conf b/salt/manager/files/repodownload.conf index 3c156a9db..67ae4b121 100644 --- a/salt/manager/files/repodownload.conf +++ b/salt/manager/files/repodownload.conf @@ -10,4 +10,9 @@ keepcache=0 name=Security Onion Repo repo mirrorlist=file:///opt/so/conf/reposync/mirror.txt enabled=1 -gpgcheck=1 \ No newline at end of file +gpgcheck=1 +[securityonionkernel] +name=Security Onion Repo repo +mirrorlist=file:///opt/so/conf/reposync/mirror-kernel.txt +enabled=1 +gpgcheck=1 diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 2353bb64b..d6c154efa 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -86,6 +86,16 @@ repo_dir: - group - show_changes: False +kernelrepo_dir: + file.directory: + - name: /nsm/kernelrepo + - user: socore + - group: socore + - recurse: + - user + - group + - show_changes: False + manager_sbin: file.recurse: - name: /usr/sbin @@ -122,6 +132,13 @@ so-repo-mirrorlist: - user: socore - group: socore +so-repo-kernel-mirrorlist: + file.managed: + - name: /opt/so/conf/reposync/mirror-kernel.txt + - source: salt://manager/files/mirror-kernel.txt + - user: socore + - group: socore + so-repo-sync: {% if MANAGERMERGED.reposync.enabled %} cron.present: diff --git a/salt/manager/tools/sbin/so-repo-sync b/salt/manager/tools/sbin/so-repo-sync index a0393a36b..bc90122d3 100755 --- a/salt/manager/tools/sbin/so-repo-sync +++ b/salt/manager/tools/sbin/so-repo-sync @@ -10,5 +10,9 @@ NOROOT=1 set -e curl --retry 5 --retry-delay 60 -A "reposync/$(sync_options)" https://sigs.securityonion.net/checkup --output /tmp/checkup + dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/ createrepo /nsm/repo + +dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/ +createrepo /nsm/kernelrepo diff --git a/salt/nginx/enabled.sls b/salt/nginx/enabled.sls index 2e4c9631c..40fde5b0e 100644 --- a/salt/nginx/enabled.sls +++ b/salt/nginx/enabled.sls @@ -59,6 +59,7 @@ so-nginx: - /opt/so/conf/navigator/layers/:/opt/socore/html/navigator/assets/so:ro - /opt/so/conf/navigator/config.json:/opt/socore/html/navigator/assets/config.json:ro - /nsm/repo:/opt/socore/html/repo:ro + - /nsm/kernelrepo:/opt/socore/html/kernelrepo:ro - /nsm/rules:/nsm/rules:ro {% if NGINXMERGED.external_suricata %} - /opt/so/rules/nids/suri:/surirules:ro diff --git a/salt/repo/client/oracle.sls b/salt/repo/client/oracle.sls index 70f529830..8c8a1ac0a 100644 --- a/salt/repo/client/oracle.sls +++ b/salt/repo/client/oracle.sls @@ -57,6 +57,22 @@ so_repo: - enabled: 1 - gpgcheck: 1 +so_kernel_repo: + pkgrepo.managed: + - name: securityonionkernel + - humanname: Security Onion Kernel Repo + {% if GLOBALS.is_manager %} + - baseurl: file:///nsm/kernelrepo/ + {% else %} + - baseurl: https://{{ GLOBALS.repo_host }}/kernelrepo + {% endif %} + - enabled: 1 + - gpgcheck: 1 + # Only assign the kernel repo once physical NIC names are pinned by MAC, so the + # UEK8 kernel update can't renumber interfaces SO binds by name (see pin_nic_names + # in salt/common/init.sls, which drops this marker via /usr/sbin/so-nic-pin). + - onlyif: 'test -e /opt/so/state/nic_names_pinned' + {% endif %} # TODO: Add a pillar entry for custom repos diff --git a/setup/so-functions b/setup/so-functions index 2d5181dc1..e8f6eb742 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -886,6 +886,7 @@ create_repo() { title "Create the repo directory" logCmd "dnf -y install yum-utils createrepo_c" logCmd "createrepo /nsm/repo" + logCmd "createrepo /nsm/kernelrepo" } @@ -1812,6 +1813,13 @@ securityonion_repo() { echo "mirrorlist=file:///etc/yum/mirror.txt" >> /etc/yum.repos.d/securityonion.repo echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo + echo "https://repo.securityonion.net/file/so-repo/prod/3/oracle/9-uek8" > /etc/yum/mirror-kernel.txt + echo "https://so-repo-east.s3.us-east-005.backblazeb2.com/prod/3/oracle/9-uek8" >> /etc/yum/mirror-kernel.txt + echo "[securityonionkernel]" >> /etc/yum.repos.d/securityonion.repo + echo "name=Security Onion Kernel Repo repo" >> /etc/yum.repos.d/securityonion.repo + echo "mirrorlist=file:///etc/yum/mirror-kernel.txt" >> /etc/yum.repos.d/securityonion.repo + echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo + echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo logCmd "dnf repolist" else echo "[securityonion]" > /etc/yum.repos.d/securityonion.repo @@ -1820,6 +1828,12 @@ securityonion_repo() { echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo echo "sslverify=0" >> /etc/yum.repos.d/securityonion.repo + echo "[securityonionkernel]" >> /etc/yum.repos.d/securityonion.repo + echo "name=Security Onion Kernel Repo" >> /etc/yum.repos.d/securityonion.repo + echo "baseurl=https://$MSRV/kernelrepo" >> /etc/yum.repos.d/securityonion.repo + echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo + echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo + echo "sslverify=0" >> /etc/yum.repos.d/securityonion.repo logCmd "dnf repolist" fi elif [[ ! $waitforstate ]]; then @@ -1829,12 +1843,23 @@ securityonion_repo() { echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo echo "sslverify=0" >> /etc/yum.repos.d/securityonion.repo + echo "[securityonionkernel]" >> /etc/yum.repos.d/securityonion.repo + echo "name=Security Onion Kernel Repo" >> /etc/yum.repos.d/securityonion.repo + echo "baseurl=https://$MSRV/kernelrepo" >> /etc/yum.repos.d/securityonion.repo + echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo + echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo + echo "sslverify=0" >> /etc/yum.repos.d/securityonion.repo elif [[ $waitforstate ]]; then echo "[securityonion]" > /etc/yum.repos.d/securityonion.repo echo "name=Security Onion Repo" >> /etc/yum.repos.d/securityonion.repo echo "baseurl=file:///nsm/repo/" >> /etc/yum.repos.d/securityonion.repo echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo + echo "[securityonionkernel]" >> /etc/yum.repos.d/securityonion.repo + echo "name=Security Onion Kernel Repo" >> /etc/yum.repos.d/securityonion.repo + echo "baseurl=file:///nsm/kernelrepo/" >> /etc/yum.repos.d/securityonion.repo + echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo + echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo fi logCmd "dnf repolist all" if [[ $waitforstate ]]; then @@ -1850,9 +1875,12 @@ repo_sync_local() { # Sync the repo from the SO repo locally. info "Adding Repo Download Configuration" mkdir -p /nsm/repo + mkdir -p /nsm/kernelrepo mkdir -p /opt/so/conf/reposync/cache echo "https://repo.securityonion.net/file/so-repo/prod/3/oracle/9" > /opt/so/conf/reposync/mirror.txt echo "https://repo-alt.securityonion.net/prod/3/oracle/9" >> /opt/so/conf/reposync/mirror.txt + echo "https://repo.securityonion.net/file/so-repo/prod/3/oracle/9-uek8" > /opt/so/conf/reposync/mirror-kernel.txt + echo "https://repo-alt.securityonion.net/prod/3/oracle/9-uek8" >> /opt/so/conf/reposync/mirror-kernel.txt echo "[main]" > /opt/so/conf/reposync/repodownload.conf echo "gpgcheck=1" >> /opt/so/conf/reposync/repodownload.conf echo "installonly_limit=3" >> /opt/so/conf/reposync/repodownload.conf @@ -1866,12 +1894,18 @@ repo_sync_local() { echo "mirrorlist=file:///opt/so/conf/reposync/mirror.txt" >> /opt/so/conf/reposync/repodownload.conf echo "enabled=1" >> /opt/so/conf/reposync/repodownload.conf echo "gpgcheck=1" >> /opt/so/conf/reposync/repodownload.conf + echo "[securityonionkernel]" >> /opt/so/conf/reposync/repodownload.conf + echo "name=Security Onion Kernel Repo repo" >> /opt/so/conf/reposync/repodownload.conf + echo "mirrorlist=file:///opt/so/conf/reposync/mirror-kernel.txt" >> /opt/so/conf/reposync/repodownload.conf + echo "enabled=1" >> /opt/so/conf/reposync/repodownload.conf + echo "gpgcheck=1" >> /opt/so/conf/reposync/repodownload.conf logCmd "dnf repolist" if [[ ! $is_airgap ]]; then curl --retry 5 --retry-delay 60 -A "netinstall/$SOVERSION/$OS/$(uname -r)/1" https://sigs.securityonion.net/checkup --output /tmp/install retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" >> "$setup_log" 2>&1 || fail_setup + retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/" >> "$setup_log" 2>&1 || fail_setup # After the download is complete run createrepo create_repo fi @@ -2228,6 +2262,13 @@ update_sudoers_for_testing() { } update_packages() { + # Pin physical NIC names by MAC BEFORE pulling packages, so the UEK8 kernel that + # the update below installs can't renumber the interfaces SO binds by name. Doing + # it here (instead of waiting for the common highstate) also drops the + # /opt/so/state/nic_names_pinned marker that gates the kernel repo, so the kernel + # repo is assigned on the very first highstate and the kernel isn't downgraded and + # then re-upgraded. Run-once: so-nic-pin no-ops if the marker already exists. + logCmd "bash ../salt/common/tools/sbin/so-nic-pin" logCmd "dnf repolist" logCmd "dnf -y update --allowerasing --exclude=salt*,docker*,containerd*" RMREPOFILES=("oracle-linux-ol9.repo" "uek-ol9.repo" "virt-ol9.repo") From 8e2753aeb833de6581c4b1947e071ad9f4cf4e10 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 23 Jun 2026 13:53:14 -0400 Subject: [PATCH 090/219] Fix duplicate securityonionkernel repo definition The install bootstrap appended the [securityonionkernel] section to the shared /etc/yum.repos.d/securityonion.repo, but the salt state so_kernel_repo (name securityonionkernel) manages its own canonical file /etc/yum.repos.d/securityonionkernel.repo. At highstate both files defined the same repo id, so dnf failed with "repository securityonionkernel is listed more than 1 time". Write the bootstrap kernel repo to /etc/yum.repos.d/securityonionkernel.repo in all four securityonion_repo() branches so the id lives in exactly one file and salt edits it in place. Mirrors how the main repo's runtime id matches its file name. --- setup/so-functions | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index e8f6eb742..15856d710 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1815,11 +1815,11 @@ securityonion_repo() { echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo echo "https://repo.securityonion.net/file/so-repo/prod/3/oracle/9-uek8" > /etc/yum/mirror-kernel.txt echo "https://so-repo-east.s3.us-east-005.backblazeb2.com/prod/3/oracle/9-uek8" >> /etc/yum/mirror-kernel.txt - echo "[securityonionkernel]" >> /etc/yum.repos.d/securityonion.repo - echo "name=Security Onion Kernel Repo repo" >> /etc/yum.repos.d/securityonion.repo - echo "mirrorlist=file:///etc/yum/mirror-kernel.txt" >> /etc/yum.repos.d/securityonion.repo - echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo - echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo + echo "[securityonionkernel]" > /etc/yum.repos.d/securityonionkernel.repo + echo "name=Security Onion Kernel Repo repo" >> /etc/yum.repos.d/securityonionkernel.repo + echo "mirrorlist=file:///etc/yum/mirror-kernel.txt" >> /etc/yum.repos.d/securityonionkernel.repo + echo "enabled=1" >> /etc/yum.repos.d/securityonionkernel.repo + echo "gpgcheck=1" >> /etc/yum.repos.d/securityonionkernel.repo logCmd "dnf repolist" else echo "[securityonion]" > /etc/yum.repos.d/securityonion.repo @@ -1828,12 +1828,12 @@ securityonion_repo() { echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo echo "sslverify=0" >> /etc/yum.repos.d/securityonion.repo - echo "[securityonionkernel]" >> /etc/yum.repos.d/securityonion.repo - echo "name=Security Onion Kernel Repo" >> /etc/yum.repos.d/securityonion.repo - echo "baseurl=https://$MSRV/kernelrepo" >> /etc/yum.repos.d/securityonion.repo - echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo - echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo - echo "sslverify=0" >> /etc/yum.repos.d/securityonion.repo + echo "[securityonionkernel]" > /etc/yum.repos.d/securityonionkernel.repo + echo "name=Security Onion Kernel Repo" >> /etc/yum.repos.d/securityonionkernel.repo + echo "baseurl=https://$MSRV/kernelrepo" >> /etc/yum.repos.d/securityonionkernel.repo + echo "enabled=1" >> /etc/yum.repos.d/securityonionkernel.repo + echo "gpgcheck=1" >> /etc/yum.repos.d/securityonionkernel.repo + echo "sslverify=0" >> /etc/yum.repos.d/securityonionkernel.repo logCmd "dnf repolist" fi elif [[ ! $waitforstate ]]; then @@ -1843,23 +1843,23 @@ securityonion_repo() { echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo echo "sslverify=0" >> /etc/yum.repos.d/securityonion.repo - echo "[securityonionkernel]" >> /etc/yum.repos.d/securityonion.repo - echo "name=Security Onion Kernel Repo" >> /etc/yum.repos.d/securityonion.repo - echo "baseurl=https://$MSRV/kernelrepo" >> /etc/yum.repos.d/securityonion.repo - echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo - echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo - echo "sslverify=0" >> /etc/yum.repos.d/securityonion.repo + echo "[securityonionkernel]" > /etc/yum.repos.d/securityonionkernel.repo + echo "name=Security Onion Kernel Repo" >> /etc/yum.repos.d/securityonionkernel.repo + echo "baseurl=https://$MSRV/kernelrepo" >> /etc/yum.repos.d/securityonionkernel.repo + echo "enabled=1" >> /etc/yum.repos.d/securityonionkernel.repo + echo "gpgcheck=1" >> /etc/yum.repos.d/securityonionkernel.repo + echo "sslverify=0" >> /etc/yum.repos.d/securityonionkernel.repo elif [[ $waitforstate ]]; then echo "[securityonion]" > /etc/yum.repos.d/securityonion.repo echo "name=Security Onion Repo" >> /etc/yum.repos.d/securityonion.repo echo "baseurl=file:///nsm/repo/" >> /etc/yum.repos.d/securityonion.repo echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo - echo "[securityonionkernel]" >> /etc/yum.repos.d/securityonion.repo - echo "name=Security Onion Kernel Repo" >> /etc/yum.repos.d/securityonion.repo - echo "baseurl=file:///nsm/kernelrepo/" >> /etc/yum.repos.d/securityonion.repo - echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo - echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo + echo "[securityonionkernel]" > /etc/yum.repos.d/securityonionkernel.repo + echo "name=Security Onion Kernel Repo" >> /etc/yum.repos.d/securityonionkernel.repo + echo "baseurl=file:///nsm/kernelrepo/" >> /etc/yum.repos.d/securityonionkernel.repo + echo "enabled=1" >> /etc/yum.repos.d/securityonionkernel.repo + echo "gpgcheck=1" >> /etc/yum.repos.d/securityonionkernel.repo fi logCmd "dnf repolist all" if [[ $waitforstate ]]; then From 81ebea04511511cc4ca131e357ef2fa226e0b245 Mon Sep 17 00:00:00 2001 From: Dan Marr Date: Tue, 23 Jun 2026 16:07:30 -0400 Subject: [PATCH 091/219] Fix non-root exit checks at start of so-setup --- setup/so-setup | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 6c77e781c..dd024e275 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -9,14 +9,17 @@ # Make sure you are root before doing anything uid="$(id -u)" if [ "$uid" -ne 0 ]; then - echo "This script must be run using sudo!" - fail_setup + echo "This script must be run using sudo!" >&2 + exit 1 fi # Save the original argument array since we modify it original_args=("$@") -cd "$(dirname "$0")" || fail_setup +cd "$(dirname "$0")" || { + echo "Unable to change to setup directory" >&2 + exit 1 +} echo "Getting started..." From 84228a819b9ca318afd843e594fb9ef24c8493f3 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 23 Jun 2026 15:30:49 -0500 Subject: [PATCH 092/219] remove heayvnode FleetServer_* directory creation, and skip empty directories during FleetServer policy management --- salt/elasticfleet/config.sls | 2 +- .../sbin/so-elastic-fleet-integration-policy-load | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/salt/elasticfleet/config.sls b/salt/elasticfleet/config.sls index 9c79dfab6..6a8919605 100644 --- a/salt/elasticfleet/config.sls +++ b/salt/elasticfleet/config.sls @@ -173,7 +173,7 @@ eaoptionalintegrationsdir: {% for minion in node_data %} {% set role = node_data[minion]["role"] %} -{% if role in [ "eval","fleet","heavynode","import","manager", "managerhype", "managersearch","standalone" ] %} +{% if role in [ "eval","fleet","import","manager", "managerhype", "managersearch","standalone" ] %} {% set optional_integrations = ELASTICFLEETMERGED.optional_integrations %} {% set integration_keys = optional_integrations.keys() %} fleet_server_integrations_{{ minion }}: diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load index 33c880dba..7c2aeb006 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load @@ -9,13 +9,11 @@ RETURN_CODE=0 if [ ! -f /opt/so/state/eaintegrations.txt ]; then - # First, check for any package upgrades - /usr/sbin/so-elastic-fleet-package-upgrade - # Second, update Fleet Server policies + # update Fleet Server policies /usr/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server - # Third, configure Elastic Defend Integration seperately + # configure Elastic Defend Integration separately /usr/sbin/so-elastic-fleet-integration-policy-elastic-defend # Each group fetches its agent policy once and dispatches create/update writes concurrently. @@ -32,9 +30,12 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then elastic_fleet_load_integrations_dir "so-grid-nodes_heavy" \ /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy "Grid Nodes Policy_Heavy" || RETURN_CODE=1 - # Fleet Server - Optional integrations (one agent policy per FleetServer_* directory) + # Fleet Server - Optional integrations (adds integration configuration to a given FleetServer_ policy) for FLEET_DIR in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/; do [ -d "$FLEET_DIR" ] || continue + INTEGRATIONS=("${FLEET_DIR%/}"/*.json) + [ -e "${INTEGRATIONS[0]}" ] || continue + FLEET_POLICY=$(basename "$FLEET_DIR") elastic_fleet_load_integrations_dir "$FLEET_POLICY" \ "${FLEET_DIR%/}" "Fleet Server Policy" "elasticsearch-logs" || RETURN_CODE=1 From 4f3b57f4954dab1419aadcbed3a26370db2a064f Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 23 Jun 2026 15:52:10 -0500 Subject: [PATCH 093/219] remove duplicate package-upgrade attempts, upgrade only when reported latest version differs from installed version --- salt/elasticfleet/manager.sls | 2 -- .../so-elastic-fleet-package-upgrade | 20 ++++++++++--------- .../tools/sbin_jinja/so-elastic-fleet-setup | 3 +++ 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/salt/elasticfleet/manager.sls b/salt/elasticfleet/manager.sls index c9fe91d4d..4ae64314b 100644 --- a/salt/elasticfleet/manager.sls +++ b/salt/elasticfleet/manager.sls @@ -67,8 +67,6 @@ so-elastic-fleet-package-upgrade: interval: 30 - require: - http: wait_for_so-kibana - - onchanges: - - file: /opt/so/state/elastic_fleet_packages.txt so-elastic-fleet-integrations: cmd.run: diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade index 8ba250c00..f3d77b852 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade @@ -12,17 +12,22 @@ PKG_LOAD_FAILURES=0 PKG_LOAD_FAILURES_NAMES=() {%- for PACKAGE in SUPPORTED_PACKAGES %} -echo "Upgrading {{ PACKAGE }} package..." -if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then - if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then - PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1)) - PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}") +if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && LATEST_VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then + + if [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then + echo "{{ PACKAGE }} integration version $INSTALLED_VERSION is already at the reported latest version $LATEST_VERSION, skipping upgrade." + else + echo "Upgrading {{ PACKAGE }} package to version $LATEST_VERSION..." + if ! elastic_fleet_package_install "{{ PACKAGE }}" "$LATEST_VERSION"; then + PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1)) + PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}") + fi fi else + echo "ERROR: Failed to get version information for integration {{ PACKAGE }}" PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1)) PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}") fi -echo {%- endfor %} if [ $PKG_LOAD_FAILURES -gt 0 ]; then @@ -35,6 +40,3 @@ if [ $PKG_LOAD_FAILURES -gt 0 ]; then else echo "Successfully upgraded all packages." fi - -echo -/usr/sbin/so-elasticsearch-templates-load diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index 5e0dc0c69..b62310375 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -181,6 +181,9 @@ if ! elastic_fleet_policy_create "so-grid-nodes_heavy" "SO Grid Nodes - Heavy No exit 1 fi +# Check for package upgrades +so-elastic-fleet-package-upgrade + # Load Integrations for default policies so-elastic-fleet-integration-policy-load From f45631af3ae03b4e684ac8327a8bdbeabe624356 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 24 Jun 2026 12:15:10 -0400 Subject: [PATCH 094/219] Guard kernel reposync on its config section existing During soup, so-repo-sync runs before the highstate deploys the new repodownload.conf. On the first upgrade to a kernel-aware version the on-disk config lacks the [securityonionkernel] section, so dnf aborts with "Unknown repo: 'securityonionkernel'" (set -e kills soup). Guard the kernel reposync on the section being present; the next sync after the highstate deploys it picks it up. --- salt/manager/tools/sbin/so-repo-sync | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/so-repo-sync b/salt/manager/tools/sbin/so-repo-sync index bc90122d3..6c1b9d509 100755 --- a/salt/manager/tools/sbin/so-repo-sync +++ b/salt/manager/tools/sbin/so-repo-sync @@ -14,5 +14,12 @@ curl --retry 5 --retry-delay 60 -A "reposync/$(sync_options)" https://sigs.secur dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/ createrepo /nsm/repo -dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/ -createrepo /nsm/kernelrepo +# The kernel repo section is deployed to repodownload.conf by the manager highstate, which +# runs AFTER this script during soup. On the first upgrade to a kernel-aware version the +# on-disk config still predates the section, so guard on its presence to avoid dnf's +# "Unknown repo: 'securityonionkernel'" aborting the sync (set -e). The next sync after the +# highstate deploys the section will pick it up. +if grep -q '^\[securityonionkernel\]' /opt/so/conf/reposync/repodownload.conf; then + dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/ + createrepo /nsm/kernelrepo +fi From 27c1c35e62e90f6d11ae92817abda2a443ab992f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 24 Jun 2026 13:20:10 -0400 Subject: [PATCH 095/219] Mark kernel repo skip_if_unavailable so an empty repo can't brick dnf When the kernel repo is assigned but /nsm/kernelrepo isn't populated yet, its missing repomd.xml makes every dnf/pkg operation fail (e.g. pkg.held for salt during highstate). The kernel repo is supplementary, so set skip_if_unavailable=1 in both the salt-managed client repo and the four install-time bootstrap repo files; dnf ignores it until it is populated instead of aborting. The main repo stays strict. --- salt/repo/client/oracle.sls | 4 ++++ setup/so-functions | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/salt/repo/client/oracle.sls b/salt/repo/client/oracle.sls index 8c8a1ac0a..2019a56d1 100644 --- a/salt/repo/client/oracle.sls +++ b/salt/repo/client/oracle.sls @@ -68,6 +68,10 @@ so_kernel_repo: {% endif %} - enabled: 1 - gpgcheck: 1 + # Supplementary kernel repo: tolerate it being empty/unreachable (e.g. before the + # manager has populated /nsm/kernelrepo) so a missing repomd.xml can't make every + # dnf/pkg operation on the grid fail. + - skip_if_unavailable: 1 # Only assign the kernel repo once physical NIC names are pinned by MAC, so the # UEK8 kernel update can't renumber interfaces SO binds by name (see pin_nic_names # in salt/common/init.sls, which drops this marker via /usr/sbin/so-nic-pin). diff --git a/setup/so-functions b/setup/so-functions index 15856d710..b9a061168 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1820,6 +1820,9 @@ securityonion_repo() { echo "mirrorlist=file:///etc/yum/mirror-kernel.txt" >> /etc/yum.repos.d/securityonionkernel.repo echo "enabled=1" >> /etc/yum.repos.d/securityonionkernel.repo echo "gpgcheck=1" >> /etc/yum.repos.d/securityonionkernel.repo + # Supplementary kernel repo: tolerate it being empty/unreachable so a missing + # repomd.xml can't make every dnf operation fail before the repo is populated. + echo "skip_if_unavailable=1" >> /etc/yum.repos.d/securityonionkernel.repo logCmd "dnf repolist" else echo "[securityonion]" > /etc/yum.repos.d/securityonion.repo @@ -1834,6 +1837,7 @@ securityonion_repo() { echo "enabled=1" >> /etc/yum.repos.d/securityonionkernel.repo echo "gpgcheck=1" >> /etc/yum.repos.d/securityonionkernel.repo echo "sslverify=0" >> /etc/yum.repos.d/securityonionkernel.repo + echo "skip_if_unavailable=1" >> /etc/yum.repos.d/securityonionkernel.repo logCmd "dnf repolist" fi elif [[ ! $waitforstate ]]; then @@ -1849,6 +1853,7 @@ securityonion_repo() { echo "enabled=1" >> /etc/yum.repos.d/securityonionkernel.repo echo "gpgcheck=1" >> /etc/yum.repos.d/securityonionkernel.repo echo "sslverify=0" >> /etc/yum.repos.d/securityonionkernel.repo + echo "skip_if_unavailable=1" >> /etc/yum.repos.d/securityonionkernel.repo elif [[ $waitforstate ]]; then echo "[securityonion]" > /etc/yum.repos.d/securityonion.repo echo "name=Security Onion Repo" >> /etc/yum.repos.d/securityonion.repo @@ -1860,6 +1865,7 @@ securityonion_repo() { echo "baseurl=file:///nsm/kernelrepo/" >> /etc/yum.repos.d/securityonionkernel.repo echo "enabled=1" >> /etc/yum.repos.d/securityonionkernel.repo echo "gpgcheck=1" >> /etc/yum.repos.d/securityonionkernel.repo + echo "skip_if_unavailable=1" >> /etc/yum.repos.d/securityonionkernel.repo fi logCmd "dnf repolist all" if [[ $waitforstate ]]; then From b0b022c3ada07e03fd4ecf2ee5ac9f06294e3174 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 24 Jun 2026 13:23:25 -0400 Subject: [PATCH 096/219] Seed an empty /nsm/kernelrepo so the manager repo is always valid so-repo-sync only populates /nsm/kernelrepo after the highstate, so on a manager the file:///nsm/kernelrepo repo could be assigned before any repodata exists, failing every dnf op. Run createrepo on the dir when repodata/repomd.xml is missing, leaving a synced repo untouched. --- salt/manager/init.sls | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index d6c154efa..e77b1a601 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -96,6 +96,18 @@ kernelrepo_dir: - group - show_changes: False +# Ensure /nsm/kernelrepo is always a valid (if empty) repo before it is ever assigned to +# a client. Without repodata/repomd.xml an enabled file:///nsm/kernelrepo repo makes every +# dnf operation fail; so-repo-sync only populates it after the highstate, so seed an empty +# repo here. Only runs when repodata is missing, so it won't clobber a synced repo. +kernelrepo_init_empty: + cmd.run: + - name: createrepo /nsm/kernelrepo + - unless: 'test -e /nsm/kernelrepo/repodata/repomd.xml' + - require: + - file: kernelrepo_dir + - pkg: install_createrepo + manager_sbin: file.recurse: - name: /usr/sbin From dfdb1fbaebe7689fdcaf8dab54e51080e55b69ea Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 24 Jun 2026 15:17:48 -0400 Subject: [PATCH 097/219] Move global.push config to salt.auto_apply The active-push tunables (enabled, highstate_interval_hours, debounce_seconds, drain_interval, batch, batch_wait) described how Salt auto-applies changes, not general grid config, so relocate them from the global namespace to a new salt.auto_apply settings module. - Add salt/salt/{defaults.yaml,auto_apply.map.jinja,soc_salt.yaml,adv_salt.yaml}. auto_apply.map.jinja is a dedicated, side-effect-free merge map (the existing salt/salt/map.jinja dereferences pillar.host.mainint at import time). - Remove the push blocks from salt/global/{defaults,soc_global}.yaml. - Register salt.soc_salt/salt.adv_salt in pillar/top.sls; seed the local pillar stubs for fresh installs (make_some_dirs) and upgrades (ensure_salt_local_pillar in soup, wired into up_to_3.2.0). - Repoint all consumers: GLOBALMERGED.push.* -> AUTOAPPLY.* (schedule, salt master, manager beacons, beacons_pushstate, orch.push_batch) and pillar.get('global:push...') -> 'salt:auto_apply...' (push reactors, so-push-drainer). - Add a salt: fleetwide-highstate entry to pillar_push_map.yaml so edits keep applying immediately, matching the prior global-namespace behavior. --- pillar/top.sls | 2 + salt/global/defaults.yaml | 7 ---- salt/global/soc_global.yaml | 37 ------------------ salt/manager/beacons.sls | 4 +- .../files/beacons_pushstate.conf.jinja | 4 +- salt/manager/tools/sbin/so-push-drainer | 6 +-- salt/manager/tools/sbin/soup | 17 +++++++++ salt/orch/push_batch.sls | 6 +-- salt/reactor/pillar_push_map.yaml | 11 ++++++ salt/reactor/push_pillar.sls | 4 +- salt/reactor/push_strelka.sls | 4 +- salt/reactor/push_suricata.sls | 4 +- salt/salt/adv_salt.yaml | 0 salt/salt/auto_apply.map.jinja | 2 + salt/salt/defaults.yaml | 8 ++++ salt/salt/master.sls | 4 +- salt/salt/soc_salt.yaml | 38 +++++++++++++++++++ salt/schedule.sls | 8 ++-- setup/so-functions | 2 +- 19 files changed, 101 insertions(+), 67 deletions(-) create mode 100644 salt/salt/adv_salt.yaml create mode 100644 salt/salt/auto_apply.map.jinja create mode 100644 salt/salt/defaults.yaml create mode 100644 salt/salt/soc_salt.yaml diff --git a/pillar/top.sls b/pillar/top.sls index 712629dbf..aba3ef4c5 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -3,6 +3,8 @@ base: - ca - global.soc_global - global.adv_global + - salt.soc_salt + - salt.adv_salt - docker.soc_docker - docker.adv_docker - influxdb.token diff --git a/salt/global/defaults.yaml b/salt/global/defaults.yaml index 23cfb1454..60f7885ac 100644 --- a/salt/global/defaults.yaml +++ b/salt/global/defaults.yaml @@ -1,10 +1,3 @@ global: pcapengine: SURICATA pipeline: REDIS - push: - enabled: true - highstate_interval_hours: 2 - debounce_seconds: 30 - drain_interval: 15 - batch: '25%' - batch_wait: 15 diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index c46102fe7..c15f3eb98 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -59,41 +59,4 @@ global: description: Allows use of Endgame with Security Onion. This feature requires a license from Endgame. global: True advanced: True - push: - enabled: - description: Master kill-switch for the active push feature. When disabled, rule and pillar changes are picked up at the next scheduled highstate instead of being pushed immediately. - forcedType: bool - helpLink: push - global: True - highstate_interval_hours: - description: How often every minion in the grid runs a scheduled state.highstate, in hours. Lower values keep minions closer in sync at the cost of more load; higher values reduce load but increase worst-case latency for non-pushed changes. The salt-minion health check restarts a minion if its last highstate is older than this value plus one hour. - forcedType: int - helpLink: push - global: True - advanced: True - debounce_seconds: - description: Trailing-edge debounce window in seconds. A push intent must be quiet for this long before the drainer dispatches. Rapid bursts of edits within this window coalesce into one dispatch. - forcedType: int - helpLink: push - global: True - advanced: True - drain_interval: - description: How often the push drainer checks for ready intents, in seconds. Small values lower dispatch latency at the cost of more background work on the manager. - forcedType: int - helpLink: push - global: True - advanced: True - batch: - description: "Host batch size for push orchestrations. A number (e.g. '10') or a percentage (e.g. '25%'). Limits how many minions run the push state at once so large fleets don't thundering-herd." - helpLink: push - global: True - advanced: True - regex: '^([0-9]+%?)$' - regexFailureMessage: Enter a whole number or a whole-number percentage (e.g. 10 or 25%). - batch_wait: - description: Seconds to wait between host batches in a push orchestration. Gives the fleet time to breathe between waves. - forcedType: int - helpLink: push - global: True - advanced: True diff --git a/salt/manager/beacons.sls b/salt/manager/beacons.sls index 11574f9e2..b04017140 100644 --- a/salt/manager/beacons.sls +++ b/salt/manager/beacons.sls @@ -1,10 +1,10 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} -{% from 'global/map.jinja' import GLOBALMERGED %} +{% from 'salt/auto_apply.map.jinja' import AUTOAPPLY %} include: - salt.minion -{% if GLOBALS.is_manager and GLOBALMERGED.push.enabled %} +{% if GLOBALS.is_manager and AUTOAPPLY.enabled %} salt_beacons_pushstate: file.managed: - name: /etc/salt/minion.d/beacons_pushstate.conf diff --git a/salt/manager/files/beacons_pushstate.conf.jinja b/salt/manager/files/beacons_pushstate.conf.jinja index ce9259359..9f45d5d85 100644 --- a/salt/manager/files/beacons_pushstate.conf.jinja +++ b/salt/manager/files/beacons_pushstate.conf.jinja @@ -1,7 +1,7 @@ -{% from 'global/map.jinja' import GLOBALMERGED %} +{% from 'salt/auto_apply.map.jinja' import AUTOAPPLY %} beacons: pillar_db: - - interval: {{ GLOBALMERGED.push.drain_interval }} + - interval: {{ AUTOAPPLY.drain_interval }} - disable_during_state_run: True inotify: - disable_during_state_run: True diff --git a/salt/manager/tools/sbin/so-push-drainer b/salt/manager/tools/sbin/so-push-drainer index 4ce198fd0..fe25d31fc 100644 --- a/salt/manager/tools/sbin/so-push-drainer +++ b/salt/manager/tools/sbin/so-push-drainer @@ -60,9 +60,9 @@ def _make_logger(): def _load_push_cfg(): - """Read the global:push pillar subtree via salt-call. Returns a dict.""" + """Read the salt:auto_apply pillar subtree via salt-call. Returns a dict.""" caller = salt.client.Caller() - cfg = caller.cmd('pillar.get', 'global:push', {}) + cfg = caller.cmd('pillar.get', 'salt:auto_apply', {}) return cfg if isinstance(cfg, dict) else {} @@ -135,7 +135,7 @@ def main(): try: push = _load_push_cfg() except Exception: - log.exception('failed to read global:push pillar; aborting drain pass') + log.exception('failed to read salt:auto_apply pillar; aborting drain pass') return 1 if not push.get('enabled', True): diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 2b8680191..e667e6448 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -690,6 +690,21 @@ ensure_postgres_local_pillar() { chown -R socore:socore "$dir" } +ensure_salt_local_pillar() { + # The salt.auto_apply settings (moved from global.push) are a new SOC settings + # module, so the new pillar/top.sls references salt.soc_salt / salt.adv_salt + # unconditionally. Managers upgrading from before this change have no + # /opt/so/saltstack/local/pillar/salt/ (make_some_dirs only runs at install + # time), so the stubs must be created here before salt-master restarts against + # the new top.sls. + echo "Ensuring salt local pillar stubs exist." + local dir=/opt/so/saltstack/local/pillar/salt + mkdir -p "$dir" + [[ -f "$dir/soc_salt.sls" ]] || touch "$dir/soc_salt.sls" + [[ -f "$dir/adv_salt.sls" ]] || touch "$dir/adv_salt.sls" + chown -R socore:socore "$dir" +} + ensure_postgres_secret() { # On a fresh install, generate_passwords + secrets_pillar seed # secrets:postgres_pass in /opt/so/saltstack/local/pillar/secrets.sls. That @@ -851,6 +866,8 @@ kibana_backport_streams_index_template() { } up_to_3.2.0() { + ensure_salt_local_pillar + fix_logstash_0013_lumberjack_pipeline_name pin_elasticsearch_data_retention_method diff --git a/salt/orch/push_batch.sls b/salt/orch/push_batch.sls index 9eb435cce..ba8676961 100644 --- a/salt/orch/push_batch.sls +++ b/salt/orch/push_batch.sls @@ -1,7 +1,7 @@ -{% from 'global/map.jinja' import GLOBALMERGED %} +{% from 'salt/auto_apply.map.jinja' import AUTOAPPLY %} {% set actions = salt['pillar.get']('actions', []) %} -{% set BATCH = GLOBALMERGED.push.batch %} -{% set BATCH_WAIT = GLOBALMERGED.push.batch_wait %} +{% set BATCH = AUTOAPPLY.batch %} +{% set BATCH_WAIT = AUTOAPPLY.batch_wait %} {% for action in actions %} {% if action.get('highstate') %} diff --git a/salt/reactor/pillar_push_map.yaml b/salt/reactor/pillar_push_map.yaml index 36699d3de..51ad1db81 100644 --- a/salt/reactor/pillar_push_map.yaml +++ b/salt/reactor/pillar_push_map.yaml @@ -185,6 +185,17 @@ registry: - state: registry tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone' +# salt: fanout to a fleetwide highstate. The salt.auto_apply settings tune the +# push pipeline itself (enabled, debounce/drain intervals, batch sizing) and the +# per-minion highstate schedule; they are consumed by the manager's schedule, +# beacons, and master reactor config as well as every minion's highstate +# schedule, so a targeted re-apply isn't meaningful. A salt audit row only fires +# for SOC-driven salt.auto_apply edits -- salt version bumps go through soup, not +# SOC, so they never reach this map. +salt: + - highstate: True + tgt: '*' + # sensoroni: universal. sensoroni: - state: sensoroni diff --git a/salt/reactor/push_pillar.sls b/salt/reactor/push_pillar.sls index 06ee474df..47e586788 100644 --- a/salt/reactor/push_pillar.sls +++ b/salt/reactor/push_pillar.sls @@ -59,9 +59,9 @@ def _load_push_map(): def _push_enabled(): try: caller = Caller() - return bool(caller.cmd('pillar.get', 'global:push:enabled', True)) + return bool(caller.cmd('pillar.get', 'salt:auto_apply:enabled', True)) except Exception: - LOG.exception('push_pillar: pillar.get global:push:enabled failed, assuming enabled') + LOG.exception('push_pillar: pillar.get salt:auto_apply:enabled failed, assuming enabled') return True diff --git a/salt/reactor/push_strelka.sls b/salt/reactor/push_strelka.sls index 1d6a2b044..21727bc91 100644 --- a/salt/reactor/push_strelka.sls +++ b/salt/reactor/push_strelka.sls @@ -35,9 +35,9 @@ def _sensor_compound(): def _push_enabled(): try: caller = Caller() - return bool(caller.cmd('pillar.get', 'global:push:enabled', True)) + return bool(caller.cmd('pillar.get', 'salt:auto_apply:enabled', True)) except Exception: - LOG.exception('push_strelka: pillar.get global:push:enabled failed, assuming enabled') + LOG.exception('push_strelka: pillar.get salt:auto_apply:enabled failed, assuming enabled') return True diff --git a/salt/reactor/push_suricata.sls b/salt/reactor/push_suricata.sls index 468249296..53900e469 100644 --- a/salt/reactor/push_suricata.sls +++ b/salt/reactor/push_suricata.sls @@ -34,9 +34,9 @@ def _sensor_compound_plus_import(): def _push_enabled(): try: caller = Caller() - return bool(caller.cmd('pillar.get', 'global:push:enabled', True)) + return bool(caller.cmd('pillar.get', 'salt:auto_apply:enabled', True)) except Exception: - LOG.exception('push_suricata: pillar.get global:push:enabled failed, assuming enabled') + LOG.exception('push_suricata: pillar.get salt:auto_apply:enabled failed, assuming enabled') return True diff --git a/salt/salt/adv_salt.yaml b/salt/salt/adv_salt.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/salt/salt/auto_apply.map.jinja b/salt/salt/auto_apply.map.jinja new file mode 100644 index 000000000..6a353f4f8 --- /dev/null +++ b/salt/salt/auto_apply.map.jinja @@ -0,0 +1,2 @@ +{% import_yaml 'salt/defaults.yaml' as SALT_DEFAULTS %} +{% set AUTOAPPLY = salt['pillar.get']('salt:auto_apply', SALT_DEFAULTS.salt.auto_apply, merge=True) %} diff --git a/salt/salt/defaults.yaml b/salt/salt/defaults.yaml new file mode 100644 index 000000000..7dcbf0ff8 --- /dev/null +++ b/salt/salt/defaults.yaml @@ -0,0 +1,8 @@ +salt: + auto_apply: + enabled: true + highstate_interval_hours: 2 + debounce_seconds: 30 + drain_interval: 15 + batch: '25%' + batch_wait: 15 diff --git a/salt/salt/master.sls b/salt/salt/master.sls index 273e97e84..bb39f1395 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -10,7 +10,7 @@ # software that is protected by the license key." {% from 'allowed_states.map.jinja' import allowed_states %} -{% from 'global/map.jinja' import GLOBALMERGED %} +{% from 'salt/auto_apply.map.jinja' import AUTOAPPLY %} {% if sls in allowed_states %} include: @@ -65,7 +65,7 @@ engines_config: - name: /etc/salt/master.d/engines.conf - source: salt://salt/files/engines.conf -{% if GLOBALMERGED.push.enabled %} +{% if AUTOAPPLY.enabled %} reactor_pushstate_config: file.managed: - name: /etc/salt/master.d/reactor_pushstate.conf diff --git a/salt/salt/soc_salt.yaml b/salt/salt/soc_salt.yaml new file mode 100644 index 000000000..530c5ab79 --- /dev/null +++ b/salt/salt/soc_salt.yaml @@ -0,0 +1,38 @@ +salt: + auto_apply: + enabled: + description: Master kill-switch for the active push feature. When disabled, rule and pillar changes are picked up at the next scheduled highstate instead of being pushed immediately. + forcedType: bool + helpLink: push + global: True + highstate_interval_hours: + description: How often every minion in the grid runs a scheduled state.highstate, in hours. Lower values keep minions closer in sync at the cost of more load; higher values reduce load but increase worst-case latency for non-pushed changes. The salt-minion health check restarts a minion if its last highstate is older than this value plus one hour. + forcedType: int + helpLink: push + global: True + advanced: True + debounce_seconds: + description: Trailing-edge debounce window in seconds. A push intent must be quiet for this long before the drainer dispatches. Rapid bursts of edits within this window coalesce into one dispatch. + forcedType: int + helpLink: push + global: True + advanced: True + drain_interval: + description: How often the push drainer checks for ready intents, in seconds. Small values lower dispatch latency at the cost of more background work on the manager. + forcedType: int + helpLink: push + global: True + advanced: True + batch: + description: "Host batch size for push orchestrations. A number (e.g. '10') or a percentage (e.g. '25%'). Limits how many minions run the push state at once so large fleets don't thundering-herd." + helpLink: push + global: True + advanced: True + regex: '^([0-9]+%?)$' + regexFailureMessage: Enter a whole number or a whole-number percentage (e.g. 10 or 25%). + batch_wait: + description: Seconds to wait between host batches in a push orchestration. Gives the fleet time to breathe between waves. + forcedType: int + helpLink: push + global: True + advanced: True diff --git a/salt/schedule.sls b/salt/schedule.sls index 014d24460..0b71dc65f 100644 --- a/salt/schedule.sls +++ b/salt/schedule.sls @@ -1,22 +1,22 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} -{% from 'global/map.jinja' import GLOBALMERGED %} +{% from 'salt/auto_apply.map.jinja' import AUTOAPPLY %} highstate_schedule: schedule.present: - function: state.highstate - - hours: {{ GLOBALMERGED.push.highstate_interval_hours }} + - hours: {{ AUTOAPPLY.highstate_interval_hours }} - maxrunning: 1 {% if not GLOBALS.is_manager %} - splay: 1800 {% endif %} -{% if GLOBALS.is_manager and GLOBALMERGED.push.enabled %} +{% if GLOBALS.is_manager and AUTOAPPLY.enabled %} push_drain_schedule: schedule.present: - function: cmd.run - job_args: - /usr/sbin/so-push-drainer - - seconds: {{ GLOBALMERGED.push.drain_interval }} + - seconds: {{ AUTOAPPLY.drain_interval }} - maxrunning: 1 - return_job: False {% elif GLOBALS.is_manager %} diff --git a/setup/so-functions b/setup/so-functions index 2d5181dc1..8859aef97 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1432,7 +1432,7 @@ make_some_dirs() { mkdir -p $local_salt_dir/salt/firewall/portgroups mkdir -p $local_salt_dir/salt/firewall/ports - for THEDIR in bpf elasticsearch ntp firewall redis backup influxdb postgres strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos hydra idh elastalert stig global kafka versionlock hypervisor vm; do + for THEDIR in bpf elasticsearch ntp firewall redis backup influxdb postgres strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos hydra idh elastalert stig global salt kafka versionlock hypervisor vm; do mkdir -p $local_salt_dir/pillar/$THEDIR touch $local_salt_dir/pillar/$THEDIR/adv_$THEDIR.sls touch $local_salt_dir/pillar/$THEDIR/soc_$THEDIR.sls From b09c3776b7c2a780ea544e50377644cfca41abec Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 24 Jun 2026 16:51:32 -0400 Subject: [PATCH 098/219] Point pillar_db beacon at securityonion database The SOC postgres database was renamed so_soc -> securityonion (see POSTGRES_DB in salt/postgres/enabled.sls and the SOC postgres config in salt/soc/defaults.yaml). The pillar_db beacon still hardcoded so_soc, so every poll failed with 'database "so_soc" does not exist' (rc=2), silently disabling active-push detection of audit_settings changes. Update DATABASE to 'securityonion' and refresh the now-stale so_soc references in the beacon and push_pillar reactor comments. --- salt/_beacons/pillar_db.py | 6 +++--- salt/reactor/push_pillar.sls | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/_beacons/pillar_db.py b/salt/_beacons/pillar_db.py index 9022d9b87..8712cce7f 100644 --- a/salt/_beacons/pillar_db.py +++ b/salt/_beacons/pillar_db.py @@ -6,7 +6,7 @@ # Custom salt beacon that watches the SOC audit_settings table in postgres for # new settings changes and emits a beacon event per new row. This replaces the # inotify watch on /opt/so/saltstack/local/pillar -- instead of monitoring pillar -# files on disk, we monitor the so_soc.audit_settings table that SOC writes to. +# files on disk, we monitor the securityonion.audit_settings table that SOC writes to. # # Detection is poll-based with a monotonic `id` watermark persisted to # WATERMARK_FILE: each pass selects rows with id greater than the last id seen, @@ -24,7 +24,7 @@ log = logging.getLogger(__name__) WATERMARK_FILE = '/opt/so/state/pillar_db_watch.id' CONTAINER = 'so-postgres' -DATABASE = 'so_soc' +DATABASE = 'securityonion' # Unaligned, tuples-only psql output with a field separator that cannot appear in # an id/setting_id/node_id, so we can split each row reliably. @@ -60,7 +60,7 @@ def _write_watermark(value): def _query(sql): - # Run a query against so_soc inside the so-postgres container over the unix + # Run a query against securityonion inside the so-postgres container over the unix # socket (trust auth, no password). Returns stdout on success, or None on any # failure so the caller can no-op and retry on the next interval. cmd = [ diff --git a/salt/reactor/push_pillar.sls b/salt/reactor/push_pillar.sls index 47e586788..c4b82959b 100644 --- a/salt/reactor/push_pillar.sls +++ b/salt/reactor/push_pillar.sls @@ -1,7 +1,7 @@ #!py # Reactor invoked by the pillar_db beacon when SOC records settings changes in -# the so_soc.audit_settings table (see salt/_beacons/pillar_db.py). The beacon +# the securityonion.audit_settings table (see salt/_beacons/pillar_db.py). The beacon # emits one event per new row carrying setting_id and node_id. # # Two branches, keyed on node_id: From a9c03e39bbe1f2b37c6626108a1ef2374082e6b7 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 25 Jun 2026 09:32:08 -0400 Subject: [PATCH 099/219] support multiple capinfos versions --- salt/common/tools/sbin_jinja/so-import-pcap | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin_jinja/so-import-pcap b/salt/common/tools/sbin_jinja/so-import-pcap index 9171c4bc6..53c9cd825 100755 --- a/salt/common/tools/sbin_jinja/so-import-pcap +++ b/salt/common/tools/sbin_jinja/so-import-pcap @@ -63,7 +63,8 @@ function status { function pcapinfo() { PCAP=$1 ARGS=$2 - docker run --rm -v "$PCAP:/input.pcap" --entrypoint capinfos {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} /input.pcap -ae $ARGS + docker run --rm -v "$PCAP:/input.pcap" --entrypoint capinfos {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} /input.pcap -ae $ARGS |\ + sed 's/First packet/Earliest packet/g' | sed 's/Last packet/Latest packet/g' } function pcapfix() { From 3effdbc91ed33ef95136834e0af7cad146ac30ed Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 25 Jun 2026 11:36:52 -0400 Subject: [PATCH 100/219] do not disable during state run --- salt/manager/files/beacons_pushstate.conf.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/files/beacons_pushstate.conf.jinja b/salt/manager/files/beacons_pushstate.conf.jinja index 9f45d5d85..05d739dd4 100644 --- a/salt/manager/files/beacons_pushstate.conf.jinja +++ b/salt/manager/files/beacons_pushstate.conf.jinja @@ -4,7 +4,7 @@ beacons: - interval: {{ AUTOAPPLY.drain_interval }} - disable_during_state_run: True inotify: - - disable_during_state_run: True + - disable_during_state_run: False - coalesce: True - files: /opt/so/saltstack/local/salt/suricata/rules: From 5bf9751adf5e1aacdee0a11a0f99402805c4acba Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 25 Jun 2026 11:44:38 -0400 Subject: [PATCH 101/219] do not disable during state run --- salt/manager/files/beacons_pushstate.conf.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/files/beacons_pushstate.conf.jinja b/salt/manager/files/beacons_pushstate.conf.jinja index 05d739dd4..4de5ef5e0 100644 --- a/salt/manager/files/beacons_pushstate.conf.jinja +++ b/salt/manager/files/beacons_pushstate.conf.jinja @@ -2,7 +2,7 @@ beacons: pillar_db: - interval: {{ AUTOAPPLY.drain_interval }} - - disable_during_state_run: True + - disable_during_state_run: False inotify: - disable_during_state_run: False - coalesce: True From d0edfd213176b75ea7bfe970df4822a7c8ee891e Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 25 Jun 2026 14:18:43 -0400 Subject: [PATCH 102/219] set transport for ssl.established:false logs --- salt/elasticsearch/files/ingest/zeek.ssl | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/files/ingest/zeek.ssl b/salt/elasticsearch/files/ingest/zeek.ssl index 0bd6fedb2..80a7b12da 100644 --- a/salt/elasticsearch/files/ingest/zeek.ssl +++ b/salt/elasticsearch/files/ingest/zeek.ssl @@ -5,6 +5,7 @@ { "remove": { "field": ["host"], "ignore_failure": true } }, { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.version", "target_field": "ssl.version", "ignore_missing": true } }, + { "set": { "description": "Set transport for the community_id processor", "if": "ctx.ssl?.version == null || !ctx.ssl.version.startsWith('DTLS')", "field": "network.transport", "value": "tcp", "ignore_failure": true } }, { "rename": { "field": "message2.cipher", "target_field": "ssl.cipher", "ignore_missing": true } }, { "rename": { "field": "message2.curve", "target_field": "ssl.curve", "ignore_missing": true } }, { "rename": { "field": "message2.server_name", "target_field": "ssl.server_name", "ignore_missing": true } }, From 94f31e1356e67230a607ba7ffb9b3b3e066ad92a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 26 Jun 2026 09:21:11 -0400 Subject: [PATCH 103/219] Add so-kernel-upgrade to switch the boot default to the UEK8 kernel Installing kernel-uek-core adds a UEK8 (6.x) boot entry but doesn't make it the default, because grubby only auto-promotes within the running kernel's flavor lineage and we cross from a 5.x kernel to the new UEK8 flavor. so-kernel-upgrade finds the newest installed 6.x UEK kernel and grubby --set-default's it (idempotent, verifies the change, no reboot). --- salt/common/tools/sbin/so-kernel-upgrade | 57 ++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100755 salt/common/tools/sbin/so-kernel-upgrade diff --git a/salt/common/tools/sbin/so-kernel-upgrade b/salt/common/tools/sbin/so-kernel-upgrade new file mode 100755 index 000000000..46d471051 --- /dev/null +++ b/salt/common/tools/sbin/so-kernel-upgrade @@ -0,0 +1,57 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. +# +# so-kernel-upgrade — switch the boot default to the installed UEK8 (6.x) kernel. +# +# Security Onion is moving off the EL9 stock kernel / UEK7 (5.x) onto UEK8 (6.x). +# Installing the kernel-uek-core package adds a UEK8 boot entry but does NOT make it the +# default: kernel-install/grubby only auto-promote a new kernel within the running +# kernel's flavor lineage, and we're crossing from a 5.x kernel to the new 6.x UEK flavor. +# So even with UPDATEDEFAULT=yes and DEFAULTKERNEL=kernel-uek-core the box keeps booting +# the old kernel. This tool finds the newest installed 6.x UEK kernel and makes it the +# GRUB default via grubby so the next boot comes up on UEK8. +# +# Idempotent: if the UEK8 kernel is already the default it does nothing. It only sets the +# boot default; it does NOT reboot — the admin reboots the node on their own schedule. + +log() { echo "[so-kernel-upgrade] $*"; } + +[ "$(id -u)" -eq 0 ] || { log "must run as root"; exit 1; } +command -v grubby >/dev/null 2>&1 || { log "grubby not found"; exit 1; } + +# Newest installed UEK8 (6.x) kernel known to the bootloader. UEK8 vmlinuz paths look like +# /boot/vmlinuz-6.12.0-203.76.7.5.el9uek.x86_64; the 5.x UEK7 and 5.14 RHCK won't match. +target="$(grubby --info=ALL 2>/dev/null \ + | sed -n 's/^kernel="\(.*\)"$/\1/p' \ + | grep -E '/vmlinuz-6\.[0-9]+.*uek' \ + | sort -V | tail -1)" + +if [ -z "$target" ]; then + log "no installed 6.x UEK (UEK8) kernel found — confirm the kernel repo is assigned and" + log "'dnf update' has installed kernel-uek-core. Nothing to do." + exit 0 +fi + +current="$(grubby --default-kernel 2>/dev/null)" +if [ "$current" = "$target" ]; then + log "UEK8 kernel is already the boot default: $target" + exit 0 +fi + +log "current default kernel: ${current:-unknown}" +log "switching boot default to UEK8 kernel: $target" +grubby --set-default="$target" || { log "ERROR: grubby --set-default failed for $target"; exit 1; } + +# Verify the change actually took before claiming success. +now="$(grubby --default-kernel 2>/dev/null)" +if [ "$now" != "$target" ]; then + log "ERROR: default kernel is still '${now:-unknown}' after set-default" + exit 1 +fi + +log "boot default is now $target" +log "REBOOT REQUIRED to start using the UEK8 kernel (currently running $(uname -r))." From 67a9abadf2292ac3005844ef4b581020adecc7ff Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 26 Jun 2026 09:21:11 -0400 Subject: [PATCH 104/219] Gate so_kernel_repo on running salt matching the shipped version During soup the grid is mid-salt-upgrade. Only assign the UEK8 kernel repo once the node's grains.saltversion matches salt.minion.version from minion.defaults.yaml, so the kernel repo and the update it enables don't activate until the node is fully on the target salt. --- salt/repo/client/oracle.sls | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/repo/client/oracle.sls b/salt/repo/client/oracle.sls index 2019a56d1..bf0a02751 100644 --- a/salt/repo/client/oracle.sls +++ b/salt/repo/client/oracle.sls @@ -6,6 +6,10 @@ {% from 'repo/client/map.jinja' import REPOPATH with context %} {% from 'vars/globals.map.jinja' import GLOBALS %} +{% import_yaml 'salt/minion.defaults.yaml' as saltversion %} +{% set saltversion = saltversion.salt.minion.version %} +{% set INSTALLEDSALTVERSION = grains.saltversion %} + {% set role = grains.id.split('_') | last %} {% set MANAGER = salt['grains.get']('master') %} {% if grains['os'] == 'OEL' %} @@ -57,6 +61,11 @@ so_repo: - enabled: 1 - gpgcheck: 1 +# Only assign the kernel repo once this node's running salt matches the version this +# SO release ships. During a soup the grid is mid-salt-upgrade; gating here keeps the +# UEK8 kernel repo (and the kernel update it enables) from activating until the node is +# fully on the target salt, the same way other states defer across the upgrade window. +{% if saltversion | string == INSTALLEDSALTVERSION | string %} so_kernel_repo: pkgrepo.managed: - name: securityonionkernel @@ -76,6 +85,7 @@ so_kernel_repo: # UEK8 kernel update can't renumber interfaces SO binds by name (see pin_nic_names # in salt/common/init.sls, which drops this marker via /usr/sbin/so-nic-pin). - onlyif: 'test -e /opt/so/state/nic_names_pinned' +{% endif %} {% endif %} From da94788255407268b2edbfcae0894e20d185b7bd Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 26 Jun 2026 10:51:41 -0400 Subject: [PATCH 105/219] Move highstate_interval_hours to salt.schedule and split schedule.sls highstate_interval_hours describes the per-minion highstate schedule, not the active-push pipeline, so relocate it from salt.auto_apply to a new salt.schedule settings subtree. Repoint so-salt-minion-check at the new pillar path (it had been left on the stale global:push path) so its restart grace period tracks the schedule again. - Add salt.schedule.highstate_interval_hours to defaults.yaml/soc_salt.yaml and a side-effect-free salt/salt/schedule.map.jinja (SCHEDULEMERGED), matching the *MERGED map convention. Consumers read SCHEDULEMERGED.highstate_interval_hours. - Split salt/schedule.sls into salt/salt/highstate_schedule.sls (every minion) and salt/salt/push_drain_schedule.sls (managers); update top.sls to apply the highstate schedule via '*' and the drainer schedule via the configured-manager block. Remove the now-empty schedule.sls aggregator. - pillar_push_map.yaml and so-push-drainer: comment/doc updates only. --- salt/common/tools/sbin_jinja/so-salt-minion-check | 6 +++--- salt/manager/tools/sbin/so-push-drainer | 2 +- salt/reactor/pillar_push_map.yaml | 12 ++++++------ salt/salt/defaults.yaml | 3 ++- salt/salt/highstate_schedule.sls | 11 +++++++++++ salt/{schedule.sls => salt/push_drain_schedule.sls} | 9 --------- salt/salt/schedule.map.jinja | 2 ++ salt/salt/soc_salt.yaml | 13 +++++++------ salt/top.sls | 4 ++-- 9 files changed, 34 insertions(+), 28 deletions(-) create mode 100644 salt/salt/highstate_schedule.sls rename salt/{schedule.sls => salt/push_drain_schedule.sls} (69%) create mode 100644 salt/salt/schedule.map.jinja diff --git a/salt/common/tools/sbin_jinja/so-salt-minion-check b/salt/common/tools/sbin_jinja/so-salt-minion-check index 3b2b32afe..1376193cc 100755 --- a/salt/common/tools/sbin_jinja/so-salt-minion-check +++ b/salt/common/tools/sbin_jinja/so-salt-minion-check @@ -5,7 +5,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - +{% from 'salt/schedule.map.jinja' import SCHEDULEMERGED %} # this script checks the time the file /opt/so/log/salt/state-apply-test was last modified and restarts the salt-minion service if it is outside a threshold date/time # the file is modified via file.touch using a scheduled job healthcheck.salt-minion.state-apply-test that runs a state.apply. @@ -23,8 +23,8 @@ SYSTEM_START_TIME=$(date -d "$( Date: Fri, 26 Jun 2026 12:02:49 -0400 Subject: [PATCH 106/219] Serve /kernelrepo through nginx so minions can reach the kernel repo The /nsm/kernelrepo bind mount exposed the files, but without a matching location block external requests to /kernelrepo/ fell through to the SOC app and returned HTML, so minions hit 'repomd.xml parser error'. Add a /kernelrepo/ location mirroring /repo/. --- salt/nginx/etc/nginx.conf | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index 8150265f5..b7a70da2b 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -323,6 +323,16 @@ http { autoindex_localtime on; } + location /kernelrepo/ { + allow all; + sendfile on; + sendfile_max_chunk 1m; + autoindex on; + autoindex_exact_size off; + autoindex_format html; + autoindex_localtime on; + } + location /influxdb/ { auth_request /auth/sessions/whoami; rewrite /influxdb/api/(.*) /api/$1 break; From b3b7ecddede9956001a3be9f56d52e5c546a8d13 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 26 Jun 2026 13:19:18 -0500 Subject: [PATCH 107/219] update so-stop | so-start | so-restart scripts --- salt/common/tools/sbin/so-common | 14 +++++++ salt/common/tools/sbin/so-restart | 54 ++++++++++++++++--------- salt/common/tools/sbin/so-start | 67 ++++++++++++++++++++++--------- salt/common/tools/sbin/so-stop | 38 ++++++++++++------ 4 files changed, 120 insertions(+), 53 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 812c1bb10..4e6580ae1 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -291,6 +291,20 @@ download_and_verify() { fi } +# check if container with name is running and optionally stop it +docker_check_running() { + # show running containers, only names + if docker ps --format '{{.Names}}' | grep -q "^so-${1}$"; then + if [[ "$2" == "--stop" ]]; then + docker stop "so-${1}" + fi + + return 0 + else + return 1 + fi +} + elastic_license() { read -r -d '' message <<- EOM diff --git a/salt/common/tools/sbin/so-restart b/salt/common/tools/sbin/so-restart index 7345078b8..14747d134 100755 --- a/salt/common/tools/sbin/so-restart +++ b/salt/common/tools/sbin/so-restart @@ -5,27 +5,41 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - - -# Usage: so-restart kibana | playbook - . /usr/sbin/so-common -if [ $# -ge 1 ]; then +usage() { + echo "Usage: $0 [args]" + echo "" + echo "Supported args:" + echo " --force | -f Force stop all Salt jobs before starting component." + echo "" + echo "Examples:" + echo " $0 kibana Restart Kibana" + echo " $0 kibana --force Force stop all Salt jobs before restarting Kibana" + exit 1 +} - echo $banner - printf "Restarting $1...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n" - echo $banner - - if [ "$2" = "--force" ]; then - printf "\nForce-stopping all Salt jobs before proceeding\n\n" - salt-call saltutil.kill_all_jobs - fi - - case $1 in - "elastic-fleet") docker stop so-elastic-fleet && docker rm so-elastic-fleet && salt-call state.apply elasticfleet queue=True;; - *) docker stop so-$1 ; docker rm so-$1 ; salt-call state.apply $1 queue=True;; - esac -else - echo -e "\nPlease provide an argument by running like so-restart $component, or by using the component-specific script.\nEx. so-restart logstash, or so-logstash-restart\n" +if [[ $# -lt 1 ]]; then + usage fi + +#shellcheck disable=SC2154 +echo "$banner" +printf "Restarting %s...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n" "$1" +echo "$banner" +if [[ "$2" = "--force" ]] || [[ "$2" = "-f" ]]; then + printf "\nForce-stopping all Salt jobs before proceeding\n\n" + salt-call saltutil.kill_all_jobs +fi +case $1 in + "elastic-fleet"|"elasticfleet") + docker_check_running "elastic-fleet" "--stop" + docker rm "so-elastic-fleet" 2> /dev/null + salt-call state.apply elasticfleet queue=True + ;; + *) + docker_check_running "$1" "--stop" + docker rm "so-${1}" 2> /dev/null + salt-call state.apply "$1" queue=True + ;; +esac diff --git a/salt/common/tools/sbin/so-start b/salt/common/tools/sbin/so-start index 1a312a94d..a5c66ffe7 100755 --- a/salt/common/tools/sbin/so-start +++ b/salt/common/tools/sbin/so-start @@ -5,27 +5,54 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - - -# Usage: so-start all | kibana | playbook - +# shellcheck disable=SC1091 . /usr/sbin/so-common -if [ $# -ge 1 ]; then - echo $banner - printf "Starting $1...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n" - echo $banner +usage() { + echo "Usage: $0 [args]" + echo "" + echo "Supported args:" + echo " --force | -f Force stop all Salt jobs before starting component." + echo "" + echo "Examples:" + echo " $0 kibana Start Kibana" + echo " $0 kibana --force Force stop all Salt jobs before starting Kibana" + exit 1 +} - if [ "$2" = "--force" ]; then - printf "\nForce-stopping all Salt jobs before proceeding\n\n" - salt-call saltutil.kill_all_jobs - fi - - case $1 in - "all") salt-call state.highstate queue=True;; - "elastic-fleet") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply elasticfleet queue=True; fi ;; - *) if docker ps | grep -E -q '^so-$1$'; then printf "\n$1 is already running\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply $1 queue=True; fi ;; - esac -else - echo -e "\nPlease provide an argument by running like so-start $component, or by using the component-specific script.\nEx. so-start logstash, or so-logstash-start\n" +if [[ $# -lt 1 ]]; then + usage fi + +#shellcheck disable=SC2154 +echo "$banner" +printf "Starting %s...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n" "$1" +echo "$banner" +if [[ "$2" = "--force" ]] || [[ "$2" == "-f" ]]; then + printf "\nForce-stopping all Salt jobs before proceeding\n\n" + salt-call saltutil.kill_all_jobs +fi + +case "$1" in + "all") + salt-call state.highstate queue=True + ;; + "elastic-fleet"|"elasticfleet") + if docker_check_running "elastic-fleet"; then + printf "\nso-%s is already running!\n\n" "elastic-fleet" + /usr/sbin/so-status + else + docker rm "so-elastic-fleet" 2> /dev/null + salt-call state.apply elasticfleet queue=True + fi + ;; + *) + if docker_check_running "$1"; then + printf "\nso-%s is already running\n\n" "$1" + /usr/sbin/so-status + else + docker rm "so-${1}" 2> /dev/null + salt-call state.apply "$1" queue=True + fi + ;; +esac diff --git a/salt/common/tools/sbin/so-stop b/salt/common/tools/sbin/so-stop index 32e24f83a..d036a7b63 100755 --- a/salt/common/tools/sbin/so-stop +++ b/salt/common/tools/sbin/so-stop @@ -5,21 +5,33 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - - -# Usage: so-stop kibana | playbook | thehive - +# shellcheck disable=SC1091 . /usr/sbin/so-common -if [ $# -ge 1 ]; then - echo $banner - printf "Stopping $1...\n" - echo $banner +usage() { + echo "Usage: $0 " + echo "" + echo "Examples:" + echo " $0 kibana Stop Kibana" + exit 1 +} - case $1 in - *) docker stop so-$1 ; docker rm so-$1 ;; - esac -else - echo -e "\nPlease provide an argument by running like so-stop $component, or by using the component-specific script.\nEx. so-stop logstash, or so-logstash-stop\n" +if [[ $# -lt 1 ]]; then + usage fi + +#shellcheck disable=SC2154 +echo "$banner" +printf "Stopping %s...\n" "$1" +echo "$banner" +case $1 in + "elasticfleet"|"elastic-fleet") + docker_check_running "elastic-fleet" "--stop" + docker rm "so-elastic-fleet" 2> /dev/null + ;; + *) + docker_check_running "$1" "--stop" + docker rm "so-${1}" 2> /dev/null + ;; +esac From 12f44478754205f5dc1128958a5f790d6ed2ed79 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 26 Jun 2026 15:40:32 -0400 Subject: [PATCH 108/219] Replace inotify rule-watch beacon with poll-based rules_db beacon Salt's stock inotify beacon leaks one kernel inotify instance every time the minion rebuilds the beacon loader's __context__ (the orphaned pyinotify.Notifier is never stopped), accumulating against fs.inotify.max_user_instances=128 until inotify_init() fails with EMFILE and rule-change push detection silently stops. This is independent of disable_during_state_run. Add a custom poll-based beacon (salt/_beacons/rules_db.py) modeled on pillar_db.py: it fingerprints the suricata/strelka rule dirs each interval (relpath + mtime_ns + size, temp files excluded) against a per-dir watermark, emitting an event only on change. It holds zero inotify instances, so the leak is impossible, and it keeps firing during state runs. Swap the inotify beacon config and reactor tag mappings accordingly; the push_suricata/push_strelka reactors are unchanged (they read only data['path']). --- salt/_beacons/rules_db.py | 139 ++++++++++++++++++ .../files/beacons_pushstate.conf.jinja | 40 +---- salt/reactor/push_strelka.sls | 4 +- salt/reactor/push_suricata.sls | 4 +- salt/salt/files/reactor_pushstate.conf | 8 +- 5 files changed, 150 insertions(+), 45 deletions(-) create mode 100644 salt/_beacons/rules_db.py diff --git a/salt/_beacons/rules_db.py b/salt/_beacons/rules_db.py new file mode 100644 index 000000000..ab63da431 --- /dev/null +++ b/salt/_beacons/rules_db.py @@ -0,0 +1,139 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Custom salt beacon that watches the suricata/strelka rule directories for changes +# and emits a beacon event per changed directory. This replaces the stock salt +# `inotify` beacon, which leaks a kernel inotify instance every time the minion +# rebuilds the beacon loader's __context__ (orphaning the old pyinotify.Notifier +# without closing it) until fs.inotify.max_user_instances is exhausted and the +# beacon dies with EMFILE. Polling holds zero inotify instances, so the leak is +# impossible, and it keeps firing during state runs (no blackout). +# +# Detection is poll-based with a per-directory fingerprint persisted to +# WATERMARK_DIR: each pass walks the directory and hashes every file's +# (relpath, st_mtime_ns, st_size), which catches content writes, additions, +# moves, and deletions. A change in the digest emits one event; an unchanged +# digest emits nothing. This makes it self-healing (a missed poll simply catches +# up on the next one). +# +# Each emitted event carries the watched directory path under the configured tag +# (e.g. salt/beacon//rules_db/suricata); the push_suricata / push_strelka +# reactors write a push intent, after which the existing so-push-drainer / +# orch.push_batch pipeline takes over unchanged. + +import hashlib +import logging +import os +import re + +log = logging.getLogger(__name__) + +WATERMARK_DIR = '/opt/so/state' + +# Temp/editor files that should not trigger a push. Mirrors the exclude regexes +# the inotify beacon used. Matched against the full pathname. +EXCLUDES = [ + re.compile(r'\.sw[a-z]$'), + re.compile(r'~$'), + re.compile(r'/4913$'), + re.compile(r'/\.#'), +] + + +def __virtual__(): + return True + + +def validate(config): + return True, 'valid' + + +def _paths_from_config(config): + # The beacon config arrives as a list of single-key dicts (salt beacon style). + # Merge it and return the {dir: tag} mapping under the 'paths' key. + merged = {} + if isinstance(config, list): + for item in config: + if isinstance(item, dict): + merged.update(item) + elif isinstance(config, dict): + merged = config + paths = merged.get('paths', {}) + return paths if isinstance(paths, dict) else {} + + +def _excluded(pathname): + for pattern in EXCLUDES: + if pattern.search(pathname): + return True + return False + + +def _fingerprint(directory): + # Stat-only walk; hash each file's (relpath, mtime_ns, size). Returns a hex + # digest, or the digest of an empty tree if the directory does not exist. + h = hashlib.sha1() + if os.path.isdir(directory): + entries = [] + for root, _dirs, files in os.walk(directory): + for name in files: + full = os.path.join(root, name) + if _excluded(full): + continue + try: + st = os.stat(full) + except OSError: + continue + rel = os.path.relpath(full, directory) + entries.append('%s\0%d\0%d' % (rel, st.st_mtime_ns, st.st_size)) + for line in sorted(entries): + h.update(line.encode('utf-8', 'surrogateescape')) + h.update(b'\n') + return h.hexdigest() + + +def _watermark_file(tag): + return os.path.join(WATERMARK_DIR, 'rules_db_%s.hash' % tag) + + +def _read_watermark(tag): + try: + with open(_watermark_file(tag), 'r') as f: + return (f.read() or '').strip() or None + except IOError: + return None + + +def _write_watermark(tag, digest): + path = _watermark_file(tag) + try: + os.makedirs(WATERMARK_DIR, exist_ok=True) + tmp = path + '.tmp' + with open(tmp, 'w') as f: + f.write(digest) + os.rename(tmp, path) + except OSError: + log.exception('rules_db beacon: failed to persist watermark to %s', path) + + +def beacon(config): + retval = [] + + for directory, tag in _paths_from_config(config).items(): + digest = _fingerprint(directory) + previous = _read_watermark(tag) + + # First run / missing watermark: seed the digest and emit nothing so a + # fresh host does not fire a spurious fleetwide push. + if previous is None: + _write_watermark(tag, digest) + continue + + if digest != previous: + _write_watermark(tag, digest) + retval.append({'tag': tag, 'path': directory}) + log.info('rules_db beacon: change detected in %s, emitting %s', directory, tag) + + return retval diff --git a/salt/manager/files/beacons_pushstate.conf.jinja b/salt/manager/files/beacons_pushstate.conf.jinja index 4de5ef5e0..2c00163e3 100644 --- a/salt/manager/files/beacons_pushstate.conf.jinja +++ b/salt/manager/files/beacons_pushstate.conf.jinja @@ -3,39 +3,9 @@ beacons: pillar_db: - interval: {{ AUTOAPPLY.drain_interval }} - disable_during_state_run: False - inotify: + rules_db: + - interval: {{ AUTOAPPLY.drain_interval }} - disable_during_state_run: False - - coalesce: True - - files: - /opt/so/saltstack/local/salt/suricata/rules: - mask: - - close_write - - moved_to - - delete - recurse: True - auto_add: True - exclude: - - '\.sw[a-z]$': - regex: True - - '~$': - regex: True - - '/4913$': - regex: True - - '/\.#': - regex: True - /opt/so/saltstack/local/salt/strelka/rules/compiled: - mask: - - close_write - - moved_to - - delete - recurse: True - auto_add: True - exclude: - - '\.sw[a-z]$': - regex: True - - '~$': - regex: True - - '/4913$': - regex: True - - '/\.#': - regex: True + - paths: + /opt/so/saltstack/local/salt/suricata/rules: suricata + /opt/so/saltstack/local/salt/strelka/rules/compiled: strelka diff --git a/salt/reactor/push_strelka.sls b/salt/reactor/push_strelka.sls index 21727bc91..d1d0207eb 100644 --- a/salt/reactor/push_strelka.sls +++ b/salt/reactor/push_strelka.sls @@ -1,7 +1,7 @@ #!py -# Reactor invoked by the inotify beacon on rule file changes under -# /opt/so/saltstack/local/salt/strelka/rules/compiled/. +# Reactor invoked by the rules_db poll beacon (salt/_beacons/rules_db.py) on rule +# file changes under /opt/so/saltstack/local/salt/strelka/rules/compiled/. # # Writes (or updates) a push intent at /opt/so/state/push_pending/rules_strelka.json # and returns {}. The so-push-drainer schedule picks up ready intents, dedupes diff --git a/salt/reactor/push_suricata.sls b/salt/reactor/push_suricata.sls index 53900e469..f50a92527 100644 --- a/salt/reactor/push_suricata.sls +++ b/salt/reactor/push_suricata.sls @@ -1,7 +1,7 @@ #!py -# Reactor invoked by the inotify beacon on rule file changes under -# /opt/so/saltstack/local/salt/suricata/rules/. +# Reactor invoked by the rules_db poll beacon (salt/_beacons/rules_db.py) on rule +# file changes under /opt/so/saltstack/local/salt/suricata/rules/. # # Writes (or updates) a push intent at /opt/so/state/push_pending/rules_suricata.json # and returns {}. The so-push-drainer schedule picks up ready intents, dedupes diff --git a/salt/salt/files/reactor_pushstate.conf b/salt/salt/files/reactor_pushstate.conf index 991c4d516..b4543b1a7 100644 --- a/salt/salt/files/reactor_pushstate.conf +++ b/salt/salt/files/reactor_pushstate.conf @@ -1,11 +1,7 @@ reactor: - - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/suricata/rules': + - 'salt/beacon/*/rules_db/suricata': - salt://reactor/push_suricata.sls - - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/suricata/rules/*': - - salt://reactor/push_suricata.sls - - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/strelka/rules/compiled': - - salt://reactor/push_strelka.sls - - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/strelka/rules/compiled/*': + - 'salt/beacon/*/rules_db/strelka': - salt://reactor/push_strelka.sls - 'salt/beacon/*/pillar_db/audit_settings': - salt://reactor/push_pillar.sls From a330bea25e1b4920589fd009d644d921ed6d98be Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 29 Jun 2026 14:29:07 -0400 Subject: [PATCH 109/219] Rename push-detection beacons to clearer names Rename the two custom push-detection beacons for clarity: - pillar_db -> postgres_pillar_beacon - rules_db -> rules_beacon Salt resolves a beacon by its config-key name to a _beacons/ module of the same filename and tags its events salt/beacon///, so each rename touches the module file, the beacon config key in beacons_pushstate.conf.jinja, and the reactor tag patterns in reactor_pushstate.conf together. Watermark filenames and log prefixes are updated to match; reactor run() logic is unchanged. --- ...{pillar_db.py => postgres_pillar_beacon.py} | 18 +++++++++--------- salt/_beacons/{rules_db.py => rules_beacon.py} | 8 ++++---- .../manager/files/beacons_pushstate.conf.jinja | 4 ++-- salt/reactor/push_pillar.sls | 6 +++--- salt/reactor/push_strelka.sls | 2 +- salt/reactor/push_suricata.sls | 2 +- salt/salt/files/reactor_pushstate.conf | 6 +++--- 7 files changed, 23 insertions(+), 23 deletions(-) rename salt/_beacons/{pillar_db.py => postgres_pillar_beacon.py} (86%) rename salt/_beacons/{rules_db.py => rules_beacon.py} (93%) diff --git a/salt/_beacons/pillar_db.py b/salt/_beacons/postgres_pillar_beacon.py similarity index 86% rename from salt/_beacons/pillar_db.py rename to salt/_beacons/postgres_pillar_beacon.py index 8712cce7f..d22eef0ea 100644 --- a/salt/_beacons/pillar_db.py +++ b/salt/_beacons/postgres_pillar_beacon.py @@ -22,7 +22,7 @@ import subprocess log = logging.getLogger(__name__) -WATERMARK_FILE = '/opt/so/state/pillar_db_watch.id' +WATERMARK_FILE = '/opt/so/state/postgres_pillar_beacon_watch.id' CONTAINER = 'so-postgres' DATABASE = 'securityonion' @@ -56,7 +56,7 @@ def _write_watermark(value): f.write(str(int(value))) os.rename(tmp, WATERMARK_FILE) except OSError: - log.exception('pillar_db beacon: failed to persist watermark to %s', WATERMARK_FILE) + log.exception('postgres_pillar_beacon: failed to persist watermark to %s', WATERMARK_FILE) def _query(sql): @@ -71,13 +71,13 @@ def _query(sql): try: result = subprocess.run(cmd, capture_output=True, text=True, timeout=30) except subprocess.TimeoutExpired: - log.warning('pillar_db beacon: psql timed out') + log.warning('postgres_pillar_beacon: psql timed out') return None except Exception: - log.exception('pillar_db beacon: failed to exec psql') + log.exception('postgres_pillar_beacon: failed to exec psql') return None if result.returncode != 0: - log.warning('pillar_db beacon: psql failed (rc=%s): %s', + log.warning('postgres_pillar_beacon: psql failed (rc=%s): %s', result.returncode, (result.stderr or '').strip()) return None return result.stdout @@ -97,7 +97,7 @@ def beacon(config): try: _write_watermark(int((seed or '0').strip() or 0)) except ValueError: - log.warning('pillar_db beacon: could not parse MAX(id) seed: %r', seed) + log.warning('postgres_pillar_beacon: could not parse MAX(id) seed: %r', seed) return retval rows = _query( @@ -116,12 +116,12 @@ def beacon(config): continue parts = line.split(FIELD_SEP) if len(parts) < 3: - log.warning('pillar_db beacon: skipping malformed row: %r', line) + log.warning('postgres_pillar_beacon: skipping malformed row: %r', line) continue try: row_id = int(parts[0]) except ValueError: - log.warning('pillar_db beacon: skipping row with non-int id: %r', line) + log.warning('postgres_pillar_beacon: skipping row with non-int id: %r', line) continue setting_id = parts[1] node_id = parts[2] @@ -136,7 +136,7 @@ def beacon(config): if max_id > watermark: _write_watermark(max_id) - log.info('pillar_db beacon: emitted %d change(s), watermark %d -> %d', + log.info('postgres_pillar_beacon: emitted %d change(s), watermark %d -> %d', len(retval), watermark, max_id) return retval diff --git a/salt/_beacons/rules_db.py b/salt/_beacons/rules_beacon.py similarity index 93% rename from salt/_beacons/rules_db.py rename to salt/_beacons/rules_beacon.py index ab63da431..3bde11c30 100644 --- a/salt/_beacons/rules_db.py +++ b/salt/_beacons/rules_beacon.py @@ -19,7 +19,7 @@ # up on the next one). # # Each emitted event carries the watched directory path under the configured tag -# (e.g. salt/beacon//rules_db/suricata); the push_suricata / push_strelka +# (e.g. salt/beacon//rules_beacon/suricata); the push_suricata / push_strelka # reactors write a push intent, after which the existing so-push-drainer / # orch.push_batch pipeline takes over unchanged. @@ -95,7 +95,7 @@ def _fingerprint(directory): def _watermark_file(tag): - return os.path.join(WATERMARK_DIR, 'rules_db_%s.hash' % tag) + return os.path.join(WATERMARK_DIR, 'rules_beacon_%s.hash' % tag) def _read_watermark(tag): @@ -115,7 +115,7 @@ def _write_watermark(tag, digest): f.write(digest) os.rename(tmp, path) except OSError: - log.exception('rules_db beacon: failed to persist watermark to %s', path) + log.exception('rules_beacon: failed to persist watermark to %s', path) def beacon(config): @@ -134,6 +134,6 @@ def beacon(config): if digest != previous: _write_watermark(tag, digest) retval.append({'tag': tag, 'path': directory}) - log.info('rules_db beacon: change detected in %s, emitting %s', directory, tag) + log.info('rules_beacon: change detected in %s, emitting %s', directory, tag) return retval diff --git a/salt/manager/files/beacons_pushstate.conf.jinja b/salt/manager/files/beacons_pushstate.conf.jinja index 2c00163e3..fba53b759 100644 --- a/salt/manager/files/beacons_pushstate.conf.jinja +++ b/salt/manager/files/beacons_pushstate.conf.jinja @@ -1,9 +1,9 @@ {% from 'salt/auto_apply.map.jinja' import AUTOAPPLY %} beacons: - pillar_db: + postgres_pillar_beacon: - interval: {{ AUTOAPPLY.drain_interval }} - disable_during_state_run: False - rules_db: + rules_beacon: - interval: {{ AUTOAPPLY.drain_interval }} - disable_during_state_run: False - paths: diff --git a/salt/reactor/push_pillar.sls b/salt/reactor/push_pillar.sls index c4b82959b..8d28ef5cf 100644 --- a/salt/reactor/push_pillar.sls +++ b/salt/reactor/push_pillar.sls @@ -1,7 +1,7 @@ #!py -# Reactor invoked by the pillar_db beacon when SOC records settings changes in -# the securityonion.audit_settings table (see salt/_beacons/pillar_db.py). The beacon +# Reactor invoked by the postgres_pillar_beacon when SOC records settings changes in +# the securityonion.audit_settings table (see salt/_beacons/postgres_pillar_beacon.py). The beacon # emits one event per new row carrying setting_id and node_id. # # Two branches, keyed on node_id: @@ -134,7 +134,7 @@ def run(): LOG.info('push_pillar: push disabled, skipping') return {} - # The pillar_db beacon nests its payload under data['data']; fall back to the + # The postgres_pillar_beacon nests its payload under data['data']; fall back to the # top level so the reactor is robust to either shape. event = data.get('data', data) # noqa: F821 -- data provided by reactor setting_id = event.get('setting_id', '') diff --git a/salt/reactor/push_strelka.sls b/salt/reactor/push_strelka.sls index d1d0207eb..52e3fd3ef 100644 --- a/salt/reactor/push_strelka.sls +++ b/salt/reactor/push_strelka.sls @@ -1,6 +1,6 @@ #!py -# Reactor invoked by the rules_db poll beacon (salt/_beacons/rules_db.py) on rule +# Reactor invoked by the rules_beacon poll beacon (salt/_beacons/rules_beacon.py) on rule # file changes under /opt/so/saltstack/local/salt/strelka/rules/compiled/. # # Writes (or updates) a push intent at /opt/so/state/push_pending/rules_strelka.json diff --git a/salt/reactor/push_suricata.sls b/salt/reactor/push_suricata.sls index f50a92527..cce95fdb7 100644 --- a/salt/reactor/push_suricata.sls +++ b/salt/reactor/push_suricata.sls @@ -1,6 +1,6 @@ #!py -# Reactor invoked by the rules_db poll beacon (salt/_beacons/rules_db.py) on rule +# Reactor invoked by the rules_beacon poll beacon (salt/_beacons/rules_beacon.py) on rule # file changes under /opt/so/saltstack/local/salt/suricata/rules/. # # Writes (or updates) a push intent at /opt/so/state/push_pending/rules_suricata.json diff --git a/salt/salt/files/reactor_pushstate.conf b/salt/salt/files/reactor_pushstate.conf index b4543b1a7..dc23e68e4 100644 --- a/salt/salt/files/reactor_pushstate.conf +++ b/salt/salt/files/reactor_pushstate.conf @@ -1,7 +1,7 @@ reactor: - - 'salt/beacon/*/rules_db/suricata': + - 'salt/beacon/*/rules_beacon/suricata': - salt://reactor/push_suricata.sls - - 'salt/beacon/*/rules_db/strelka': + - 'salt/beacon/*/rules_beacon/strelka': - salt://reactor/push_strelka.sls - - 'salt/beacon/*/pillar_db/audit_settings': + - 'salt/beacon/*/postgres_pillar_beacon/audit_settings': - salt://reactor/push_pillar.sls From 52574e21c67bdb7b3bdf75f627df47003158f65a Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 30 Jun 2026 09:40:23 -0400 Subject: [PATCH 110/219] suricata: treat in-progress rule reload as success so-suricata-reload-rules failed the surirulereload state when a rule reload was already running: suricatasc returns {"message":"Reload already in progress","return":"NOK"}, which never matched the expected output, so retry looped all 60 attempts (~3 min) and called fail. Wrap the suricatasc calls so an in-progress reload is treated as success (the in-flight reload picks up the new rules) while genuine container-not-ready conditions still retry and ultimately fail. --- .../tools/sbin/so-suricata-reload-rules | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/salt/suricata/tools/sbin/so-suricata-reload-rules b/salt/suricata/tools/sbin/so-suricata-reload-rules index e21e28e2f..aec6bc966 100644 --- a/salt/suricata/tools/sbin/so-suricata-reload-rules +++ b/salt/suricata/tools/sbin/so-suricata-reload-rules @@ -7,5 +7,19 @@ . /usr/sbin/so-common -retry 60 3 'docker exec so-suricata /opt/suricata/bin/suricatasc -c reload-rules /var/run/suricata/suricata-command.socket' '{"message":"done","return":"OK"}' || fail "The Suricata container was not ready in time." -retry 60 3 'docker exec so-suricata /opt/suricata/bin/suricatasc -c ruleset-reload-nonblocking /var/run/suricata/suricata-command.socket' '{"message":"done","return":"OK"}' || fail "The Suricata container was not ready in time." +reload_suricata_rules() { + # $1 = suricatasc command (reload-rules | ruleset-reload-nonblocking) + local output + output=$(docker exec so-suricata /opt/suricata/bin/suricatasc -c "$1" /var/run/suricata/suricata-command.socket) + echo "$output" + # A reload already running is fine — the new rules get picked up by it. + if [[ "$output" =~ "Reload already in progress" ]]; then + echo "A rule reload is already in progress; treating as success." + return 0 + fi + [[ "$output" =~ '{"message":"done","return":"OK"}' ]] && return 0 + return 1 +} + +retry 60 3 'reload_suricata_rules reload-rules' || fail "The Suricata container was not ready in time." +retry 60 3 'reload_suricata_rules ruleset-reload-nonblocking' || fail "The Suricata container was not ready in time." From 3b8459c6ec474c20b4e60c07b0577b6dd1fa98c2 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 30 Jun 2026 12:43:42 -0500 Subject: [PATCH 111/219] soup upgrade kafka cluster metadata v4 --- salt/manager/tools/sbin/soup | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 2b8680191..be99292d8 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -850,6 +850,28 @@ kibana_backport_streams_index_template() { } +# Runs kafka-features.sh upgrade --release-version $1 +# Upgrades Kafka KRaft cluster metadata +update_kafka_metadata() { + metadata_version="$1" + global_pillar="/opt/so/saltstack/local/pillar/global/soc_global.sls" + if PIPELINE=$(so-yaml.py get -r "$global_pillar" global.pipeline 2> /dev/null) && [[ "$PIPELINE" == "KAFKA" ]]; then + kafka_nodes_raw=$(salt-call pillar.get kafka:nodes --out=json) + if kafka_nodes=$(jq -er '.local | select(type == "object" and length > 0)' <<< "$kafka_nodes_raw"); then + bootstrap_servers=$(jq -r '[to_entries[] | select(.value.role | contains("broker")) | "\(.value.ip):9092"] | join(",")' <<< "$kafka_nodes") + echo "Upgrading Kafka KRaft cluster version" + so-kafka-cli kafka-features.sh --bootstrap-server "$bootstrap_servers" --command-config /opt/kafka/config/kraft/client.properties upgrade --release-version "$metadata_version" 2>/dev/null || true + + return 0 + else + FINAL_MESSAGE_QUEUE+=("WARNING: Unable to automatically perform Kafka Kraft cluster metadata update. This step can be performed manually using the following command (replacing \$BROKER_IP with the ip of atleast 1 available Kafka broker):") + FINAL_MESSAGE_QUEUE+=(" - so-kafka-cli kafka-features.sh --bootstrap-server \$BROKER_IP:9092 --command-config /opt/kafka/config/kraft/client.properties upgrade --release-version $metadata_version") + fi + else + echo "Nothing to do!" + fi +} + up_to_3.2.0() { fix_logstash_0013_lumberjack_pipeline_name @@ -867,6 +889,8 @@ post_to_3.2.0() { kibana_backport_streams_index_template + update_kafka_metadata "4.3" + POSTVERSION=3.2.0 } From 670d2b2757a7cd808d22aa05f6605c59a8b994a6 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 30 Jun 2026 12:57:56 -0500 Subject: [PATCH 112/219] casing --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index be99292d8..3c4cbe7c4 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -864,7 +864,7 @@ update_kafka_metadata() { return 0 else - FINAL_MESSAGE_QUEUE+=("WARNING: Unable to automatically perform Kafka Kraft cluster metadata update. This step can be performed manually using the following command (replacing \$BROKER_IP with the ip of atleast 1 available Kafka broker):") + FINAL_MESSAGE_QUEUE+=("WARNING: Unable to automatically perform Kafka KRaft cluster metadata update. This step can be performed manually using the following command (replacing \$BROKER_IP with the ip of atleast 1 available Kafka broker):") FINAL_MESSAGE_QUEUE+=(" - so-kafka-cli kafka-features.sh --bootstrap-server \$BROKER_IP:9092 --command-config /opt/kafka/config/kraft/client.properties upgrade --release-version $metadata_version") fi else From 9217670bab8319d5fa9487e5b853c0425935c3a7 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 30 Jun 2026 16:21:01 -0400 Subject: [PATCH 113/219] support sigma playbooks --- salt/soc/config.sls | 16 ++++++ salt/soc/defaults.yaml | 4 +- salt/soc/enabled.sls | 3 ++ .../files/soc/playbook_placeholder_map.yaml | 49 ++++++++++++++++++ .../files/soc/sigma_playbook_pipeline.yaml | 12 +++++ salt/soc/files/soc/sigma_so_pipeline.yaml | 51 +++++++++++++++++++ 6 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 salt/soc/files/soc/playbook_placeholder_map.yaml create mode 100644 salt/soc/files/soc/sigma_playbook_pipeline.yaml diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 7e2beefa0..14fdd6288 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -134,6 +134,22 @@ socsigmasopipeline: - group: 939 - mode: 600 +socsigmaplaybookpipeline: + file.managed: + - name: /opt/so/conf/soc/sigma_playbook_pipeline.yaml + - source: salt://soc/files/soc/sigma_playbook_pipeline.yaml + - user: 939 + - group: 939 + - mode: 600 + +socplaybookplaceholdermap: + file.managed: + - name: /opt/so/conf/soc/playbook_placeholder_map.yaml + - source: salt://soc/files/soc/playbook_placeholder_map.yaml + - user: 939 + - group: 939 + - mode: 600 + socbanner: file.managed: - name: /opt/so/conf/soc/banner.md diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 7e8e76094..58c689c72 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1499,9 +1499,9 @@ soc: playbookRepoPath: /opt/sensoroni/playbooks/ playbookRepos: default: - - repo: https://github.com/Security-Onion-Solutions/securityonion-resources-playbooks + - repo: https://github.com/defensivedepth/HCIP-Sigma branch: main - folder: securityonion-normalized + folder: playbooks airgap: - repo: file:///nsm/airgap-resources/playbooks/securityonion-resources-playbooks branch: main diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 1805bacaf..bdd6d25d7 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -45,7 +45,9 @@ so-soc: - /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro - /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro - /opt/so/conf/soc/sigma_so_pipeline.yaml:/opt/sensoroni/sigma_so_pipeline.yaml:ro + - /opt/so/conf/soc/sigma_playbook_pipeline.yaml:/opt/sensoroni/sigma_playbook_pipeline.yaml:ro - /opt/so/conf/soc/sigma_final_pipeline.yaml:/opt/sensoroni/sigma_final_pipeline.yaml:rw + - /opt/so/conf/soc/playbook_placeholder_map.yaml:/opt/sensoroni/playbook_placeholder_map.yaml:ro - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro - /opt/so/conf/soc/custom_roles:/opt/sensoroni/rbac/custom_roles:ro - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw @@ -99,6 +101,7 @@ so-soc: - file: soccustomroles - file: socusersroles - file: socclientsroles + - file: socplaybookplaceholdermap delete_so-soc_so-status.disabled: file.uncomment: diff --git a/salt/soc/files/soc/playbook_placeholder_map.yaml b/salt/soc/files/soc/playbook_placeholder_map.yaml new file mode 100644 index 000000000..582f052dc --- /dev/null +++ b/salt/soc/files/soc/playbook_placeholder_map.yaml @@ -0,0 +1,49 @@ +# Global playbook placeholder map: %token% -> event field path. +# +# Loaded by the SOC Playbook module and used to resolve `field|expand:%placeholder%` values +# from an alert when converting playbook questions to OQL. +# Left: the %token% used in a question +# Right: the event field its value is read from (event_data.-nested or bare; the module +# tries both). +# +# Example: with `src_ip: source.ip` (below), a question that writes +# `source.ip|expand: '%src_ip%'` resolves %src_ip% to the alert's source.ip at convert time. +# +# This is the base layer - a playbook repo can extend it with a co-located *.placeholders.yaml +# config file that overlays these for that repo's playbooks. + +CommandLine: process.command_line +CurrentDirectory: process.working_directory +Image: process.executable +ImageLoaded: dll.name +ParentImage: process.parent.executable +ParentName: process.parent.name +ParentProcessGuid: process.parent.entity_id +ProcessGuid: process.entity_id +TargetFilename: file.name +TargetObject: registry.path +TargetUserName: user.target.name +User: user.name +community_id: network.community_id +dns_resolved_ip: dns.resolved_ip +document_id: soc_id +dst_ip: destination.ip +dst_port: destination.port +event_data_source_ip: source.ip +file_path: file.path +file_dirs: process.file_dirs +file_name: process.name +file_paths: process.file_paths +hostname: host.name +private_ip: network.private_ip +public_ip: network.public_ip +related_hosts: related.hosts +related_ip: related.ip +src_ip: source.ip +dns_query_name: dns.query_name +flow_id: log.id.uid +payload: network.data.decoded +rule_category: rule.category +rule_name: rule.name +rule_uuid: rule.uuid +src_port: source.port diff --git a/salt/soc/files/soc/sigma_playbook_pipeline.yaml b/salt/soc/files/soc/sigma_playbook_pipeline.yaml new file mode 100644 index 000000000..6fcdedbb4 --- /dev/null +++ b/salt/soc/files/soc/sigma_playbook_pipeline.yaml @@ -0,0 +1,12 @@ +name: Security Onion - Playbook Pipeline +priority: 97 +transformations: + # Route string fields to their lowercase-normalized .caseless subfield so wildcard + # matches are case-insensitive. + - id: case_insensitive_string_fields + type: field_name_mapping + mapping: + process.executable: process.executable.caseless + process.parent.executable: process.parent.executable.caseless + process.command_line: process.command_line.caseless + process.parent.command_line: process.parent.command_line.caseless diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index b5bc96dc4..dee93c84e 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -63,6 +63,14 @@ transformations: rule_conditions: - type: logsource category: antivirus + # OS-agnostic process_creation scoping for product-less (NIDS/host-pivot) rules. + - id: process_creation_os_agnostic + type: add_condition + conditions: + event.category: process + rule_conditions: + - type: logsource + category: process_creation # Transforms the `Hashes` field to ECS fields # ECS fields are used by the hash fields emitted by Elastic Defend # If shipped with Elastic Agent, sysmon logs will also have hashes mapped to ECS fields @@ -108,6 +116,40 @@ transformations: - type: logsource product: windows category: driver_load + - id: ecs_fix_process_creation + type: field_name_mapping + mapping: + # bare `Hashes` (the combined-string case is broken out above) + winlog.event_data.Hashes: process.hash.sha256 + winlog.event_data.IntegrityLevel: process.Ext.token.integrity_level_name + winlog.event_data.ParentName: process.parent.name + rule_conditions: + - type: logsource + product: windows + category: process_creation + - id: ecs_fix_registry_set + type: field_name_mapping + mapping: + winlog.event_data.Details: registry.data.strings + # field rename only; EventType values (SetValue/CreateKey) still differ from + # event.action values (modification/creation) + winlog.event_data.EventType: event.action + rule_conditions: + - type: logsource + product: windows + category: registry_set + - id: ecs_fix_image_load + type: field_name_mapping + mapping: + file.path: dll.path + file.code_signature.signed: dll.code_signature.exists + winlog.event_data.Signature: dll.code_signature.subject_name + file.code_signature.status: dll.code_signature.status + winlog.event_data.Hashes: dll.hash.sha256 + rule_conditions: + - type: logsource + product: windows + category: image_load - id: linux_security_add-fields type: add_condition conditions: @@ -281,6 +323,15 @@ transformations: rule_conditions: - type: logsource category: file_event + # Scope image_load rules to Elastic Endpoint library events (event.category:library, dll.* + # populated). + - id: endpoint_image_load_add-fields + type: add_condition + conditions: + event.category: 'library' + rule_conditions: + - type: logsource + category: image_load # Maps network rules to all network logs # This targets all network logs, all services, generated from endpoints and network - id: network_add-fields From ee36f5f84c7ea9c6d3ab512377f732883fe6c3b4 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 1 Jul 2026 09:00:36 -0400 Subject: [PATCH 114/219] suricata: verify reloaded ruleset is newer than the rules file Treating an in-progress reload as instant success could report success while Suricata was still running a stale ruleset (the in-flight reload may have started before the new all-rulesets.rules was written). Make success conditional on Suricata actually having loaded the current ruleset: capture the rules-file mtime up front, trigger a blocking reload-rules, then query ruleset-reload-time and only succeed when last_reload >= mtime. An in-progress reload now retries (waits for it to clear so our own fresh reload runs) instead of short-circuiting, and a ruleset that never catches up within the retry window fails via fail(). Also drop the redundant ruleset-reload-nonblocking call (the verified blocking reload is authoritative and the async call was what left a reload running) and log human-readable timestamps. --- .../tools/sbin/so-suricata-reload-rules | 53 +++++++++++++++---- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/salt/suricata/tools/sbin/so-suricata-reload-rules b/salt/suricata/tools/sbin/so-suricata-reload-rules index aec6bc966..b966e4bc0 100644 --- a/salt/suricata/tools/sbin/so-suricata-reload-rules +++ b/salt/suricata/tools/sbin/so-suricata-reload-rules @@ -7,19 +7,50 @@ . /usr/sbin/so-common -reload_suricata_rules() { - # $1 = suricatasc command (reload-rules | ruleset-reload-nonblocking) - local output - output=$(docker exec so-suricata /opt/suricata/bin/suricatasc -c "$1" /var/run/suricata/suricata-command.socket) - echo "$output" - # A reload already running is fine — the new rules get picked up by it. - if [[ "$output" =~ "Reload already in progress" ]]; then - echo "A rule reload is already in progress; treating as success." +RULES_FILE="/opt/so/rules/suricata/all-rulesets.rules" +SOCKET="/var/run/suricata/suricata-command.socket" +SURICATASC="docker exec so-suricata /opt/suricata/bin/suricatasc" + +# Epoch mtime of the ruleset we need Suricata to have loaded. Captured once so a +# file update mid-reload does not move the goalpost. +target_mtime=$(stat -c %Y "$RULES_FILE") || fail "Could not stat the Suricata rules file: $RULES_FILE" + +# Format an epoch as a human-readable local timestamp for log messages. +fmt_time() { date -d "@$1" '+%Y-%m-%d %H:%M:%S %Z' 2>/dev/null; } + +# Epoch of Suricata's last *completed* ruleset reload; non-zero return on failure. +suricata_reload_epoch() { + local out ts + out=$($SURICATASC -c ruleset-reload-time "$SOCKET" 2>/dev/null) + ts=$(echo "$out" | jq -r '.message[0].last_reload // empty' 2>/dev/null) + [ -n "$ts" ] || return 1 + date -d "$ts" +%s 2>/dev/null +} + +# Trigger a fresh reload and confirm Suricata is running a ruleset at least as new +# as the rules file. Returns 0 only when both hold, so retry keeps going until an +# in-progress reload clears and our own reload completes. +reload_and_verify() { + local out reload_epoch + out=$($SURICATASC -c reload-rules "$SOCKET") + echo "reload-rules: $out" + + if [[ "$out" =~ "Reload already in progress" ]]; then + echo "A reload is already in progress; waiting for it to clear so a fresh reload can load the current ruleset." + return 1 + fi + if [[ ! "$out" =~ '{"message":"done","return":"OK"}' ]]; then + echo "Suricata not ready or unexpected reload output; will retry." + return 1 + fi + + reload_epoch=$(suricata_reload_epoch) || { echo "Could not read ruleset-reload-time; will retry."; return 1; } + if [ "$reload_epoch" -ge "$target_mtime" ]; then + echo "Loaded ruleset is current: last reload ($(fmt_time "$reload_epoch")) is newer than rules file ($(fmt_time "$target_mtime"))." return 0 fi - [[ "$output" =~ '{"message":"done","return":"OK"}' ]] && return 0 + echo "Loaded ruleset is stale: last reload ($(fmt_time "$reload_epoch")) is older than rules file ($(fmt_time "$target_mtime")); retrying." return 1 } -retry 60 3 'reload_suricata_rules reload-rules' || fail "The Suricata container was not ready in time." -retry 60 3 'reload_suricata_rules ruleset-reload-nonblocking' || fail "The Suricata container was not ready in time." +retry 60 3 'reload_and_verify' || fail "Suricata did not load the current ruleset in time." From 2a6cc58306d91ebf5c7b1b44c6958f5d793924fe Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 1 Jul 2026 09:07:02 -0400 Subject: [PATCH 115/219] Simplify mappings --- salt/soc/config.sls | 8 ++++++++ salt/soc/enabled.sls | 2 ++ salt/soc/files/soc/playbook_placeholder_map.yaml | 6 +++--- .../files/soc/playbook_placeholder_map_custom.yaml | 14 ++++++++++++++ salt/soc/soc_soc.yaml | 10 +++++++++- 5 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 salt/soc/files/soc/playbook_placeholder_map_custom.yaml diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 14fdd6288..3609b7024 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -150,6 +150,14 @@ socplaybookplaceholdermap: - group: 939 - mode: 600 +socplaybookplaceholdermapcustom: + file.managed: + - name: /opt/so/conf/soc/playbook_placeholder_map_custom.yaml + - source: salt://soc/files/soc/playbook_placeholder_map_custom.yaml + - user: 939 + - group: 939 + - mode: 600 + socbanner: file.managed: - name: /opt/so/conf/soc/banner.md diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index bdd6d25d7..d84e24dd8 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -48,6 +48,7 @@ so-soc: - /opt/so/conf/soc/sigma_playbook_pipeline.yaml:/opt/sensoroni/sigma_playbook_pipeline.yaml:ro - /opt/so/conf/soc/sigma_final_pipeline.yaml:/opt/sensoroni/sigma_final_pipeline.yaml:rw - /opt/so/conf/soc/playbook_placeholder_map.yaml:/opt/sensoroni/playbook_placeholder_map.yaml:ro + - /opt/so/conf/soc/playbook_placeholder_map_custom.yaml:/opt/sensoroni/playbook_placeholder_map_custom.yaml:rw - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro - /opt/so/conf/soc/custom_roles:/opt/sensoroni/rbac/custom_roles:ro - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw @@ -102,6 +103,7 @@ so-soc: - file: socusersroles - file: socclientsroles - file: socplaybookplaceholdermap + - file: socplaybookplaceholdermapcustom delete_so-soc_so-status.disabled: file.uncomment: diff --git a/salt/soc/files/soc/playbook_placeholder_map.yaml b/salt/soc/files/soc/playbook_placeholder_map.yaml index 582f052dc..eeddb4474 100644 --- a/salt/soc/files/soc/playbook_placeholder_map.yaml +++ b/salt/soc/files/soc/playbook_placeholder_map.yaml @@ -1,4 +1,4 @@ -# Global playbook placeholder map: %token% -> event field path. +# Global Playbook placeholder map: %token% -> event field path. # # Loaded by the SOC Playbook module and used to resolve `field|expand:%placeholder%` values # from an alert when converting playbook questions to OQL. @@ -9,8 +9,8 @@ # Example: with `src_ip: source.ip` (below), a question that writes # `source.ip|expand: '%src_ip%'` resolves %src_ip% to the alert's source.ip at convert time. # -# This is the base layer - a playbook repo can extend it with a co-located *.placeholders.yaml -# config file that overlays these for that repo's playbooks. +# This is the global base layer. To add or override tokens edit playbook_placeholder_map_custom.yaml. +# those entries overlay this map and win on conflict. CommandLine: process.command_line CurrentDirectory: process.working_directory diff --git a/salt/soc/files/soc/playbook_placeholder_map_custom.yaml b/salt/soc/files/soc/playbook_placeholder_map_custom.yaml new file mode 100644 index 000000000..0d586db18 --- /dev/null +++ b/salt/soc/files/soc/playbook_placeholder_map_custom.yaml @@ -0,0 +1,14 @@ +# Custom Playbook placeholder map: %token% -> event field path. +# +# +# Left: the %token% used in a playbook question. +# Right: the event field its value is read from (event_data.-nested or bare; the module tries +# both). Note: a token that is simply named after a flat event field resolves automatically +# without an entry here - only add a mapping when the token name differs from the field name. +# +# Example: +# +# account_id: cloudflare.account_id +# +# A question that writes +# `account_id|expand: '%account_id%'` resolves %account_id% from the alert at convert time. diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 19853196a..594dbf88c 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -46,7 +46,15 @@ soc: syntax: yaml file: True global: True - advanced: True + advanced: False + helpLink: security-onion-console-customization + playbook_placeholder_map_custom__yaml: + title: Playbook Placeholder Map + description: Custom mappings of Playbook %placeholder% tokens to event fields. + syntax: yaml + file: True + global: True + advanced: False helpLink: security-onion-console-customization config: licenseKey: From dc8c80633b59db2b15d0647c980800103d61b657 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 1 Jul 2026 10:23:04 -0500 Subject: [PATCH 116/219] update airgap soup to sync uek repo from iso and retain latest packages only --- salt/manager/tools/sbin/soup | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 3c4cbe7c4..92d89f5e6 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -245,6 +245,7 @@ check_airgap() { UPDATE_DIR=/tmp/soagupdate/SecurityOnion AGDOCKER=/tmp/soagupdate/docker AGREPO=/tmp/soagupdate/minimal/Packages + AGUEKREPO=/tmp/soagupdate/uek/Packages else is_airgap=1 fi @@ -1004,13 +1005,19 @@ update_airgap_rules() { rsync -a $UPDATE_DIR/agrules/securityonion-resources/* /nsm/securityonion-resources/ } -update_airgap_repo() { +update_airgap_repos() { # Update the files in the repo - echo "Syncing new updates to /nsm/repo" - rsync -a $AGREPO/* /nsm/repo/ - echo "Creating repo" + echo "Syncing new updates to /nsm/repo & /nsm/kernelrepo" + # using --delete to replicate so-repo-sync behavior of keeping only latest packages in local repo + rsync -a --delete "$AGREPO"/ /nsm/repo/ + rsync -a --delete "$AGUEKREPO"/ /nsm/kernelrepo/ + dnf -y install yum-utils createrepo_c + + echo "Running createrepo for /nsm/repo" createrepo /nsm/repo + echo "Running createrepo for /nsm/kernelrepo" + createrepo /nsm/kernelrepo } update_salt_mine() { @@ -1766,7 +1773,7 @@ main() { set -e if [[ $is_airgap -eq 0 ]]; then - update_airgap_repo + update_airgap_repos dnf clean all check_os_updates elif [[ $OS == 'oracle' ]]; then From e88eb65a4400b3d3b5f43bc765b9dc79ce1edb18 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 1 Jul 2026 10:29:05 -0500 Subject: [PATCH 117/219] keep old packages for rollback ability --- salt/manager/tools/sbin/soup | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 92d89f5e6..b84c38087 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -244,8 +244,7 @@ check_airgap() { is_airgap=0 UPDATE_DIR=/tmp/soagupdate/SecurityOnion AGDOCKER=/tmp/soagupdate/docker - AGREPO=/tmp/soagupdate/minimal/Packages - AGUEKREPO=/tmp/soagupdate/uek/Packages + AGREPO=/tmp/soagupdate/minimal/Packages AGUEKREPO=/tmp/soagupdate/uek/Packages else is_airgap=1 fi @@ -1008,9 +1007,9 @@ update_airgap_rules() { update_airgap_repos() { # Update the files in the repo echo "Syncing new updates to /nsm/repo & /nsm/kernelrepo" - # using --delete to replicate so-repo-sync behavior of keeping only latest packages in local repo - rsync -a --delete "$AGREPO"/ /nsm/repo/ - rsync -a --delete "$AGUEKREPO"/ /nsm/kernelrepo/ + # Airgap soup copies new files into the local repo, but doesn't remove old packages. Retaining the ability to rollback package updates + rsync -a "$AGREPO"/ /nsm/repo/ + rsync -a "$AGUEKREPO"/ /nsm/kernelrepo/ dnf -y install yum-utils createrepo_c From c33db9d00fe82162b682532661fed7357b3ac1cd Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 1 Jul 2026 11:12:39 -0500 Subject: [PATCH 118/219] remove outdated eval script and associated salt utility state --- pillar/data/addtotab.sh | 59 ----------------------------------- salt/allowed_states.map.jinja | 3 +- salt/top.sls | 6 ---- salt/utility/bin/eval | 29 ----------------- salt/utility/init.sls | 22 ------------- 5 files changed, 1 insertion(+), 118 deletions(-) delete mode 100644 pillar/data/addtotab.sh delete mode 100644 salt/utility/bin/eval delete mode 100644 salt/utility/init.sls diff --git a/pillar/data/addtotab.sh b/pillar/data/addtotab.sh deleted file mode 100644 index 65f9446dd..000000000 --- a/pillar/data/addtotab.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bash - -# This script adds sensors/nodes/etc to the nodes tab -default_salt_dir=/opt/so/saltstack/default -local_salt_dir=/opt/so/saltstack/local -TYPE=$1 -NAME=$2 -IPADDRESS=$3 -CPUS=$4 -GUID=$5 -MANINT=$6 -ROOTFS=$7 -NSM=$8 -MONINT=$9 -#NODETYPE=$10 -#HOTNAME=$11 - -echo "Seeing if this host is already in here. If so delete it" -if grep -q $NAME "$local_salt_dir/pillar/data/$TYPE.sls"; then - echo "Node Already Present - Let's re-add it" - awk -v blah=" $NAME:" 'BEGIN{ print_flag=1 } -{ - if( $0 ~ blah ) - { - print_flag=0; - next - } - if( $0 ~ /^ [a-zA-Z0-9]+:$/ ) - { - print_flag=1; - } - if ( print_flag == 1 ) - print $0 - -} ' $local_salt_dir/pillar/data/$TYPE.sls > $local_salt_dir/pillar/data/tmp.$TYPE.sls -mv $local_salt_dir/pillar/data/tmp.$TYPE.sls $local_salt_dir/pillar/data/$TYPE.sls -echo "Deleted $NAME from the tab. Now adding it in again with updated info" -fi -echo " $NAME:" >> $local_salt_dir/pillar/data/$TYPE.sls -echo " ip: $IPADDRESS" >> $local_salt_dir/pillar/data/$TYPE.sls -echo " manint: $MANINT" >> $local_salt_dir/pillar/data/$TYPE.sls -echo " totalcpus: $CPUS" >> $local_salt_dir/pillar/data/$TYPE.sls -echo " guid: $GUID" >> $local_salt_dir/pillar/data/$TYPE.sls -echo " rootfs: $ROOTFS" >> $local_salt_dir/pillar/data/$TYPE.sls -echo " nsmfs: $NSM" >> $local_salt_dir/pillar/data/$TYPE.sls -if [ $TYPE == 'sensorstab' ]; then - echo " monint: bond0" >> $local_salt_dir/pillar/data/$TYPE.sls -fi -if [ $TYPE == 'evaltab' ] || [ $TYPE == 'standalonetab' ]; then - echo " monint: bond0" >> $local_salt_dir/pillar/data/$TYPE.sls - if [ ! $10 ]; then - salt-call state.apply utility queue=True - fi -fi -if [ $TYPE == 'nodestab' ]; then - salt-call state.apply elasticsearch queue=True -# echo " nodetype: $NODETYPE" >> $local_salt_dir/pillar/data/$TYPE.sls -# echo " hotname: $HOTNAME" >> $local_salt_dir/pillar/data/$TYPE.sls -fi diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index c831b45fe..72e4bbe82 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -37,8 +37,7 @@ 'elasticfleet', 'elasticfleet.manager', 'elasticsearch.cluster', - 'elastic-fleet-package-registry', - 'utility' + 'elastic-fleet-package-registry' ] %} {% set sensor_states = [ diff --git a/salt/top.sls b/salt/top.sls index cf743edd1..ffa43864c 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -83,7 +83,6 @@ base: - zeek - strelka - elastalert - - utility - elasticfleet - pcap.cleanup @@ -113,7 +112,6 @@ base: - zeek - strelka - elastalert - - utility - elasticfleet - stig - kafka @@ -141,7 +139,6 @@ base: - elastic-fleet-package-registry - kibana - elastalert - - utility - elasticfleet - stig - kafka @@ -168,7 +165,6 @@ base: - elastic-fleet-package-registry - kibana - elastalert - - utility - elasticfleet - kafka @@ -198,7 +194,6 @@ base: - elastic-fleet-package-registry - kibana - elastalert - - utility - elasticfleet - stig - kafka @@ -222,7 +217,6 @@ base: - elasticsearch - elastic-fleet-package-registry - kibana - - utility - suricata - zeek - elasticfleet diff --git a/salt/utility/bin/eval b/salt/utility/bin/eval deleted file mode 100644 index f30f0f421..000000000 --- a/salt/utility/bin/eval +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# Wait for ElasticSearch to come up, so that we can query for version infromation -echo -n "Waiting for ElasticSearch..." -COUNT=0 -ELASTICSEARCH_CONNECTED="no" -while [[ "$COUNT" -le 30 ]]; do - curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://{{ GLOBALS.manager_ip }}:9200 - if [ $? -eq 0 ]; then - ELASTICSEARCH_CONNECTED="yes" - echo "connected!" - break - else - ((COUNT+=1)) - sleep 1 - echo -n "." - fi -done -if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then - echo - echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'docker ps' \n -running 'sudo so-elastic-restart'" - echo - - exit -fi - -echo "Applying cross cluster search config..." - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -XPUT -L https://{{ GLOBALS.manager_ip }}:9200/_cluster/settings \ - -H 'Content-Type: application/json' \ - -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ grains.host }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}" diff --git a/salt/utility/init.sls b/salt/utility/init.sls deleted file mode 100644 index 49bb2cb0c..000000000 --- a/salt/utility/init.sls +++ /dev/null @@ -1,22 +0,0 @@ -{% from 'allowed_states.map.jinja' import allowed_states %} -{% from 'vars/globals.map.jinja' import GLOBALS %} - -{% if sls in allowed_states %} - {% if grains['role'] in ['so-eval', 'so-import'] %} -fixsearch: - cmd.script: - - shell: /bin/bash - - cwd: /opt/so - - source: salt://utility/bin/eval - - template: jinja - - defaults: - GLOBALS: {{ GLOBALS }} - {% endif %} - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} From 868b2175496e45a53b8026bba9066c5d89c5a322 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 1 Jul 2026 11:37:46 -0500 Subject: [PATCH 119/219] update default hunt query --- salt/soc/defaults.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 7e8e76094..ff8a504ac 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1771,13 +1771,13 @@ soc: enabled: true queries: - name: Default Query - description: Show all events grouped by the observer host - query: '* | groupby observer.name' - showSubtitle: true - - name: Log Type description: Show all events grouped by module and dataset query: '* | groupby event.module* event.dataset' showSubtitle: true + - name: Observer + description: Show all events grouped by the observer host + query: '* | groupby observer.name' + showSubtitle: true - name: SOC - Auth description: Users authenticated to SOC grouped by IP address and identity query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby http.request.headers.x-real-ip user.name' From 24b75b4a2ba5ce8f4548bbfd3f72d1c1172d3c36 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 1 Jul 2026 12:50:23 -0500 Subject: [PATCH 120/219] typo --- salt/manager/tools/sbin/soup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index b84c38087..6725cc95c 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -244,7 +244,8 @@ check_airgap() { is_airgap=0 UPDATE_DIR=/tmp/soagupdate/SecurityOnion AGDOCKER=/tmp/soagupdate/docker - AGREPO=/tmp/soagupdate/minimal/Packages AGUEKREPO=/tmp/soagupdate/uek/Packages + AGREPO=/tmp/soagupdate/minimal/Packages + AGUEKREPO=/tmp/soagupdate/uek/Packages else is_airgap=1 fi From 87b9276c79784b892b0e9dd51eeeba952b535734 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 1 Jul 2026 13:19:47 -0500 Subject: [PATCH 121/219] increase wait_for_so-kibana timeout to 10m --- salt/kibana/enabled.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/enabled.sls b/salt/kibana/enabled.sls index 1257f66c6..e8b561754 100644 --- a/salt/kibana/enabled.sls +++ b/salt/kibana/enabled.sls @@ -69,7 +69,7 @@ wait_for_so-kibana: - ssl: True - verify_ssl: False - status: 200 - - wait_for: 300 + - wait_for: 600 - request_interval: 15 - require: - docker_container: so-kibana From 69d77382f1ac1219c92a239d734b3505e5a7a43b Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 1 Jul 2026 15:12:53 -0400 Subject: [PATCH 122/219] suricata: timestamp each line of reload log output Route the reload/verify output (ours plus so-common's retry/fail lines) through a synchronous timestamping pipeline so every line in reload.log is prefixed with a date/time, and preserve the real exit code via PIPESTATUS. --- .../tools/sbin/so-suricata-reload-rules | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/salt/suricata/tools/sbin/so-suricata-reload-rules b/salt/suricata/tools/sbin/so-suricata-reload-rules index b966e4bc0..6db519413 100644 --- a/salt/suricata/tools/sbin/so-suricata-reload-rules +++ b/salt/suricata/tools/sbin/so-suricata-reload-rules @@ -11,13 +11,12 @@ RULES_FILE="/opt/so/rules/suricata/all-rulesets.rules" SOCKET="/var/run/suricata/suricata-command.socket" SURICATASC="docker exec so-suricata /opt/suricata/bin/suricatasc" -# Epoch mtime of the ruleset we need Suricata to have loaded. Captured once so a -# file update mid-reload does not move the goalpost. -target_mtime=$(stat -c %Y "$RULES_FILE") || fail "Could not stat the Suricata rules file: $RULES_FILE" - # Format an epoch as a human-readable local timestamp for log messages. fmt_time() { date -d "@$1" '+%Y-%m-%d %H:%M:%S %Z' 2>/dev/null; } +# Prefix each input line with the current timestamp. +timestamp_lines() { while IFS= read -r line; do printf '%s %s\n' "$(date '+%Y-%m-%d %H:%M:%S %Z')" "$line"; done; } + # Epoch of Suricata's last *completed* ruleset reload; non-zero return on failure. suricata_reload_epoch() { local out ts @@ -53,4 +52,14 @@ reload_and_verify() { return 1 } -retry 60 3 'reload_and_verify' || fail "Suricata did not load the current ruleset in time." +# Run the reload/verify, timestamping every line of output (ours and the +# retry/fail helpers') so reload.log shows when each step ran. The pipeline is +# synchronous, so the log is fully flushed and ordered before we exit; the +# script's real exit code is preserved via PIPESTATUS. +{ + # Epoch mtime of the ruleset we need Suricata to have loaded. Captured once so + # a file update mid-reload does not move the goalpost. + target_mtime=$(stat -c %Y "$RULES_FILE") || fail "Could not stat the Suricata rules file: $RULES_FILE" + retry 60 3 'reload_and_verify' || fail "Suricata did not load the current ruleset in time." +} 2>&1 | timestamp_lines +exit "${PIPESTATUS[0]}" From 795aa898a30edbacb3d79769d23a0c26b6ad7729 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 1 Jul 2026 15:12:54 -0400 Subject: [PATCH 123/219] suricata: only reload rules once the ruleset file exists On a fresh install the surirulesync file.recurse creates .gitkeep before SOC has generated all-rulesets.rules. That change satisfied the surirulereload onchanges requisite, so the reload ran with no ruleset present, failed to stat the file, and reported the state (and install) as failed. Add an onlyif guard so the reload only runs when all-rulesets.rules exists. A .gitkeep-only sync now leaves the state a clean success (onlyif condition false); once SOC writes the ruleset, the reload fires normally. --- salt/suricata/enabled.sls | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index d9d7f32ae..bb31b2c78 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -65,10 +65,11 @@ so-suricata: - file: suriclassifications surirulereload: - cmd.run: + cmd.run: - name: /usr/sbin/so-suricata-reload-rules >> /opt/so/log/suricata/reload.log 2>&1 - - onchanges: + - onchanges: - file: surirulesync + - onlyif: test -f /opt/so/rules/suricata/all-rulesets.rules - require: - docker_container: so-suricata From e7352eb841afe82603d7e0be2f3e4b0e69e7dbaf Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 1 Jul 2026 15:17:55 -0500 Subject: [PATCH 124/219] duplicate repo name in so-repo-sync --- salt/manager/files/repodownload.conf | 4 ++-- salt/manager/tools/sbin/so-repo-sync | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/manager/files/repodownload.conf b/salt/manager/files/repodownload.conf index 67ae4b121..9c9cb5109 100644 --- a/salt/manager/files/repodownload.conf +++ b/salt/manager/files/repodownload.conf @@ -11,8 +11,8 @@ name=Security Onion Repo repo mirrorlist=file:///opt/so/conf/reposync/mirror.txt enabled=1 gpgcheck=1 -[securityonionkernel] -name=Security Onion Repo repo +[securityonionkernelsync] +name=Security Onion Kernel Repo repo mirrorlist=file:///opt/so/conf/reposync/mirror-kernel.txt enabled=1 gpgcheck=1 diff --git a/salt/manager/tools/sbin/so-repo-sync b/salt/manager/tools/sbin/so-repo-sync index 6c1b9d509..d6a290c25 100755 --- a/salt/manager/tools/sbin/so-repo-sync +++ b/salt/manager/tools/sbin/so-repo-sync @@ -17,9 +17,9 @@ createrepo /nsm/repo # The kernel repo section is deployed to repodownload.conf by the manager highstate, which # runs AFTER this script during soup. On the first upgrade to a kernel-aware version the # on-disk config still predates the section, so guard on its presence to avoid dnf's -# "Unknown repo: 'securityonionkernel'" aborting the sync (set -e). The next sync after the +# "Unknown repo: 'securityonionkernelsync'" aborting the sync (set -e). The next sync after the # highstate deploys the section will pick it up. -if grep -q '^\[securityonionkernel\]' /opt/so/conf/reposync/repodownload.conf; then - dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/ +if grep -q '^\[securityonionkernelsync\]' /opt/so/conf/reposync/repodownload.conf; then + dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernelsync --download-metadata -p /nsm/kernelrepo/ createrepo /nsm/kernelrepo fi From 23f04e28665af7569a7e837f5c66db87547a0c21 Mon Sep 17 00:00:00 2001 From: Matthew Wright Date: Mon, 15 Jun 2026 11:40:01 -0400 Subject: [PATCH 125/219] maxSubSessionTokens and maxDelegationDepth config settings --- salt/soc/defaults.yaml | 2 ++ salt/soc/soc_soc.yaml | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 7e8e76094..afb4c37d1 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1509,6 +1509,8 @@ soc: assistant: systemPromptAddendum: "" systemPromptAddendumMaxLength: 50000 + maxSubSessionTokens: 0 + maxDelegationDepth: 0 adapters: - name: SOAI protocol: securityonion_ai_cloud diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 19853196a..cf483c406 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -719,6 +719,16 @@ soc: description: Maximum length of the system prompt addendum. Longer prompts will be truncated. global: True advanced: True + maxSubSessionTokens: + description: Maximum number of output tokens a delegated sub-session may generate across all of its turns. When the budget is reached, the sub-agent is halted and its result is returned to the parent agent. Set to 0 to disable the limit. + global: True + advanced: True + forcedType: int + maxDelegationDepth: + description: Maximum delegation nesting depth for sub-agents. For example, a value of 2 lets the main agent delegate to a sub-agent that may itself delegate one level deeper. Any deeper delegation is refused and the requesting agent continues without it. Set to 0 to disable the limit. + global: True + advanced: True + forcedType: int adapters: description: Configuration for AI adapters used by the Onion AI assistant. Please see documentation for help on which fields are required for which protocols. global: True From 8675296393aa3dd9b5f4f64177602546ec18cf00 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Wed, 1 Jul 2026 15:04:42 -0600 Subject: [PATCH 126/219] More Agentic Fields The big agentic switch, a specific maxDelegationDepth, and the agentMapping dict --- salt/soc/defaults.yaml | 6 +++++- salt/soc/soc_soc.yaml | 11 +++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index afb4c37d1..c751b25f4 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1510,7 +1510,7 @@ soc: systemPromptAddendum: "" systemPromptAddendumMaxLength: 50000 maxSubSessionTokens: 0 - maxDelegationDepth: 0 + maxDelegationDepth: 5 adapters: - name: SOAI protocol: securityonion_ai_cloud @@ -1522,6 +1522,10 @@ soc: serviceAccountJSON: "" serviceAccountLocation: "" healthTimeoutSeconds: 5 + agentic: false + agentMapping: + Orchestrator: sonnet + Hunter: sonnet onionconfig: saltstackDir: /opt/so/saltstack bypassEnabled: false diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index cf483c406..a11f8e0e4 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -767,6 +767,17 @@ soc: label: Health Timeout Seconds required: False forcedType: int + agentic: + description: Indicates if the Assistant Module should operate in agentic mode or not. If true, agents can work together to solve tasks. + global: True + forcedType: bool + agentMapping: + Orchestrator: + description: The initial agent in most agentic conversations. This agent will delegate requests to specialized agents. + global: True + Hunter: + description: This agent is specialized in querying events. + global: True client: assistant: enabled: From 1243a25bd3565b29bc776d5e0024b561816245a1 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 2 Jul 2026 08:59:52 -0400 Subject: [PATCH 127/219] avoid setup failure reason ambiguity --- setup/so-functions | 31 ++++++++++++++++--------------- setup/so-setup | 14 +++++--------- setup/so-verify | 6 +++--- 3 files changed, 24 insertions(+), 27 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 2d5181dc1..c1f8e11f8 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -29,8 +29,12 @@ title() { } fail_setup() { + local err_msg=$1 + if [[ -n "$err_msg" ]]; then + error "$err_msg" + fi error "Setup encountered an unrecoverable failure, exiting" - touch /root/failure + echo "setup incomplete: $err_msg" > /root/failure exit 1 } @@ -697,7 +701,7 @@ compare_main_nic_ip() { EOM [[ -n $TESTING ]] || whiptail --title "$whiptail_title" --msgbox "$message" 11 75 - kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup + kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup "Main IP mismatch" fi else # Setup uses MAINIP, but since we ignore the equality condition when using a VPN @@ -755,8 +759,7 @@ configure_management_bond() { info "Setting up $bond_name management interface with mode $bond_mode" if [[ ${#MBNICS[@]} -eq 0 ]]; then - error "[ERROR] No management bond NICs were selected." - fail_setup + fail_setup "No management bond NICs selected" fi nmcli -t -f NAME con show | grep -Fxq "$bond_name" @@ -913,8 +916,7 @@ detect_os() { is_rpm=true is_supported=true else - info "This OS is not supported. Security Onion requires Oracle Linux 9." - fail_setup + fail_setup "This OS is not supported. Security Onion requires Oracle Linux 9." fi info "Found OS: $OS $OSVER" @@ -922,7 +924,7 @@ detect_os() { download_elastic_agent_artifacts() { if ! update_elastic_agent 2>&1 | tee -a "$setup_log"; then - fail_setup + fail_setup "Failed to update Elastic Agent" fi } @@ -1566,7 +1568,7 @@ proxy_validate() { error "Received error: $proxy_test_err" if [[ -n $TESTING ]]; then error "Exiting setup" - kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup + kill -SIGINT "$(ps --pid $$ -oppid=)"; fail_setup "Proxy validation failed" fi fi return $ret @@ -1773,8 +1775,7 @@ ensure_pyyaml() { local result=$? set +o pipefail if [[ $result -ne 0 ]] || ! rpm -q python3-pyyaml >/dev/null 2>&1; then - error "Failed to install python3-pyyaml (exit=$result)" - fail_setup + fail_setup "Failed to install python3-pyyaml (exit=$result)" fi info "python3-pyyaml installed successfully" } @@ -1871,7 +1872,7 @@ repo_sync_local() { if [[ ! $is_airgap ]]; then curl --retry 5 --retry-delay 60 -A "netinstall/$SOVERSION/$OS/$(uname -r)/1" https://sigs.securityonion.net/checkup --output /tmp/install - retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" >> "$setup_log" 2>&1 || fail_setup + retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" >> "$setup_log" 2>&1 || fail_setup "Repo sync failed" # After the download is complete run createrepo create_repo fi @@ -1884,10 +1885,10 @@ saltify() { if [[ $waitforstate ]]; then # install all for a manager - retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -M -X stable $SALTVERSION" || fail_setup + retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -M -X stable $SALTVERSION" || fail_setup "Failed to install salt master" else # just a minion - retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -X stable $SALTVERSION" || fail_setup + retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -X stable $SALTVERSION" || fail_setup "Failed to install salt minion" fi salt_install_module_deps @@ -1959,7 +1960,7 @@ set_main_ip() { info "MAINIP=$MAINIP" info "MNIC_IP=$MNIC_IP" whiptail_error_message "The management IP could not be determined. Please check the log at /root/sosetup.log and verify the network configuration. Select OK to exit." - fail_setup + fail_setup "Could not determine MAINIP or MNIC_IP" fi sleep 1 done @@ -2163,7 +2164,7 @@ set_initial_firewall_access() { set_management_interface() { title "Setting up the main interface" if [[ $MNIC == "bond1" ]]; then - configure_management_bond || fail_setup + configure_management_bond || fail_setup "Failed to configure management bond" fi if [ "$address_type" = 'DHCP' ]; then diff --git a/setup/so-setup b/setup/so-setup index 4886d56ed..e4b74716b 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -87,8 +87,7 @@ if [[ "$setup_type" == 'iso' ]]; then if [[ $is_rpm ]]; then is_iso=true else - echo "Only use 'so-setup iso' for an ISO install on Security Onion ISO images. Please run 'so-setup network' instead." - fail_setup + fail_setup "Only use 'so-setup iso' for an ISO install on Security Onion ISO images. Please run 'so-setup network' instead." fi fi @@ -127,7 +126,7 @@ catch() { info "Fatal error occurred at $1 in so-setup, failing setup." grep --color=never "ERROR" "$setup_log" > "$error_log" whiptail_setup_failed - fail_setup + fail_setup "Fatal error occurred at $1 in so-setup" } # Add the progress function for manager node type installs @@ -235,8 +234,7 @@ case "$setup_type" in info "Beginning Security Onion $setup_type install" ;; *) - error "Invalid install type, must be 'iso', 'network' or 'desktop'." - fail_setup + fail_setup "Invalid install type, must be 'iso', 'network' or 'desktop'." ;; esac @@ -770,8 +768,7 @@ if ! [[ -f $install_opt_file ]]; then logCmd "salt-call state.apply -l info registry" title "Seeding the docker registry" if ! docker_seed_registry; then - error "Failed to seed the docker registry" - fail_setup + fail_setup "Failed to seed the docker registry" fi title "Applying the manager state" logCmd "salt-call state.apply -l info manager" @@ -794,8 +791,7 @@ if ! [[ -f $install_opt_file ]]; then title "Setting up Elastic Fleet" logCmd "salt-call state.apply elasticfleet.config" if ! logCmd so-elastic-fleet-setup; then - error "Failed to run so-elastic-fleet-setup" - fail_setup + fail_setup "Failed to run so-elastic-fleet-setup" fi mark_setup_complete set_initial_firewall_access diff --git a/setup/so-verify b/setup/so-verify index 672ed70cc..660424c72 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -143,15 +143,15 @@ main() { cat $error_log echo "--------------------------" exit_code=1 - touch /root/failure + echo "Found setup errors. Check $error_log for details" > /root/failure elif using_iso && cron_error_in_mail_spool; then echo "WARNING: Unexpected cron job output in mail spool" exit_code=1 - touch /root/failure + echo "Unexpected cron job output found in /var/spool/mail/" > /root/failure elif is_manager_node && status_failed; then echo "WARNING: Containers are not in a healthy state" exit_code=1 - touch /root/failure + echo "Containers are not in a healthy state. Check so-status for details" > /root/failure else echo "Successfully completed setup!" touch /root/success From 83cf1f0793efaf8ee446e216698eebb37bd842cb Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Thu, 2 Jul 2026 10:11:52 -0600 Subject: [PATCH 128/219] New Client Params for Tool Retries --- salt/soc/defaults.yaml | 2 ++ salt/soc/soc_soc.yaml | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index c751b25f4..490a09e3c 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -2695,6 +2695,8 @@ soc: thresholdColorRatioLow: 0.5 thresholdColorRatioMed: 0.75 thresholdColorRatioMax: 1 + toolBusyMaxRetries: 30 + toolBusyRetryDelayMs: 1000 availableModels: - id: sonnet displayName: Claude Sonnet diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index a11f8e0e4..5e171cd43 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -784,6 +784,12 @@ soc: description: Set to true to enable the Onion AI assistant in SOC. global: True forcedType: bool + toolBusyMaxRetries: + description: How many times to retry auto approving a tool while a tool is already running. + global: True + toolBusyRetryDelayMs: + description: How long in milliseconds to wait between each retry when auto approving a tool. + global: True investigationPrompt: description: Prompt given to Onion AI when beginning an investigation. global: True From 1fe7726affc3032fb57c7ded0d982dd48a993c76 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 2 Jul 2026 14:58:48 -0400 Subject: [PATCH 129/219] Changes from feedback --- salt/soc/defaults.yaml | 7 +++++-- salt/soc/enabled.sls | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 58c689c72..e9946d633 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1499,9 +1499,12 @@ soc: playbookRepoPath: /opt/sensoroni/playbooks/ playbookRepos: default: - - repo: https://github.com/defensivedepth/HCIP-Sigma + - repo: https://github.com/Security-Onion-Solutions/securityonion-resources-playbooks branch: main - folder: playbooks + folder: securityonion-normalized + - repo: https://github.com/Security-Onion-Solutions/securityonion-resources-playbooks + branch: published + folder: sigma airgap: - repo: file:///nsm/airgap-resources/playbooks/securityonion-resources-playbooks branch: main diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index d84e24dd8..b907864a2 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -46,9 +46,9 @@ so-soc: - /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro - /opt/so/conf/soc/sigma_so_pipeline.yaml:/opt/sensoroni/sigma_so_pipeline.yaml:ro - /opt/so/conf/soc/sigma_playbook_pipeline.yaml:/opt/sensoroni/sigma_playbook_pipeline.yaml:ro - - /opt/so/conf/soc/sigma_final_pipeline.yaml:/opt/sensoroni/sigma_final_pipeline.yaml:rw + - /opt/so/conf/soc/sigma_final_pipeline.yaml:/opt/sensoroni/sigma_final_pipeline.yaml:ro - /opt/so/conf/soc/playbook_placeholder_map.yaml:/opt/sensoroni/playbook_placeholder_map.yaml:ro - - /opt/so/conf/soc/playbook_placeholder_map_custom.yaml:/opt/sensoroni/playbook_placeholder_map_custom.yaml:rw + - /opt/so/conf/soc/playbook_placeholder_map_custom.yaml:/opt/sensoroni/playbook_placeholder_map_custom.yaml:ro - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro - /opt/so/conf/soc/custom_roles:/opt/sensoroni/rbac/custom_roles:ro - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw From 18212cad0d9d6e5e2d3ab8b0d3eccacae185836c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 6 Jul 2026 17:27:37 -0400 Subject: [PATCH 130/219] Add cluster_health option to telegraf.conf Enable cluster health monitoring in Telegraf configuration. --- salt/telegraf/etc/telegraf.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 02d969ff3..d7cc15a38 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -244,6 +244,7 @@ username = "{{ ES_USER }}" password = "{{ ES_PASS }}" insecure_skip_verify = true + cluster_health = true {%- elif grains['role'] in ['so-searchnode'] %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] From db91ce981d0c74178c27aa4c6b892a27d7725aa0 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 7 Jul 2026 07:49:11 -0400 Subject: [PATCH 131/219] Add repo names --- salt/soc/defaults.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index d041e3318..e95008a96 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1500,13 +1500,16 @@ soc: playbookRepos: default: - repo: https://github.com/Security-Onion-Solutions/securityonion-resources-playbooks + rulesetName: sos-playbook-resources branch: main folder: securityonion-normalized - repo: https://github.com/Security-Onion-Solutions/securityonion-resources-playbooks + rulesetName: sos-published branch: published folder: sigma airgap: - repo: file:///nsm/airgap-resources/playbooks/securityonion-resources-playbooks + rulesetName: sos-resources-ag branch: main folder: securityonion-normalized assistant: From ef83450107ce82935eb6fa46339e36a348160615 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 7 Jul 2026 16:54:06 -0400 Subject: [PATCH 132/219] recollate --- salt/manager/tools/sbin/soup | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 6725cc95c..f0be61546 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -789,7 +789,7 @@ bootstrap_so_soc_database() { # and runs automatically only on a fresh data directory. Hosts upgrading from # 3.1.0 already have /nsm/postgres populated, so the so_soc bootstrap block # added in 3.2 never fires. Re-run the script explicitly; it's idempotent. - echo "Bootstrapping so_soc database via init-db.sh." + echo "Bootstrapping database via init-db.sh." # The postgres image has no USER directive, so `docker exec` defaults to # root, and the container env intentionally omits POSTGRES_USER (the upstream # entrypoint defaults it transiently during first-init only). Recreate both @@ -800,10 +800,10 @@ bootstrap_so_soc_database() { return 0 fi if ! $exec_cmd; then - FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd") + FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the database may not have been bootstrapped. Re-run manually: $exec_cmd") return 0 fi - echo "so_soc bootstrap complete." + echo "Database bootstrap complete." } # Existing grids should keep ILM unless an admin explicitly opts in to DLM. @@ -873,6 +873,16 @@ update_kafka_metadata() { fi } +recollate_postgres() { + for db in postgres securityonion so-telegraf; do + exec_cmd="docker exec so-postgres psql -U postgres $db -c \"reindex database $db; alter database $db refresh collation version;\"" + if ! $exec_cmd; then + FINAL_MESSAGE_QUEUE+=("WARNING: recollating $db database failed. Re-run manually: $exec_cmd") + return 0 + fi + done +} + up_to_3.2.0() { fix_logstash_0013_lumberjack_pipeline_name @@ -884,6 +894,9 @@ up_to_3.2.0() { post_to_3.2.0() { bootstrap_so_soc_database + # Recollate due to image OS rebase + recollate_postgres + # Including agent regen script here since it was missed in post_to_3.1.0 echo "Regenerating Elastic Agent Installers" /sbin/so-elastic-agent-gen-installers From 57b7d59387269ab6a26c45bf7fc5e35cdad57d26 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 7 Jul 2026 16:42:47 -0500 Subject: [PATCH 133/219] verify elastic-agent reports healthy status before completing installation --- salt/elasticfleet/install_agent_grid.sls | 8 +- .../tools/sbin/so-elastic-agent-install | 75 +++++++++++++++++++ 2 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 salt/elasticfleet/tools/sbin/so-elastic-agent-install diff --git a/salt/elasticfleet/install_agent_grid.sls b/salt/elasticfleet/install_agent_grid.sls index 482af2e1e..431aa6f97 100644 --- a/salt/elasticfleet/install_agent_grid.sls +++ b/salt/elasticfleet/install_agent_grid.sls @@ -21,11 +21,9 @@ pull_agent_installer: run_installer: cmd.run: - - name: ./so-elastic-agent_linux_amd64 -token={{ GRIDNODETOKEN }} -force - - cwd: /opt/so - - retry: - attempts: 3 - interval: 20 + - name: /usr/sbin/so-elastic-agent-install "{{ GRIDNODETOKEN }}" + - require: + - file: pull_agent_installer cleanup_agent_installer: file.absent: diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install new file mode 100644 index 000000000..dda3f1672 --- /dev/null +++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install @@ -0,0 +1,75 @@ +#!/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-elastic-fleet-common + + +# passed in as arg from elasticfleet/install_agent_grid.sls, else pulled from pillar later +GRIDNODETOKEN="$1" + +check_agent_health() { + timeout=180 + interval=10 + start=$SECONDS + + while (( SECONDS - start < timeout )); do + if elastic-agent status 2>/dev/null | grep -A1 'elastic-agent$' | grep -q 'status: (HEALTHY)'; then + return 0 + fi + echo "elastic-agent is not yet healthy. Waiting for ${interval} seconds before checking again..." + sleep "$interval" + done + + echo "elastic-agent did not become healthy within ${timeout} seconds" + return 1 +} + +if command -v elastic-agent >/dev/null 2>&1; then + elastic-agent uninstall -f +fi + +if [[ -z "$GRIDNODETOKEN" ]]; then + noderole=$(so-yaml.py get -r /etc/salt/grains role) + if [[ "$noderole" == "so-heavynode" ]]; then + GRIDNODETOKEN=$(salt-call pillar.get global:fleet_grid_enrollment_token_heavy --out=newline_values_only) + else + GRIDNODETOKEN=$(salt-call pillar.get global:fleet_grid_enrollment_token_general --out=newline_values_only) + fi +fi + +if [[ ! -x /opt/so/so-elastic-agent_linux_amd64 ]]; then + echo "Downloading elastic-agent installer... This could take a while if another Salt job is running." + salt-call state.sls_id pull_agent_installer elasticfleet.install_agent_grid queue=True +fi + +if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then + attempts=0 + cd /opt/so/ || exit 1 + + while [[ $attempts -lt 3 ]]; do + if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force; echo "Verifying elastic-agent health..." && check_agent_health; then + rm -f /opt/so/so-elastic-agent_linux_amd64 + elastic-agent status + + exit 0 + fi + + attempts=$((attempts + 1)) + + if [[ $attempts -lt 3 ]]; then + echo "so-elastic-agent installer failed. Retrying in 20 seconds..." + sleep 20 + fi + done + + echo "so-elastic-agent installer failed after 3 attempts. Exiting." + + exit 1 +else + echo "Unable to locate so-elastic-agent installer. Exiting." + + exit 1 +fi From 70af3cec534dfa2c5122893ce73ed1c9b29632d4 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 7 Jul 2026 16:48:10 -0500 Subject: [PATCH 134/219] avoid using 'failure' until all loops are done so so-verify doesn't flag it --- .../elasticfleet/tools/sbin/so-elastic-agent-install | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install index dda3f1672..fd6bceeff 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-agent-install +++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install @@ -19,11 +19,11 @@ check_agent_health() { if elastic-agent status 2>/dev/null | grep -A1 'elastic-agent$' | grep -q 'status: (HEALTHY)'; then return 0 fi - echo "elastic-agent is not yet healthy. Waiting for ${interval} seconds before checking again..." + echo "The Elastic Agent is not yet healthy. Waiting for ${interval} seconds before checking again..." sleep "$interval" done - echo "elastic-agent did not become healthy within ${timeout} seconds" + echo "The Elastic Agent did not become healthy within ${timeout} seconds" return 1 } @@ -41,7 +41,7 @@ if [[ -z "$GRIDNODETOKEN" ]]; then fi if [[ ! -x /opt/so/so-elastic-agent_linux_amd64 ]]; then - echo "Downloading elastic-agent installer... This could take a while if another Salt job is running." + echo "Downloading so-elastic-agent installer... This could take a while if another Salt job is running." salt-call state.sls_id pull_agent_installer elasticfleet.install_agent_grid queue=True fi @@ -50,7 +50,7 @@ if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then cd /opt/so/ || exit 1 while [[ $attempts -lt 3 ]]; do - if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force; echo "Verifying elastic-agent health..." && check_agent_health; then + if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force; echo "Verifying Elastic Agent health..." && check_agent_health; then rm -f /opt/so/so-elastic-agent_linux_amd64 elastic-agent status @@ -60,12 +60,12 @@ if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then attempts=$((attempts + 1)) if [[ $attempts -lt 3 ]]; then - echo "so-elastic-agent installer failed. Retrying in 20 seconds..." + echo "Unable to verify Elastic Agent health... Retrying in 20 seconds..." sleep 20 fi done - echo "so-elastic-agent installer failed after 3 attempts. Exiting." + echo "The so-elastic-agent installer failed after 3 attempts. Exiting." exit 1 else From 7f6014096b199e4aa38bc36fa17e4433ee9aa20e Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 7 Jul 2026 22:01:27 -0400 Subject: [PATCH 135/219] recollate db --- salt/manager/tools/sbin/soup | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index f0be61546..9264d9dca 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -874,12 +874,9 @@ update_kafka_metadata() { } recollate_postgres() { - for db in postgres securityonion so-telegraf; do - exec_cmd="docker exec so-postgres psql -U postgres $db -c \"reindex database $db; alter database $db refresh collation version;\"" - if ! $exec_cmd; then - FINAL_MESSAGE_QUEUE+=("WARNING: recollating $db database failed. Re-run manually: $exec_cmd") - return 0 - fi + for db in postgres securityonion so_telegraf; do + docker exec so-postgres psql -U postgres $db -c "reindex database $db" + docker exec so-postgres psql -U postgres $db -c "alter database $db refresh collation version" done } From 8a8f2c4a336a103ea90449414b0fa16cb8b2d962 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 7 Jul 2026 22:22:00 -0400 Subject: [PATCH 136/219] change order to recollate first --- salt/manager/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 9264d9dca..56f4c66dd 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -889,11 +889,11 @@ up_to_3.2.0() { } post_to_3.2.0() { - bootstrap_so_soc_database - # Recollate due to image OS rebase recollate_postgres + bootstrap_so_soc_database + # Including agent regen script here since it was missed in post_to_3.1.0 echo "Regenerating Elastic Agent Installers" /sbin/so-elastic-agent-gen-installers From d131d167de8e2b5633b84076a8f9116497185dcd Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 8 Jul 2026 07:00:38 -0400 Subject: [PATCH 137/219] provide explanation text --- salt/manager/tools/sbin/soup | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 56f4c66dd..aee767db7 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -874,10 +874,14 @@ update_kafka_metadata() { } recollate_postgres() { + echo "" + echo "Recollating PostgreSQL databases. The following output may contain warnings about a version mismatch, followed by a note indicating that the collation version has been changed." for db in postgres securityonion so_telegraf; do docker exec so-postgres psql -U postgres $db -c "reindex database $db" docker exec so-postgres psql -U postgres $db -c "alter database $db refresh collation version" done + echo "Recollating PostgreSQL databases complete." + echo "" } up_to_3.2.0() { From f8de176f4bbbde8b8fd5f6aae06b48c93d1709b1 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 8 Jul 2026 07:47:47 -0500 Subject: [PATCH 138/219] uninstall agent on final failed attempt --- .../tools/sbin/so-elastic-agent-install | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install index fd6bceeff..920b17137 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-agent-install +++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install @@ -27,9 +27,13 @@ check_agent_health() { return 1 } -if command -v elastic-agent >/dev/null 2>&1; then - elastic-agent uninstall -f -fi +uninstall_agent() { + if command -v elastic-agent >/dev/null 2>&1; then + elastic-agent uninstall -f + fi +} + +uninstall_agent if [[ -z "$GRIDNODETOKEN" ]]; then noderole=$(so-yaml.py get -r /etc/salt/grains role) @@ -40,6 +44,12 @@ if [[ -z "$GRIDNODETOKEN" ]]; then fi fi +if [[ -z "$GRIDNODETOKEN" ]]; then + echo "Unable to determine Elastic Fleet enrollment token. Exiting." + + exit 1 +fi + if [[ ! -x /opt/so/so-elastic-agent_linux_amd64 ]]; then echo "Downloading so-elastic-agent installer... This could take a while if another Salt job is running." salt-call state.sls_id pull_agent_installer elasticfleet.install_agent_grid queue=True @@ -50,7 +60,7 @@ if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then cd /opt/so/ || exit 1 while [[ $attempts -lt 3 ]]; do - if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force; echo "Verifying Elastic Agent health..." && check_agent_health; then + if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force && echo "Verifying Elastic Agent health..." && check_agent_health; then rm -f /opt/so/so-elastic-agent_linux_amd64 elastic-agent status @@ -65,6 +75,8 @@ if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then fi done + uninstall_agent + rm -f /opt/so/so-elastic-agent_linux_amd64 echo "The so-elastic-agent installer failed after 3 attempts. Exiting." exit 1 From 2a4a7307f70d2b39229b3abb545d80554999a51f Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 8 Jul 2026 07:51:13 -0500 Subject: [PATCH 139/219] uninstall agent after downloading new one and getting gridtoken --- salt/elasticfleet/tools/sbin/so-elastic-agent-install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install index 920b17137..bda8306f0 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-agent-install +++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install @@ -33,8 +33,6 @@ uninstall_agent() { fi } -uninstall_agent - if [[ -z "$GRIDNODETOKEN" ]]; then noderole=$(so-yaml.py get -r /etc/salt/grains role) if [[ "$noderole" == "so-heavynode" ]]; then @@ -55,6 +53,8 @@ if [[ ! -x /opt/so/so-elastic-agent_linux_amd64 ]]; then salt-call state.sls_id pull_agent_installer elasticfleet.install_agent_grid queue=True fi +uninstall_agent + if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then attempts=0 cd /opt/so/ || exit 1 From 85d7f6bebcf406fde9425171b12a1b5f182ac076 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 8 Jul 2026 08:19:15 -0500 Subject: [PATCH 140/219] independently download so-elastic-agent installer --- .../tools/sbin/so-elastic-agent-install | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install index bda8306f0..c6a499e94 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-agent-install +++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install @@ -50,15 +50,23 @@ fi if [[ ! -x /opt/so/so-elastic-agent_linux_amd64 ]]; then echo "Downloading so-elastic-agent installer... This could take a while if another Salt job is running." - salt-call state.sls_id pull_agent_installer elasticfleet.install_agent_grid queue=True -fi -uninstall_agent + # When running outside of elasticfleet/install_agent_grid.sls we need to download the installer independently. + # PYTHONWARNINGS="ignore" to avoid messages like the following when running salt-call: + # '/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/base.py:129: TransportWarning: Unclosed transport! + # File "/bin/salt-call", line 12, in + # sys.exit(salt_call())' + + PYTHONWARNINGS="ignore" salt-call state.single file.managed name=/opt/so/so-elastic-agent_linux_amd64 source=salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64 mode=755 makedirs=True queue=True + +fi if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then attempts=0 cd /opt/so/ || exit 1 + uninstall_agent + while [[ $attempts -lt 3 ]]; do if ./so-elastic-agent_linux_amd64 -token="$GRIDNODETOKEN" -force && echo "Verifying Elastic Agent health..." && check_agent_health; then rm -f /opt/so/so-elastic-agent_linux_amd64 From 1f44e9868102ddabf5fee09d73c45615c53df90a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 8 Jul 2026 14:34:27 -0400 Subject: [PATCH 141/219] restart soc after re-initing db --- salt/manager/tools/sbin/soup | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index aee767db7..e33c27f6f 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -784,6 +784,17 @@ post_to_3.1.0() { ### 3.2.0 Scripts ### +recollate_postgres() { + echo "" + echo "Recollating PostgreSQL databases. The following output may contain warnings about a version mismatch, followed by a note indicating that the collation version has been changed." + for db in postgres securityonion so_telegraf; do + docker exec so-postgres psql -U postgres $db -c "reindex database $db" + docker exec so-postgres psql -U postgres $db -c "alter database $db refresh collation version" + done + echo "Recollating PostgreSQL databases complete." + echo "" +} + bootstrap_so_soc_database() { # init-db.sh is mounted into so-postgres at /docker-entrypoint-initdb.d/init-db.sh # and runs automatically only on a fresh data directory. Hosts upgrading from @@ -804,6 +815,9 @@ bootstrap_so_soc_database() { return 0 fi echo "Database bootstrap complete." + + echo "Restarting so-soc container to pick up database changes" + docker restart so-soc } # Existing grids should keep ILM unless an admin explicitly opts in to DLM. @@ -873,17 +887,6 @@ update_kafka_metadata() { fi } -recollate_postgres() { - echo "" - echo "Recollating PostgreSQL databases. The following output may contain warnings about a version mismatch, followed by a note indicating that the collation version has been changed." - for db in postgres securityonion so_telegraf; do - docker exec so-postgres psql -U postgres $db -c "reindex database $db" - docker exec so-postgres psql -U postgres $db -c "alter database $db refresh collation version" - done - echo "Recollating PostgreSQL databases complete." - echo "" -} - up_to_3.2.0() { fix_logstash_0013_lumberjack_pipeline_name From 5a57bbe4de6bb60e7c55ebfa3b44cdad5bd2c308 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 8 Jul 2026 14:03:31 -0500 Subject: [PATCH 142/219] add missing so-logs-soc annotation --- salt/elasticsearch/soc_elasticsearch.yaml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 46e836c9c..95880bb9d 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -28,14 +28,14 @@ elasticsearch: description: The maximum number of memory map areas a process may use. Elasticsearch uses a mmapfs directory by default to store its indices. The default operating system limits on mmap counts could be too low, which may result in out of memory exceptions. forcedType: int helpLink: elasticsearch - retention: + retention: retention_pct: decription: Total percentage of space used by Elasticsearch for multi node clusters helpLink: elasticsearch global: True config: cluster: - name: + name: description: The name of the Security Onion Elasticsearch cluster, for identification purposes. readonly: True global: True @@ -55,13 +55,13 @@ elasticsearch: forcedType: bool helpLink: elasticsearch watermark: - low: + low: description: The lower percentage of used disk space representing a healthy node. helpLink: elasticsearch - high: + high: description: The higher percentage of used disk space representing an unhealthy node. helpLink: elasticsearch - flood_stage: + flood_stage: description: The max percentage of used disk space that will cause the node to take protective actions, such as blocking incoming events. helpLink: elasticsearch action: @@ -172,11 +172,11 @@ elasticsearch: forcedType: int global: True helpLink: elasticsearch - refresh_interval: + refresh_interval: description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation. global: True helpLink: elasticsearch - number_of_shards: + number_of_shards: description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs. global: True helpLink: elasticsearch @@ -269,7 +269,7 @@ elasticsearch: global: True advanced: True warm: - min_age: + min_age: description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier. regex: ^[0-9]{1,5}d$ forcedType: string @@ -370,7 +370,7 @@ elasticsearch: template: settings: index: - number_of_replicas: + number_of_replicas: description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs. forcedType: int global: True @@ -391,12 +391,12 @@ elasticsearch: global: True advanced: True helpLink: elasticsearch - refresh_interval: + refresh_interval: description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation. global: True advanced: True helpLink: elasticsearch - number_of_shards: + number_of_shards: description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs. global: True advanced: True @@ -645,6 +645,7 @@ elasticsearch: global: True advanced: True helpLink: elasticsearch + so-logs-soc: *dataStreamSettings so-logs-system_x_auth: *dataStreamSettings so-logs-system_x_syslog: *dataStreamSettings so-logs-system_x_system: *dataStreamSettings From c04a30785f06218510b3d37a917ab9e02e21b977 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 8 Jul 2026 16:15:02 -0500 Subject: [PATCH 143/219] make elastic agent state persistent to prevent re-enrollment during soup / reboots --- salt/common/tools/sbin/so-restart | 3 +++ salt/common/tools/sbin/so-stop | 2 ++ salt/elasticfleet/enabled.sls | 10 ++++++++++ 3 files changed, 15 insertions(+) diff --git a/salt/common/tools/sbin/so-restart b/salt/common/tools/sbin/so-restart index 14747d134..b3864d2d1 100755 --- a/salt/common/tools/sbin/so-restart +++ b/salt/common/tools/sbin/so-restart @@ -35,6 +35,9 @@ case $1 in "elastic-fleet"|"elasticfleet") docker_check_running "elastic-fleet" "--stop" docker rm "so-elastic-fleet" 2> /dev/null + # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy + rm -rf /opt/so/conf/elastic-fleet/state + salt-call state.apply elasticfleet queue=True ;; *) diff --git a/salt/common/tools/sbin/so-stop b/salt/common/tools/sbin/so-stop index d036a7b63..9218b44f7 100755 --- a/salt/common/tools/sbin/so-stop +++ b/salt/common/tools/sbin/so-stop @@ -29,6 +29,8 @@ case $1 in "elasticfleet"|"elastic-fleet") docker_check_running "elastic-fleet" "--stop" docker rm "so-elastic-fleet" 2> /dev/null + # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy + rm -rf /opt/so/conf/elastic-fleet/state ;; *) docker_check_running "$1" "--stop" diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 60cf27deb..ea64a4cb1 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -11,6 +11,10 @@ {# This value is generated during node install and stored in minion pillar #} {% set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %} +{# Prevent Elastic Agent from re-enrolling with a new agent.id everytime the container starts up. + - if a fresh enrollment is needed use 'so-stop elasticfleet' + #} +{% set ENROLLED = salt['file.file_exists']('/opt/so/conf/elastic-fleet/state/fleet.enc') %} include: - ca @@ -65,6 +69,7 @@ so-elastic-fleet: - /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro + - /opt/so/conf/elastic-fleet/state:/usr/share/elastic-agent/state - /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs {% if DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %} {% for BIND in DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %} @@ -72,6 +77,7 @@ so-elastic-fleet: {% endfor %} {% endif %} - environment: + {% if not ENROLLED %} - FLEET_SERVER_ENABLE=true - FLEET_URL=https://{{ GLOBALS.hostname }}:8220 - FLEET_SERVER_ELASTICSEARCH_HOST=https://{{ GLOBALS.manager }}:9200 @@ -81,6 +87,9 @@ so-elastic-fleet: - FLEET_SERVER_CERT_KEY=/etc/pki/elasticfleet-server.key - FLEET_CA=/etc/pki/tls/certs/intca.crt - FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt + {% endif %} + - STATE_PATH=/usr/share/elastic-agent/state + - CONFIG_PATH=/usr/share/elastic-agent/state - LOGS_PATH=logs {% if DOCKERMERGED.containers['so-elastic-fleet'].extra_env %} {% for XTRAENV in DOCKERMERGED.containers['so-elastic-fleet'].extra_env %} @@ -99,6 +108,7 @@ so-elastic-fleet: - x509: etc_elasticfleet_crt - require: - file: trusttheca + - file: eastatedir - x509: etc_elasticfleet_key - x509: etc_elasticfleet_crt From 3394e9aab78bb832392682593b1a05c2c45190cf Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 9 Jul 2026 11:29:43 -0500 Subject: [PATCH 144/219] increase agent health timeout and add logging for elastic agent status output --- salt/elasticfleet/tools/sbin/so-elastic-agent-install | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install index c6a499e94..93889c002 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-agent-install +++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install @@ -9,14 +9,19 @@ # passed in as arg from elasticfleet/install_agent_grid.sls, else pulled from pillar later GRIDNODETOKEN="$1" +LOGFILE="/opt/so/SO-Elastic-Agent_Installer_Health.log" check_agent_health() { - timeout=180 + timeout=300 interval=10 start=$SECONDS + truncate -s 0 "$LOGFILE" + while (( SECONDS - start < timeout )); do - if elastic-agent status 2>/dev/null | grep -A1 'elastic-agent$' | grep -q 'status: (HEALTHY)'; then + agent_status=$(elastic-agent status 2>&1) + echo -e "\n$(date)\n$agent_status\n" >> "$LOGFILE" + if echo "$agent_status" | grep -A1 'elastic-agent$' | grep -q 'status: (HEALTHY)'; then return 0 fi echo "The Elastic Agent is not yet healthy. Waiting for ${interval} seconds before checking again..." From 5fd5df54b487e5c81cd5851a736d69efab994fd6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 9 Jul 2026 13:47:50 -0400 Subject: [PATCH 145/219] Install UEK8 in so-kernel-upgrade when no UEK kernel is present The script assumed the UEK8 kernel was already installed and only switched the boot default to it. On a node running the EL9 stock kernel (RHCK 5.14) there is no kernel-uek* package at all, so `dnf update` has nothing to upgrade and UEK8 never lands -- the script just logged "nothing to do" and exited 0. When no 6.x UEK boot entry exists, install the kernel-uek metapackage (it pulls kernel-uek-core plus the module subpackages, including kernel-uek-modules-extra-netfilter) and then proceed with the grubby switch. Fail loudly if securityonionkernel is not an enabled repo, since that assignment is gated on the NIC-pin marker and the salt version match and a silent no-op there is hard to diagnose. Also point DEFAULTKERNEL at kernel-uek-core so later kernel updates stay on the UEK line rather than falling back to RHCK. Still idempotent and still never reboots. --- salt/common/tools/sbin/so-kernel-upgrade | 77 ++++++++++++++++++------ 1 file changed, 59 insertions(+), 18 deletions(-) diff --git a/salt/common/tools/sbin/so-kernel-upgrade b/salt/common/tools/sbin/so-kernel-upgrade index 46d471051..f901dad8d 100755 --- a/salt/common/tools/sbin/so-kernel-upgrade +++ b/salt/common/tools/sbin/so-kernel-upgrade @@ -5,40 +5,81 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. # -# so-kernel-upgrade — switch the boot default to the installed UEK8 (6.x) kernel. +# so-kernel-upgrade — install the UEK8 (6.x) kernel if needed and make it the boot default. # -# Security Onion is moving off the EL9 stock kernel / UEK7 (5.x) onto UEK8 (6.x). -# Installing the kernel-uek-core package adds a UEK8 boot entry but does NOT make it the -# default: kernel-install/grubby only auto-promote a new kernel within the running -# kernel's flavor lineage, and we're crossing from a 5.x kernel to the new 6.x UEK flavor. -# So even with UPDATEDEFAULT=yes and DEFAULTKERNEL=kernel-uek-core the box keeps booting -# the old kernel. This tool finds the newest installed 6.x UEK kernel and makes it the -# GRUB default via grubby so the next boot comes up on UEK8. +# Security Onion is moving off the EL9 stock kernel (RHCK, 5.14) / UEK7 (5.15) onto UEK8 (6.x). +# Two separate things have to happen, and neither is automatic: # -# Idempotent: if the UEK8 kernel is already the default it does nothing. It only sets the -# boot default; it does NOT reboot — the admin reboots the node on their own schedule. +# 1. Install it. A node running RHCK has no kernel-uek* package at all, so 'dnf update' +# never pulls UEK8 in — there is nothing to upgrade. The kernel-uek metapackage has to +# be installed explicitly from the securityonionkernel repo. +# 2. Boot it. Installing kernel-uek-core adds a UEK8 boot entry but does NOT make it the +# default: kernel-install/grubby only auto-promote a new kernel within the running +# kernel's flavor lineage, and we're crossing from a 5.x kernel to the new 6.x UEK +# flavor. So even with UPDATEDEFAULT=yes and DEFAULTKERNEL=kernel-uek-core the box +# keeps booting the old kernel until grubby is told otherwise. +# +# This tool does both, then points DEFAULTKERNEL at kernel-uek-core so later kernel updates +# stay on the UEK line. +# +# Idempotent: an already-installed, already-default UEK8 kernel is left alone. It only sets +# the boot default; it does NOT reboot — the admin reboots the node on their own schedule. + +KERNEL_REPO="securityonionkernel" +KERNEL_PKG="kernel-uek" log() { echo "[so-kernel-upgrade] $*"; } [ "$(id -u)" -eq 0 ] || { log "must run as root"; exit 1; } command -v grubby >/dev/null 2>&1 || { log "grubby not found"; exit 1; } +command -v dnf >/dev/null 2>&1 || { log "dnf not found"; exit 1; } # Newest installed UEK8 (6.x) kernel known to the bootloader. UEK8 vmlinuz paths look like -# /boot/vmlinuz-6.12.0-203.76.7.5.el9uek.x86_64; the 5.x UEK7 and 5.14 RHCK won't match. -target="$(grubby --info=ALL 2>/dev/null \ - | sed -n 's/^kernel="\(.*\)"$/\1/p' \ - | grep -E '/vmlinuz-6\.[0-9]+.*uek' \ - | sort -V | tail -1)" +# /boot/vmlinuz-6.12.0-203.76.7.5.el9uek.x86_64; the 5.15 UEK7 and 5.14 RHCK won't match. +find_uek8() { + grubby --info=ALL 2>/dev/null \ + | sed -n 's/^kernel="\(.*\)"$/\1/p' \ + | grep -E '/vmlinuz-6\.[0-9]+.*uek' \ + | sort -V | tail -1 +} + +target="$(find_uek8)" if [ -z "$target" ]; then - log "no installed 6.x UEK (UEK8) kernel found — confirm the kernel repo is assigned and" - log "'dnf update' has installed kernel-uek-core. Nothing to do." - exit 0 + log "no UEK8 kernel installed — installing $KERNEL_PKG from $KERNEL_REPO" + + # The repo is assigned by the repo.client highstate, and only once NICs are pinned by MAC + # (/opt/so/state/nic_names_pinned) so the kernel swap can't renumber interfaces SO binds + # by name. Without the repo we'd silently pull nothing, so fail loudly instead. + if ! dnf -q repolist --enabled 2>/dev/null | awk '{print $1}' | grep -qx "$KERNEL_REPO"; then + log "ERROR: repo '$KERNEL_REPO' is not enabled on this node." + log "Run a highstate first; it is skipped until /opt/so/state/nic_names_pinned exists" + log "(run so-nic-pin) and this node's salt matches the version this release ships." + exit 1 + fi + + dnf -y install "$KERNEL_PKG" || { log "ERROR: failed to install $KERNEL_PKG"; exit 1; } + + target="$(find_uek8)" + if [ -z "$target" ]; then + log "ERROR: $KERNEL_PKG installed but no 6.x UEK boot entry appeared — check 'grubby --info=ALL'" + exit 1 + fi + log "installed UEK8 kernel: $target" +fi + +# Keep future kernel updates on the UEK line rather than falling back to RHCK. Oracle ships +# this file; only rewrite it when it's actually pointing somewhere else. +if [ -f /etc/sysconfig/kernel ] && ! grep -q '^DEFAULTKERNEL=kernel-uek-core$' /etc/sysconfig/kernel; then + log "setting DEFAULTKERNEL=kernel-uek-core in /etc/sysconfig/kernel" + sed -i 's/^DEFAULTKERNEL=.*/DEFAULTKERNEL=kernel-uek-core/' /etc/sysconfig/kernel fi current="$(grubby --default-kernel 2>/dev/null)" if [ "$current" = "$target" ]; then log "UEK8 kernel is already the boot default: $target" + [ "$(uname -r)" = "$(basename "$target" | sed 's/^vmlinuz-//')" ] \ + || log "REBOOT REQUIRED to start using it (currently running $(uname -r))." exit 0 fi From 40c02b3149c1bb4edf5d30f074c3607f8f52594c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 9 Jul 2026 14:21:08 -0400 Subject: [PATCH 146/219] Make so-kernel-upgrade populate the kernel repo and fail loudly Three stages of the UEK8 path fail silently, and the script only handled the last one: 1. Populate. so-repo-sync runs before the highstate deploys the [securityonionkernel] section into repodownload.conf, so the first kernel-aware soup skips the kernel sync. kernelrepo_init_empty then seeds valid-but-empty repodata, leaving an enabled repo with zero packages. dnf resolves it happily and installs nothing, no error. 2. Install. `dnf install kernel-uek` on a UEK7 node sees kernel-uek 5.15 already installed, prints "Nothing to do" and exits 0 -- so the script sailed past the install and died later with a misleading grubby error. 3. Boot. Already handled: grubby only auto-promotes within the running kernel's flavor lineage, so 5.x -> 6.x UEK never promotes on its own. Add ensure_kernel_repo(), which verifies the repo is enabled (necessary because skip_if_unavailable=1 hides a broken repo) and that it can serve a 6.x kernel-uek. When it cannot, a manager runs so-repo-sync to populate /nsm/kernelrepo and re-checks; a minion cannot fix it and exits non-zero pointing the admin at the manager. Airgap managers bail, since their repo comes from the ISO rather than a sync. Install the explicit UEK8 NEVRA instead of the bare package name so the "Nothing to do" exit-0 case cannot mask a no-op, and pin the repoquery to securityonionkernel so a UEK7 kernel-uek in the main repo is never picked. Still idempotent and still never reboots. --- salt/common/tools/sbin/so-kernel-upgrade | 160 +++++++++++++++++------ 1 file changed, 119 insertions(+), 41 deletions(-) diff --git a/salt/common/tools/sbin/so-kernel-upgrade b/salt/common/tools/sbin/so-kernel-upgrade index f901dad8d..d04ca533c 100755 --- a/salt/common/tools/sbin/so-kernel-upgrade +++ b/salt/common/tools/sbin/so-kernel-upgrade @@ -5,37 +5,56 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. # -# so-kernel-upgrade — install the UEK8 (6.x) kernel if needed and make it the boot default. +# so-kernel-upgrade — install the UEK8 (6.x) kernel and make it the boot default. # -# Security Onion is moving off the EL9 stock kernel (RHCK, 5.14) / UEK7 (5.15) onto UEK8 (6.x). -# Two separate things have to happen, and neither is automatic: +# Security Onion is moving off the EL9 stock kernel (RHCK, 5.14) and UEK7 (5.15) onto UEK8 +# (6.x). Three things have to happen, and none of them are automatic: # -# 1. Install it. A node running RHCK has no kernel-uek* package at all, so 'dnf update' -# never pulls UEK8 in — there is nothing to upgrade. The kernel-uek metapackage has to -# be installed explicitly from the securityonionkernel repo. -# 2. Boot it. Installing kernel-uek-core adds a UEK8 boot entry but does NOT make it the -# default: kernel-install/grubby only auto-promote a new kernel within the running -# kernel's flavor lineage, and we're crossing from a 5.x kernel to the new 6.x UEK -# flavor. So even with UPDATEDEFAULT=yes and DEFAULTKERNEL=kernel-uek-core the box -# keeps booting the old kernel until grubby is told otherwise. +# 1. Populate. The manager mirrors the UEK8 packages into /nsm/kernelrepo via so-repo-sync, +# and serves them to the grid over https:///kernelrepo. Until that sync runs the +# repo is valid but EMPTY -- dnf resolves it happily and installs nothing, with no error. +# 2. Install. A node on RHCK has no kernel-uek* package at all, so there is nothing for +# 'dnf update' to upgrade. A node on UEK7 does have kernel-uek installed, so +# 'dnf install kernel-uek' reports "Nothing to do" and exits 0 without installing 6.x. +# Both cases need an explicit install of the UEK8 NEVRA. +# 3. Boot it. Installing kernel-uek-core adds a UEK8 boot entry but does NOT make it the +# default: kernel-install/grubby only auto-promote within the running kernel's flavor +# lineage, and we're crossing from 5.x to the 6.x UEK flavor. So even with +# UPDATEDEFAULT=yes and DEFAULTKERNEL=kernel-uek-core the box keeps booting the old +# kernel until grubby is told otherwise. # -# This tool does both, then points DEFAULTKERNEL at kernel-uek-core so later kernel updates -# stay on the UEK line. +# Every one of those failure modes is silent by default. This tool does all three and fails +# loudly when it cannot, rather than reporting success while changing nothing. +# +# Manager vs minion: only the manager owns /nsm/kernelrepo, so only the manager can populate +# it. If the repo is empty here, a manager runs so-repo-sync itself; a minion has no way to +# fix it and exits non-zero telling the admin to sync the manager first. # # Idempotent: an already-installed, already-default UEK8 kernel is left alone. It only sets -# the boot default; it does NOT reboot — the admin reboots the node on their own schedule. +# the boot default; it does NOT reboot -- the admin reboots the node on their own schedule. + +. /usr/sbin/so-common KERNEL_REPO="securityonionkernel" KERNEL_PKG="kernel-uek" +KERNEL_REPO_DIR="/nsm/kernelrepo" +REPOSYNC_CONF="/opt/so/conf/reposync/repodownload.conf" +GLOBAL_PILLAR="/opt/so/saltstack/local/pillar/global/soc_global.sls" log() { echo "[so-kernel-upgrade] $*"; } +die() { echo "[so-kernel-upgrade] ERROR: $*" >&2; exit 1; } -[ "$(id -u)" -eq 0 ] || { log "must run as root"; exit 1; } -command -v grubby >/dev/null 2>&1 || { log "grubby not found"; exit 1; } -command -v dnf >/dev/null 2>&1 || { log "dnf not found"; exit 1; } +command -v grubby >/dev/null 2>&1 || die "grubby not found" +command -v dnf >/dev/null 2>&1 || die "dnf not found" + +ARCH="$(rpm -E '%{_arch}')" + +is_airgap() { + [ -f "$GLOBAL_PILLAR" ] && grep -q 'airgap: *[Tt]rue' "$GLOBAL_PILLAR" +} # Newest installed UEK8 (6.x) kernel known to the bootloader. UEK8 vmlinuz paths look like -# /boot/vmlinuz-6.12.0-203.76.7.5.el9uek.x86_64; the 5.15 UEK7 and 5.14 RHCK won't match. +# /boot/vmlinuz-6.12.0-204.92.4.2.el9uek.x86_64; UEK7 (5.15) and RHCK (5.14) won't match. find_uek8() { grubby --info=ALL 2>/dev/null \ | sed -n 's/^kernel="\(.*\)"$/\1/p' \ @@ -43,28 +62,86 @@ find_uek8() { | sort -V | tail -1 } +# Newest UEK8 kernel-uek NEVRA offered by the kernel repo, empty if the repo has none. +# Restricted to the kernel repo so a UEK7 kernel-uek in the main repo can't be picked up, +# and filtered to 6.x so we never "succeed" by reinstalling the 5.15 we already have. +uek8_available() { + dnf -q repoquery --disablerepo='*' --enablerepo="$KERNEL_REPO" \ + --arch="$ARCH" --latest-limit=1 \ + --qf '%{name}-%{evr}.%{arch}\n' "$KERNEL_PKG" 2>/dev/null \ + | grep -E "^${KERNEL_PKG}-6\." | tail -1 +} + +kernelrepo_rpm_count() { + find "$KERNEL_REPO_DIR" -maxdepth 1 -name '*.rpm' 2>/dev/null | wc -l +} + +# The kernel repo starts life as valid-but-empty (kernelrepo_init_empty in +# salt/manager/init.sls) and is filled by so-repo-sync. During a soup, so-repo-sync runs +# BEFORE the highstate deploys the [securityonionkernel] section into repodownload.conf, so +# the first kernel-aware soup leaves the repo empty until the next nightly sync. +sync_kernel_repo() { + if is_airgap; then + log "airgap install: $KERNEL_REPO_DIR is populated from the airgap ISO, not by so-repo-sync." + return 1 + fi + if ! grep -q "^\[${KERNEL_REPO}\]" "$REPOSYNC_CONF" 2>/dev/null; then + log "$REPOSYNC_CONF has no [${KERNEL_REPO}] section -- run a highstate to deploy it." + return 1 + fi + + log "populating $KERNEL_REPO_DIR with so-repo-sync (mirrors upstream; can take several minutes)" + su socore -c '/usr/sbin/so-repo-sync' || { log "so-repo-sync failed"; return 1; } + + dnf -q clean expire-cache >/dev/null 2>&1 + return 0 +} + +# Make the kernel repo actually able to serve a UEK8 package, or fail trying. +ensure_kernel_repo() { + # The repo is assigned by the repo.client highstate, and only once NICs are pinned by MAC + # (/opt/so/state/nic_names_pinned) so the kernel swap can't renumber interfaces SO binds + # by name. skip_if_unavailable=1 means a broken repo is silently ignored, so check first. + if ! dnf -q repolist --enabled 2>/dev/null | awk '{print $1}' | grep -qx "$KERNEL_REPO"; then + log "repo '$KERNEL_REPO' is not enabled on this node." + log "Run a highstate first; the repo is skipped until /opt/so/state/nic_names_pinned" + log "exists (run so-nic-pin) and this node's salt matches the version this release ships." + die "kernel repo unavailable" + fi + + [ -n "$(uek8_available)" ] && return 0 + + log "repo '$KERNEL_REPO' is enabled but offers no UEK8 $KERNEL_PKG package" + + if ! is_manager_node; then + log "This is a minion; it consumes the kernel repo from the manager and cannot populate it." + log "On the manager, run: su socore -c /usr/sbin/so-repo-sync" + log "then re-run this script here." + die "manager's kernel repo is empty" + fi + + log "this is a manager and $KERNEL_REPO_DIR holds $(kernelrepo_rpm_count) rpm(s)" + sync_kernel_repo || die "could not populate $KERNEL_REPO_DIR" + + [ -n "$(uek8_available)" ] \ + || die "so-repo-sync completed but $KERNEL_REPO still offers no UEK8 $KERNEL_PKG" +} + target="$(find_uek8)" if [ -z "$target" ]; then - log "no UEK8 kernel installed — installing $KERNEL_PKG from $KERNEL_REPO" + log "no UEK8 kernel installed (running $(uname -r))" - # The repo is assigned by the repo.client highstate, and only once NICs are pinned by MAC - # (/opt/so/state/nic_names_pinned) so the kernel swap can't renumber interfaces SO binds - # by name. Without the repo we'd silently pull nothing, so fail loudly instead. - if ! dnf -q repolist --enabled 2>/dev/null | awk '{print $1}' | grep -qx "$KERNEL_REPO"; then - log "ERROR: repo '$KERNEL_REPO' is not enabled on this node." - log "Run a highstate first; it is skipped until /opt/so/state/nic_names_pinned exists" - log "(run so-nic-pin) and this node's salt matches the version this release ships." - exit 1 - fi + ensure_kernel_repo + nevra="$(uek8_available)" - dnf -y install "$KERNEL_PKG" || { log "ERROR: failed to install $KERNEL_PKG"; exit 1; } + # Install the explicit NEVRA, not the bare package name. On a UEK7 node 'dnf install + # kernel-uek' sees kernel-uek-5.15 already installed, prints "Nothing to do" and exits 0. + log "installing $nevra from $KERNEL_REPO" + dnf -y install "$nevra" || die "failed to install $nevra" target="$(find_uek8)" - if [ -z "$target" ]; then - log "ERROR: $KERNEL_PKG installed but no 6.x UEK boot entry appeared — check 'grubby --info=ALL'" - exit 1 - fi + [ -n "$target" ] || die "$nevra installed but no 6.x UEK boot entry appeared -- check 'grubby --info=ALL'" log "installed UEK8 kernel: $target" fi @@ -75,24 +152,25 @@ if [ -f /etc/sysconfig/kernel ] && ! grep -q '^DEFAULTKERNEL=kernel-uek-core$' / sed -i 's/^DEFAULTKERNEL=.*/DEFAULTKERNEL=kernel-uek-core/' /etc/sysconfig/kernel fi +reboot_notice() { + [ "$(uname -r)" = "$(basename "$1" | sed 's/^vmlinuz-//')" ] \ + || log "REBOOT REQUIRED to start using the UEK8 kernel (currently running $(uname -r))." +} + current="$(grubby --default-kernel 2>/dev/null)" if [ "$current" = "$target" ]; then log "UEK8 kernel is already the boot default: $target" - [ "$(uname -r)" = "$(basename "$target" | sed 's/^vmlinuz-//')" ] \ - || log "REBOOT REQUIRED to start using it (currently running $(uname -r))." + reboot_notice "$target" exit 0 fi log "current default kernel: ${current:-unknown}" log "switching boot default to UEK8 kernel: $target" -grubby --set-default="$target" || { log "ERROR: grubby --set-default failed for $target"; exit 1; } +grubby --set-default="$target" || die "grubby --set-default failed for $target" # Verify the change actually took before claiming success. now="$(grubby --default-kernel 2>/dev/null)" -if [ "$now" != "$target" ]; then - log "ERROR: default kernel is still '${now:-unknown}' after set-default" - exit 1 -fi +[ "$now" = "$target" ] || die "default kernel is still '${now:-unknown}' after set-default" log "boot default is now $target" -log "REBOOT REQUIRED to start using the UEK8 kernel (currently running $(uname -r))." +reboot_notice "$target" From 9a71f64a358121ce0bd7bca6fd872c03d17a5ba0 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 9 Jul 2026 15:10:13 -0400 Subject: [PATCH 147/219] Branch so-kernel-upgrade on the running kernel flavor Only the RHCK->UEK flavor cross needs grubby --set-default; a UEK7->UEK8 update stays in the kernel-uek lineage and auto-promotes on its own. Detect the running kernel and act accordingly: - UEK8: already on target, no-op. - UEK7: populate the repo and install UEK8, then verify it auto-promoted (warn with the manual grubby command if it did not) -- no grubby change. - RHCK: install UEK8 and set the boot default explicitly, as before. Also make an already-installed UEK8 skip the repo entirely so a disabled or empty kernel repo can't block flipping the default, and correct the header comment that claimed every transition needs grubby. --- salt/common/tools/sbin/so-kernel-upgrade | 153 ++++++++++++++++------- 1 file changed, 108 insertions(+), 45 deletions(-) diff --git a/salt/common/tools/sbin/so-kernel-upgrade b/salt/common/tools/sbin/so-kernel-upgrade index d04ca533c..960505aee 100755 --- a/salt/common/tools/sbin/so-kernel-upgrade +++ b/salt/common/tools/sbin/so-kernel-upgrade @@ -8,7 +8,7 @@ # so-kernel-upgrade — install the UEK8 (6.x) kernel and make it the boot default. # # Security Onion is moving off the EL9 stock kernel (RHCK, 5.14) and UEK7 (5.15) onto UEK8 -# (6.x). Three things have to happen, and none of them are automatic: +# (6.x). Three things have to happen, and the tool has to drive each one: # # 1. Populate. The manager mirrors the UEK8 packages into /nsm/kernelrepo via so-repo-sync, # and serves them to the grid over https:///kernelrepo. Until that sync runs the @@ -17,13 +17,17 @@ # 'dnf update' to upgrade. A node on UEK7 does have kernel-uek installed, so # 'dnf install kernel-uek' reports "Nothing to do" and exits 0 without installing 6.x. # Both cases need an explicit install of the UEK8 NEVRA. -# 3. Boot it. Installing kernel-uek-core adds a UEK8 boot entry but does NOT make it the -# default: kernel-install/grubby only auto-promote within the running kernel's flavor -# lineage, and we're crossing from 5.x to the 6.x UEK flavor. So even with -# UPDATEDEFAULT=yes and DEFAULTKERNEL=kernel-uek-core the box keeps booting the old -# kernel until grubby is told otherwise. +# 3. Boot it. Whether a newly installed UEK8 kernel becomes the boot default depends on the +# RUNNING kernel's flavor. kernel-install/grubby (with UPDATEDEFAULT=yes) only auto-promote +# within the running kernel's flavor lineage: +# - From UEK7 (5.x, kernel-uek) the install stays in the kernel-uek lineage and IS +# auto-promoted, so no grubby change is needed -- just make sure the repo is populated +# and install UEK8. +# - From the stock EL9 kernel (RHCK, 5.14, no UEK) it is a flavor CROSS that is NOT +# auto-promoted, so the box keeps booting RHCK until grubby is told otherwise. +# This tool inspects the running kernel and only runs 'grubby --set-default' for RHCK. # -# Every one of those failure modes is silent by default. This tool does all three and fails +# Every one of those failure modes is silent by default. This tool handles each case and fails # loudly when it cannot, rather than reporting success while changing nothing. # # Manager vs minion: only the manager owns /nsm/kernelrepo, so only the manager can populate @@ -62,6 +66,19 @@ find_uek8() { | sort -V | tail -1 } +# Classify the RUNNING kernel (uname -r) -- this, not what's installed, is what decides whether +# a UEK8 install auto-promotes to the boot default: +# uek8 6.x UEK already on the target line; nothing to do +# uek7 5.x UEK a UEK8 install stays in the kernel-uek lineage and auto-promotes (no grubby) +# rhck 5.14 EL9 crossing into the UEK flavor does NOT auto-promote (needs grubby --set-default) +running_flavor() { + case "$(uname -r)" in + 6.*uek*) echo uek8 ;; + *uek*) echo uek7 ;; + *) echo rhck ;; + esac +} + # Newest UEK8 kernel-uek NEVRA offered by the kernel repo, empty if the repo has none. # Restricted to the kernel repo so a UEK7 kernel-uek in the main repo can't be picked up, # and filtered to 6.x so we never "succeed" by reinstalling the 5.15 we already have. @@ -127,50 +144,96 @@ ensure_kernel_repo() { || die "so-repo-sync completed but $KERNEL_REPO still offers no UEK8 $KERNEL_PKG" } -target="$(find_uek8)" - -if [ -z "$target" ]; then - log "no UEK8 kernel installed (running $(uname -r))" - - ensure_kernel_repo - nevra="$(uek8_available)" - - # Install the explicit NEVRA, not the bare package name. On a UEK7 node 'dnf install - # kernel-uek' sees kernel-uek-5.15 already installed, prints "Nothing to do" and exits 0. - log "installing $nevra from $KERNEL_REPO" - dnf -y install "$nevra" || die "failed to install $nevra" - - target="$(find_uek8)" - [ -n "$target" ] || die "$nevra installed but no 6.x UEK boot entry appeared -- check 'grubby --info=ALL'" - log "installed UEK8 kernel: $target" -fi - -# Keep future kernel updates on the UEK line rather than falling back to RHCK. Oracle ships -# this file; only rewrite it when it's actually pointing somewhere else. -if [ -f /etc/sysconfig/kernel ] && ! grep -q '^DEFAULTKERNEL=kernel-uek-core$' /etc/sysconfig/kernel; then - log "setting DEFAULTKERNEL=kernel-uek-core in /etc/sysconfig/kernel" - sed -i 's/^DEFAULTKERNEL=.*/DEFAULTKERNEL=kernel-uek-core/' /etc/sysconfig/kernel -fi - reboot_notice() { [ "$(uname -r)" = "$(basename "$1" | sed 's/^vmlinuz-//')" ] \ || log "REBOOT REQUIRED to start using the UEK8 kernel (currently running $(uname -r))." } -current="$(grubby --default-kernel 2>/dev/null)" -if [ "$current" = "$target" ]; then - log "UEK8 kernel is already the boot default: $target" - reboot_notice "$target" +# Keep future kernel updates on the UEK line rather than falling back to RHCK. Oracle ships +# /etc/sysconfig/kernel; only rewrite it when it's actually pointing somewhere else. +set_default_kernel_conf() { + if [ -f /etc/sysconfig/kernel ] && ! grep -q '^DEFAULTKERNEL=kernel-uek-core$' /etc/sysconfig/kernel; then + log "setting DEFAULTKERNEL=kernel-uek-core in /etc/sysconfig/kernel" + sed -i 's/^DEFAULTKERNEL=.*/DEFAULTKERNEL=kernel-uek-core/' /etc/sysconfig/kernel + fi +} + +# Make sure a UEK8 kernel is installed, leaving its boot entry in INSTALLED_UEK8. If one is +# already present we leave the repo alone -- it may be disabled or empty and we don't need it +# just to flip the boot default. Otherwise install the explicit NEVRA, not the bare package +# name: on a UEK7 node 'dnf install kernel-uek' sees 5.15 already present, prints "Nothing to +# do" and exits 0 without installing 6.x. +ensure_uek8_installed() { + INSTALLED_UEK8="$(find_uek8)" + if [ -n "$INSTALLED_UEK8" ]; then + log "UEK8 kernel already installed: $INSTALLED_UEK8" + return 0 + fi + + ensure_kernel_repo + local nevra; nevra="$(uek8_available)" + log "installing $nevra from $KERNEL_REPO" + dnf -y install "$nevra" || die "failed to install $nevra" + + INSTALLED_UEK8="$(find_uek8)" + [ -n "$INSTALLED_UEK8" ] || die "$nevra installed but no 6.x UEK boot entry appeared -- check 'grubby --info=ALL'" + log "installed UEK8 kernel: $INSTALLED_UEK8" +} + +case "$(running_flavor)" in +uek8) + # Already on the 6.x UEK line. A plain 'dnf update' keeps this node current within the + # lineage and auto-promotes newer builds, so there is nothing for this tool to do. + log "already running a UEK8 kernel ($(uname -r)); nothing to do." exit 0 -fi + ;; -log "current default kernel: ${current:-unknown}" -log "switching boot default to UEK8 kernel: $target" -grubby --set-default="$target" || die "grubby --set-default failed for $target" +uek7) + # On a 5.x UEK kernel. Installing UEK8 stays inside the kernel-uek lineage, so dnf/grubby + # (UPDATEDEFAULT=yes) auto-promote it and we do NOT touch grubby. A node still on UEK7 + # usually means the kernel repo was empty when it last updated, so populate it and install. + log "running UEK7 kernel ($(uname -r)); the kernel repo was likely not yet populated when" + log "this node last updated. Populating it and installing UEK8 -- the update stays on the" + log "kernel-uek line, so it becomes the boot default automatically (no grubby change needed)." + set_default_kernel_conf + ensure_uek8_installed -# Verify the change actually took before claiming success. -now="$(grubby --default-kernel 2>/dev/null)" -[ "$now" = "$target" ] || die "default kernel is still '${now:-unknown}' after set-default" + now="$(grubby --default-kernel 2>/dev/null)" + if [ "$now" = "$INSTALLED_UEK8" ]; then + log "boot default auto-promoted to UEK8 kernel: $INSTALLED_UEK8" + else + log "WARNING: expected the UEK8 kernel to auto-promote but the default is still" + log "'${now:-unknown}'. Run 'grubby --set-default=$INSTALLED_UEK8' to force it." + fi + reboot_notice "$INSTALLED_UEK8" + ;; -log "boot default is now $target" -reboot_notice "$target" +rhck) + # On the stock EL9 kernel (5.14, no UEK installed). Crossing from RHCK into the UEK flavor + # does NOT auto-promote -- kernel-install/grubby only auto-promote within the running + # kernel's flavor lineage -- so after installing we must set the boot default explicitly. + log "running stock EL9 (RHCK) kernel ($(uname -r)); installing UEK8 and setting it as the" + log "boot default explicitly (a RHCK->UEK flavor change does not auto-promote)." + set_default_kernel_conf + ensure_uek8_installed + target="$INSTALLED_UEK8" + + current="$(grubby --default-kernel 2>/dev/null)" + if [ "$current" = "$target" ]; then + log "UEK8 kernel is already the boot default: $target" + reboot_notice "$target" + exit 0 + fi + + log "current default kernel: ${current:-unknown}" + log "switching boot default to UEK8 kernel: $target" + grubby --set-default="$target" || die "grubby --set-default failed for $target" + + # Verify the change actually took before claiming success. + now="$(grubby --default-kernel 2>/dev/null)" + [ "$now" = "$target" ] || die "default kernel is still '${now:-unknown}' after set-default" + + log "boot default is now $target" + reboot_notice "$target" + ;; +esac From 6fa0d327cbafd9a18d38a4f3bb51b1f145e74cfe Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:14:58 -0500 Subject: [PATCH 148/219] cause so-setup to fail if there are issues setting up fleet --- .../tools/sbin_jinja/so-elastic-fleet-setup | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index b62310375..2b677606a 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -244,11 +244,27 @@ printf '%s\n'\ "" >> "$global_pillar_file" # Call Elastic-Fleet Salt State -printf "\nApplying elasticfleet state" -salt-call state.apply elasticfleet queue=True +printf "\nApplying elasticfleet state\n" +if ! salt-call state.apply elasticfleet queue=True; then + printf "\nFailure(s) in elasticfleet state... Exiting...\n" + exit 1 +fi # Generate installers & install Elastic Agent on the node -so-elastic-agent-gen-installers -printf "\nApplying elasticfleet.install_agent_grid state" -salt-call state.apply elasticfleet.install_agent_grid queue=True -exit 0 +for agent_gen_attempt in {1..3}; do + if so-elastic-agent-gen-installers; then + break + elif [[ $agent_gen_attempt -lt 3 ]]; then + printf "\nUnable to generate Elastic Agent installers... Attempt (%s/3). Retrying...\n" "$agent_gen_attempt" + sleep 10 + else + printf "\nFailed to generate Elastic Agent installers after 3 attempts. Exiting...\n" + exit 1 + fi +done + +printf "\nApplying elasticfleet.install_agent_grid state\n" +if ! salt-call state.apply elasticfleet.install_agent_grid queue=True; then + printf "\nFailure(s) in elasticfleet.install_agent_grid state... Exiting...\n" + exit 1 +fi \ No newline at end of file From 8b0759866ea8d2c121c0eb32b9bdf581fed6c7e8 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:15:22 -0500 Subject: [PATCH 149/219] verify installers are generated --- .../so-elastic-agent-gen-installers | 39 +++++++++++++++---- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers index d25c18e29..de342657f 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -30,7 +30,7 @@ done if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then printf "\nFleet Host URL, Enrollment Token or Elastic Version empty - exiting..." printf "\nFleet Host: $FLEETHOST, Enrollment Token: $ENROLLMENTOKEN\n" - exit + exit 1 fi OSARCH=( "linux-x86_64" "windows-x86_64" "darwin-x86_64" "darwin-aarch64" ) @@ -62,31 +62,54 @@ do done GOTARGETOS=( "linux" "windows" "darwin" "darwin/arm64" ) -GOARCH="amd64" printf "\n### Generating OS packages using the cleaned up tarballs" -for GOOS in "${GOTARGETOS[@]}" -do +for GOOS in "${GOTARGETOS[@]}"; do + GOARCH="amd64" if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi printf "\n\n### Generating $GOOS/$GOARCH Installer...\n" docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \ --mount type=bind,source=/etc/pki/tls/certs/,target=/workspace/files/cert/ \ --mount type=bind,source=/nsm/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \ - --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \ + --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ \ {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN" -o /output/so-elastic-agent_${GOOS}_${GOARCH} printf "\n### $GOOS/$GOARCH Installer Generated...\n" done printf "\n\n### Generating MSI...\n" -cp /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe +cp /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe docker run \ ---mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ -w /output \ +--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ -w /output \ {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs printf "\n### MSI Generated...\n" +# Verify installers were created +for GOOS in "${GOTARGETOS[@]}"; do + GOARCH="amd64" + if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin"; GOARCH="arm64"; fi + if [[ ! -f /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_${GOOS}_${GOARCH} ]]; then + printf "\n### ERROR: Installer for %s/%s was not generated. Exiting...\n" "$GOOS" "$GOARCH" + exit 1 + fi + # After verifying new installer was generated, move it to so_agent-installers directory + mv /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_${GOOS}_${GOARCH} /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ +done + +# Verify MSI installer +if [[ ! -f /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64_msi ]]; then + printf "\n### ERROR: Installer MSI was not generated. Exiting...\n" + exit 1 +else + # After verifying new installer MSI was generated, move it to so_agent-installers directory + mv /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64_msi /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ +fi + printf "\n### Cleaning up temp files \n" rm -rf /nsm/elastic-agent-workspace -rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe +rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe printf "\n### Copying so_agent-installers to /nsm/elastic-fleet/ for nginx.\n" \cp -vr /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ /nsm/elastic-fleet/ chmod 644 /nsm/elastic-fleet/so_agent-installers/* + +# if we got here all installers have been generated successfully +exit 0 From 2959dc95644a4372dadd425fd72929d32e093177 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:18:08 -0500 Subject: [PATCH 150/219] keep logs for all 3 attempts --- salt/elasticfleet/tools/sbin/so-elastic-agent-install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-install b/salt/elasticfleet/tools/sbin/so-elastic-agent-install index 93889c002..8f57d217c 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-agent-install +++ b/salt/elasticfleet/tools/sbin/so-elastic-agent-install @@ -16,8 +16,6 @@ check_agent_health() { interval=10 start=$SECONDS - truncate -s 0 "$LOGFILE" - while (( SECONDS - start < timeout )); do agent_status=$(elastic-agent status 2>&1) echo -e "\n$(date)\n$agent_status\n" >> "$LOGFILE" @@ -70,6 +68,8 @@ if [[ -x /opt/so/so-elastic-agent_linux_amd64 ]]; then attempts=0 cd /opt/so/ || exit 1 + truncate -s 0 "$LOGFILE" + uninstall_agent while [[ $attempts -lt 3 ]]; do From 0b078c48041a30947efbb6dc8bf4945e26272339 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:23:45 -0500 Subject: [PATCH 151/219] give the elasticfleet state a few chances to complete successfully before exiting 1 causing so-setup to fail --- .../tools/sbin_jinja/so-elastic-fleet-setup | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index 2b677606a..21d74cf6f 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -245,10 +245,17 @@ printf '%s\n'\ # Call Elastic-Fleet Salt State printf "\nApplying elasticfleet state\n" -if ! salt-call state.apply elasticfleet queue=True; then - printf "\nFailure(s) in elasticfleet state... Exiting...\n" - exit 1 -fi +for state_attempt in {1..3}; do + if salt-call state.apply elasticfleet queue=True; then + break + elif [[ $state_attempt -lt 3 ]]; then + printf "\nElasticfleet state did not complete successfully... Attempt (%s/3). Retrying...\n" "$state_attempt" + sleep 10 + else + printf "\nFailure(s) in elasticfleet state... Exiting...\n" + exit 1 + fi +done # Generate installers & install Elastic Agent on the node for agent_gen_attempt in {1..3}; do From f6a2758321891c9e12f0d4efe29a76ee446067e5 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:44:42 -0500 Subject: [PATCH 152/219] add so-elastic-agent-install script to minions --- salt/elasticfleet/install_agent_grid.sls | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/elasticfleet/install_agent_grid.sls b/salt/elasticfleet/install_agent_grid.sls index 431aa6f97..7ed727b48 100644 --- a/salt/elasticfleet/install_agent_grid.sls +++ b/salt/elasticfleet/install_agent_grid.sls @@ -10,6 +10,15 @@ {% set AGENT_STATUS = salt['service.available']('elastic-agent') %} {% set AGENT_EXISTS = salt['file.file_exists']('/opt/Elastic/Agent/elastic-agent') %} +so-elastic-agent-install: + file.managed: + - name: /usr/sbin/so-elastic-agent-install + - source: salt://elasticfleet/tools/sbin/so-elastic-agent-install + - user: 947 + - group: 939 + - mode: 755 + - show_changes: False + {% if not AGENT_STATUS or not AGENT_EXISTS %} pull_agent_installer: From 8a3f5d0f819ebd2b2059aa71a446e0af9335a31a Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 9 Jul 2026 16:55:30 -0400 Subject: [PATCH 153/219] Gate so-salt-minion-wait on real minion readiness The previous gate did not detect whether the restarted minion was back: systemctl is-active --quiet salt-minion \ && salt-call --local --timeout=5 --out=quiet test.ping Both halves are near-vacuous. `--local` sets file_client=local, so test.ping runs in a throwaway minion that never contacts the master and never inspects the running daemon; it only proves python and the module loader work. And the shipped unit is Type=notify with notify_systemd() called before the daemon imports salt.cli.daemons, so is-active goes true at process launch, not at connection. The script could return ready while the minion was still authenticating, which is the race it exists to prevent. Gate instead on the condition salt itself uses to log "Minion is ready to receive requests!", requiring both signals of the current daemon instance: 1. the pid-tagged ready line in the minion log. tune_in() emits it only after sync_connect_master() returns, i.e. the pub channel authenticated, the req channel connected, and _post_master_init() finished loading modules and compiling pillar. 2. that same pid holding an ESTABLISHED req connection to a master on 4506 plus a second (publish) connection to the same master IP. The publish port is absent from minion config -- the minion learns it from the master's auth reply -- so it is derived from the connection. Resolve the daemon pid from systemd (MainPID -> pgrep -P), never from /var/run/salt-minion.pid. salt_minion() runs the minion in a multiprocessing child; that child writes the pidfile, owns the sockets and logs the ready line, while MainPID is the parent. During a restart the pidfile still names the old child, whose own ready line is already in the log, so keying off it reports ready instantly. Children of the current MainPID exclude the old instance structurally, with no timing assumptions. Degrade deterministically rather than spinning to the timeout: if log_level_logfile does not emit INFO records the ready line can never appear, so detect that up front from the merged config and fall back to the socket check. log_level_logfile defaults to None (inherit log_level), so resolve the inheritance before deciding. If ss is unavailable, fall back to the log gate. If neither signal is usable, fail immediately with a clear message. Requiring master connectivity adds no new dependency: every path that applies salt.minion or a highstate does so without --local, so file_client=remote already required a reachable master to fetch salt:// files. No salt-call master round-trip is added; the daemon's own successful auth already proves the key is accepted. Also fix the comment above wait_for_salt_minion_ready, which attributed the script to common_sbin/common/tools/sbin (it is deployed by salt_sbin from salt/tools/sbin) and asserted a --no-block restart that appears nowhere in the repo. No state logic changed. --- salt/salt/minion/init.sls | 13 ++- salt/salt/tools/sbin/so-salt-minion-wait | 142 +++++++++++++++++++++-- 2 files changed, 138 insertions(+), 17 deletions(-) diff --git a/salt/salt/minion/init.sls b/salt/salt/minion/init.sls index a251aa633..035d9e7f8 100644 --- a/salt/salt/minion/init.sls +++ b/salt/salt/minion/init.sls @@ -131,11 +131,14 @@ salt_minion_service: {% endif %} - order: last -# block until the just-restarted salt-minion is back and can execute modules locally, so -# follow-on jobs and the next highstate iteration do not race the restart. onchanges + -# require on salt_minion_service catches every restart trigger uniformly because watch -# mod_watch results replace the service state's running entry. wait logic lives in -# /usr/sbin/so-salt-minion-wait (deployed by common_sbin from common/tools/sbin/). +# block until the just-restarted salt-minion daemon logs "Minion is ready to receive requests!" +# for the current instance, so follow-on jobs and the next highstate iteration do not race the +# restart. onchanges + require on salt_minion_service catches every restart trigger uniformly +# because watch mod_watch results replace the service state's running entry. wait logic lives in +# /usr/sbin/so-salt-minion-wait (deployed by salt_sbin from salt/tools/sbin/); it keys the ready +# line to the current daemon pid (resolved via systemd, not the pidfile) and corroborates with the +# master req/publish sockets. set_log_levels above enforces the log_level_logfile: info that the +# ready line depends on. wait_for_salt_minion_ready: cmd.run: - name: /usr/sbin/so-salt-minion-wait diff --git a/salt/salt/tools/sbin/so-salt-minion-wait b/salt/salt/tools/sbin/so-salt-minion-wait index a30c67e80..9fb2a41c7 100644 --- a/salt/salt/tools/sbin/so-salt-minion-wait +++ b/salt/salt/tools/sbin/so-salt-minion-wait @@ -5,31 +5,149 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -# Block until the local salt-minion service is back up and can execute modules locally. -# Invoked from the wait_for_salt_minion_ready state in salt/minion/init.sls after -# salt_minion_service fires its watch-driven mod_watch (a non-blocking systemctl restart), -# so follow-on jobs and the next highstate iteration do not race the in-flight restart. +# Block until the just-restarted salt-minion daemon reaches the point where salt itself logs +# "Minion is ready to receive requests!". Invoked from the wait_for_salt_minion_ready state in +# salt/minion/init.sls after salt_minion_service fires its watch-driven restart, so follow-on jobs +# and the next highstate iteration do not race it. +# +# Salt logs that line from Minion.tune_in() only after sync_connect_master() returns, which means +# the pub channel authenticated, the long-running req channel connected, and _post_master_init() +# finished loading modules and compiling pillar. Two signals reproduce that: +# +# 1. Primary the pid-tagged ready line in the minion log. Salt's log_fmt_logfile embeds +# [%(process)d] just before the message, so this is keyed to one daemon instance. +# 2. Corroborating that same pid holds an ESTABLISHED req connection to a master on 4506 plus a +# second (publish) connection to that same master IP on another port. The publish +# port is learned from the master's auth reply and is absent from minion config, +# so it is derived from the connection rather than read from config. +# +# The daemon pid is resolved from systemd, never from /var/run/salt-minion.pid. salt_minion() runs +# the real minion in a multiprocessing child; that child writes the pidfile, owns the sockets and +# logs the ready line, while systemd's MainPID is the parent. During a restart the pidfile can still +# name the OLD child, whose own ready line is already in the log -- matching it would report ready +# instantly. Children of the current MainPID structurally exclude the old instance. . /usr/sbin/so-common -# Initial sleep gives the systemctl restart (--no-block by default for salt-minion on -# >=3006.15) time to begin tearing down the old process before we probe for readiness. +set -u + INITIAL_SLEEP=3 TIMEOUT=120 -PING_TIMEOUT=5 +MASTER_PORT=4506 +LOG_TAIL_LINES=10000 +DEFAULT_LOG_FILE="/opt/so/log/salt/minion" +LOG_FILE="$DEFAULT_LOG_FILE" + +# Decide whether the ready line can ever appear. salt-call --local sets file_client=local, so this +# reads the merged config (honoring minion.d overrides) without contacting the master. salt defaults +# log_level_logfile to None, meaning it inherits log_level, so resolve that before deciding. +LOG_LEVEL_LOGFILE=$(salt-call --local --out=newline_values_only config.get log_level_logfile 2>/dev/null | head -n1) +case "${LOG_LEVEL_LOGFILE,,}" in + ""|none) LOG_LEVEL_LOGFILE=$(salt-call --local --out=newline_values_only config.get log_level 2>/dev/null | head -n1) ;; +esac + +case "${LOG_LEVEL_LOGFILE,,}" in + all|garbage|trace|debug|profile|info) USE_LOG_GATE=1 ;; + *) USE_LOG_GATE=0 ;; +esac + +if [ "$USE_LOG_GATE" -eq 1 ]; then + LOG_FILE=$(salt-call --local --out=newline_values_only config.get log_file 2>/dev/null | head -n1) + [ -z "$LOG_FILE" ] && LOG_FILE="$DEFAULT_LOG_FILE" + [ -d "$(dirname "$LOG_FILE")" ] || USE_LOG_GATE=0 +fi + +if command -v ss >/dev/null 2>&1; then + USE_SOCKET_GATE=1 +else + USE_SOCKET_GATE=0 +fi + +if [ "$USE_LOG_GATE" -eq 0 ] && [ "$USE_SOCKET_GATE" -eq 0 ]; then + echo "so-salt-minion-wait: no usable readiness signal (log_level_logfile='${LOG_LEVEL_LOGFILE:-unset}', ss not found)" >&2 + exit 1 +fi + +if [ "$USE_LOG_GATE" -eq 1 ] && [ "$USE_SOCKET_GATE" -eq 1 ]; then + echo "so-salt-minion-wait: gating on pid-tagged ready line in ${LOG_FILE} plus master sockets" +elif [ "$USE_LOG_GATE" -eq 1 ]; then + echo "so-salt-minion-wait: ss not found; gating on pid-tagged ready line in ${LOG_FILE} only" +else + echo "so-salt-minion-wait: INFO file logging unavailable (log_level_logfile='${LOG_LEVEL_LOGFILE:-unset}'); gating on master sockets only" +fi + +# Emit the pid(s) of the current daemon instance. systemd's MainPID is the parent keepalive process; +# its child runs tune_in. Fall back to MainPID when there is no child (--disable-keepalive path). +resolve_daemon_pids() { + local mainpid children + mainpid=$(systemctl show -p MainPID --value salt-minion 2>/dev/null) + if [ -z "$mainpid" ] || [ "$mainpid" = "0" ]; then + return 1 + fi + children=$(pgrep -P "$mainpid" 2>/dev/null) + printf '%s\n' "${children:-$mainpid}" +} + +# True iff the ready line tagged with this pid is in the current or most recently rotated log. +ready_logged() { + local pid=$1 f + for f in "$LOG_FILE" "$LOG_FILE.1"; do + [ -r "$f" ] || continue + if tail -n "$LOG_TAIL_LINES" "$f" 2>/dev/null | grep -Fq "[$pid] Minion is ready to receive requests!"; then + return 0 + fi + done + return 1 +} + +# True iff this pid holds an ESTABLISHED req connection to a master on MASTER_PORT and a second +# ESTABLISHED connection to that same master IP on another port. The trailing comma in "pid=N," +# keeps pid=123 from matching pid=1234. Grid comms are IPv4 (the unit's ExecStartPre gates on ip -4). +socket_ready() { + local pid=$1 mip master_ips + master_ips=$(ss -tnp state established "dport = :${MASTER_PORT}" 2>/dev/null \ + | grep -F "pid=${pid}," \ + | grep -oE "[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+:${MASTER_PORT}" \ + | sed "s/:${MASTER_PORT}\$//" \ + | sort -u) + [ -z "$master_ips" ] && return 1 + for mip in $master_ips; do + if ss -tnp state established "dst ${mip} and dport != :${MASTER_PORT}" 2>/dev/null | grep -qF "pid=${pid},"; then + return 0 + fi + done + return 1 +} + +instance_ready() { + local pid=$1 + if [ "$USE_LOG_GATE" -eq 1 ] && ! ready_logged "$pid"; then + return 1 + fi + if [ "$USE_SOCKET_GATE" -eq 1 ] && ! socket_ready "$pid"; then + return 1 + fi + return 0 +} sleep "$INITIAL_SLEEP" elapsed="$INITIAL_SLEEP" +pids="" while [ "$elapsed" -lt "$TIMEOUT" ]; do - if systemctl is-active --quiet salt-minion \ - && salt-call --local --timeout="$PING_TIMEOUT" --out=quiet test.ping >/dev/null 2>&1; then - echo "salt-minion ready after ${elapsed}s" - exit 0 + if pids=$(resolve_daemon_pids); then + # shellcheck disable=SC2086 + for pid in $pids; do + if instance_ready "$pid"; then + echo "salt-minion (pid ${pid}) ready after ${elapsed}s" + exit 0 + fi + done fi sleep 1 elapsed=$((elapsed + 1)) done -echo "salt-minion did not become ready within ${TIMEOUT}s" >&2 +mainpid=$(systemctl show -p MainPID --value salt-minion 2>/dev/null) +echo "salt-minion did not become ready within ${TIMEOUT}s (MainPID=${mainpid:-unknown}, candidate pids='${pids:-none}', log_gate=${USE_LOG_GATE}, socket_gate=${USE_SOCKET_GATE}, log_file=${LOG_FILE})" >&2 exit 1 From fbeac25ee9ca07341335e1551fe035303c958120 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 9 Jul 2026 16:57:19 -0400 Subject: [PATCH 154/219] so-salt-minion-check: highstate after minion restart When the minion is deemed hung and restarted, wait for it to become ready via so-salt-minion-wait, then kick off salt-call state.highstate (queued, backgrounded) so the box re-applies its states and recovers on its own rather than waiting for the next scheduled highstate. --- salt/common/tools/sbin_jinja/so-salt-minion-check | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin_jinja/so-salt-minion-check b/salt/common/tools/sbin_jinja/so-salt-minion-check index 1376193cc..281efbcb7 100755 --- a/salt/common/tools/sbin_jinja/so-salt-minion-check +++ b/salt/common/tools/sbin_jinja/so-salt-minion-check @@ -91,6 +91,8 @@ if [ $CURRENT_TIME -ge $((SYSTEM_START_TIME+$UPTIME_REQ)) ]; then logCmd "pkill -9 -ef /usr/bin/salt-minion" I log "starting salt-minion service" I logCmd "systemctl start salt-minion" I + log "waiting for salt-minion to become ready, then applying highstate in the background (queued)" I + nohup bash -c '/usr/sbin/so-salt-minion-wait; salt-call state.highstate queue=True' >> "/opt/so/log/salt/so-salt-minion-check" 2>&1 & else log "/opt/so/log/salt/healthcheck-state-apply last touched: `date -d @$LAST_HEALTHCHECK_STATE_APPLY` must be touched by `date -d @$THRESHOLD_DATE` to avoid salt-minion restart" I fi From 52885e28c5012116fad1d096caad45fa0c70856e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 9 Jul 2026 17:02:47 -0400 Subject: [PATCH 155/219] Name the reposync-side kernel repo securityonionkernelsync The reposync section in repodownload.conf and the client repo assigned in repo/client/oracle.sls both used the bare name securityonionkernel, colliding across the two roles. Rename the reposync-side section (and its --repoid, the so-repo-sync guard, and the so-kernel-upgrade presence check) to securityonionkernelsync, mirroring the existing securityonion/securityonionsync split for the main repo. The client repo stays securityonionkernel. Also give the section its own name=Security Onion Kernel Repo repo. --- salt/common/tools/sbin/so-kernel-upgrade | 10 +++++++--- salt/manager/files/repodownload.conf | 4 ++-- salt/manager/tools/sbin/so-repo-sync | 6 +++--- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/salt/common/tools/sbin/so-kernel-upgrade b/salt/common/tools/sbin/so-kernel-upgrade index 960505aee..e750c1a52 100755 --- a/salt/common/tools/sbin/so-kernel-upgrade +++ b/salt/common/tools/sbin/so-kernel-upgrade @@ -39,7 +39,11 @@ . /usr/sbin/so-common +# Client-side repo id (what dnf enables on this node, from repo/client/oracle.sls) vs the +# reposync-side section in repodownload.conf that the manager mirrors from (mirrors the +# securityonion/securityonionsync split for the main repo). KERNEL_REPO="securityonionkernel" +KERNEL_REPO_SYNC="securityonionkernelsync" KERNEL_PKG="kernel-uek" KERNEL_REPO_DIR="/nsm/kernelrepo" REPOSYNC_CONF="/opt/so/conf/reposync/repodownload.conf" @@ -95,15 +99,15 @@ kernelrepo_rpm_count() { # The kernel repo starts life as valid-but-empty (kernelrepo_init_empty in # salt/manager/init.sls) and is filled by so-repo-sync. During a soup, so-repo-sync runs -# BEFORE the highstate deploys the [securityonionkernel] section into repodownload.conf, so +# BEFORE the highstate deploys the [securityonionkernelsync] section into repodownload.conf, so # the first kernel-aware soup leaves the repo empty until the next nightly sync. sync_kernel_repo() { if is_airgap; then log "airgap install: $KERNEL_REPO_DIR is populated from the airgap ISO, not by so-repo-sync." return 1 fi - if ! grep -q "^\[${KERNEL_REPO}\]" "$REPOSYNC_CONF" 2>/dev/null; then - log "$REPOSYNC_CONF has no [${KERNEL_REPO}] section -- run a highstate to deploy it." + if ! grep -q "^\[${KERNEL_REPO_SYNC}\]" "$REPOSYNC_CONF" 2>/dev/null; then + log "$REPOSYNC_CONF has no [${KERNEL_REPO_SYNC}] section -- run a highstate to deploy it." return 1 fi diff --git a/salt/manager/files/repodownload.conf b/salt/manager/files/repodownload.conf index 67ae4b121..9c9cb5109 100644 --- a/salt/manager/files/repodownload.conf +++ b/salt/manager/files/repodownload.conf @@ -11,8 +11,8 @@ name=Security Onion Repo repo mirrorlist=file:///opt/so/conf/reposync/mirror.txt enabled=1 gpgcheck=1 -[securityonionkernel] -name=Security Onion Repo repo +[securityonionkernelsync] +name=Security Onion Kernel Repo repo mirrorlist=file:///opt/so/conf/reposync/mirror-kernel.txt enabled=1 gpgcheck=1 diff --git a/salt/manager/tools/sbin/so-repo-sync b/salt/manager/tools/sbin/so-repo-sync index 6c1b9d509..d6a290c25 100755 --- a/salt/manager/tools/sbin/so-repo-sync +++ b/salt/manager/tools/sbin/so-repo-sync @@ -17,9 +17,9 @@ createrepo /nsm/repo # The kernel repo section is deployed to repodownload.conf by the manager highstate, which # runs AFTER this script during soup. On the first upgrade to a kernel-aware version the # on-disk config still predates the section, so guard on its presence to avoid dnf's -# "Unknown repo: 'securityonionkernel'" aborting the sync (set -e). The next sync after the +# "Unknown repo: 'securityonionkernelsync'" aborting the sync (set -e). The next sync after the # highstate deploys the section will pick it up. -if grep -q '^\[securityonionkernel\]' /opt/so/conf/reposync/repodownload.conf; then - dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/ +if grep -q '^\[securityonionkernelsync\]' /opt/so/conf/reposync/repodownload.conf; then + dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernelsync --download-metadata -p /nsm/kernelrepo/ createrepo /nsm/kernelrepo fi From 89e6a746c8e730b4b5a51bf4c16365ca486657a4 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 10 Jul 2026 09:35:21 -0400 Subject: [PATCH 156/219] so-salt-minion-wait: wait for the restart job before reading MainPID Live testing on a standalone node found the previous commit still reported ready on the OUTGOING daemon. Reproduced by running the production sequence: systemctl restart --no-block salt-minion # what service.restart issues /usr/sbin/so-salt-minion-wait # what cmd.run then runs so-salt-minion-wait: gating on pid-tagged ready line ... plus master sockets salt-minion (pid 2750297) ready after 3s # 2750297 is the OLD child salt restarts this unit with --no-block -- _no_block_default() in salt/modules/systemd_service.py returns True when the unit is the salt-minion service -- so service.restart returns as soon as the job is enqueued. Measured on the host, systemd does not swap MainPID until ~7.3s later. Throughout that window the old daemon is still running, still holds its master sockets, and its own ready line is already in the log, so every gate passes on the instance that is about to be torn down. INITIAL_SLEEP=3 expired inside that window. Wait for systemd's job queue for the unit to drain before resolving MainPID. That is deterministic rather than a timing guess: the job exists from the moment --no-block returns until the new instance signals READY, and MainPID is new by the time it clears. Measured transition: t=0.0s job pending, child=OLD, sockets up, ready line present t=7.9s job drained, child=NEW, sockets up, ready line ABSENT t=10.7s ready line for NEW child appears <- script returns here The same run also confirms empirically why the log line is required in addition to the sockets: for 2.8s the new child has both master connections while _post_master_init() is still loading modules and compiling pillar, so a socket-only gate would return that much too early. Correct the comment claim from the previous commit. The --no-block restart is real; it lives in salt's systemd_service module, not in this repo, which is why searching the repo for it turned up nothing. --- salt/salt/minion/init.sls | 3 ++- salt/salt/tools/sbin/so-salt-minion-wait | 28 ++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/salt/salt/minion/init.sls b/salt/salt/minion/init.sls index 035d9e7f8..1f1ec8305 100644 --- a/salt/salt/minion/init.sls +++ b/salt/salt/minion/init.sls @@ -138,7 +138,8 @@ salt_minion_service: # /usr/sbin/so-salt-minion-wait (deployed by salt_sbin from salt/tools/sbin/); it keys the ready # line to the current daemon pid (resolved via systemd, not the pidfile) and corroborates with the # master req/publish sockets. set_log_levels above enforces the log_level_logfile: info that the -# ready line depends on. +# ready line depends on. salt restarts this unit with --no-block, so mod_watch returns while the old +# daemon is still up; the script waits for systemd's restart job to drain before it reads MainPID. wait_for_salt_minion_ready: cmd.run: - name: /usr/sbin/so-salt-minion-wait diff --git a/salt/salt/tools/sbin/so-salt-minion-wait b/salt/salt/tools/sbin/so-salt-minion-wait index 9fb2a41c7..984f144f3 100644 --- a/salt/salt/tools/sbin/so-salt-minion-wait +++ b/salt/salt/tools/sbin/so-salt-minion-wait @@ -26,6 +26,14 @@ # logs the ready line, while systemd's MainPID is the parent. During a restart the pidfile can still # name the OLD child, whose own ready line is already in the log -- matching it would report ready # instantly. Children of the current MainPID structurally exclude the old instance. +# +# That is only true once systemd has actually swapped MainPID. Salt restarts this unit with +# --no-block (salt/modules/systemd_service.py:_no_block_default returns True for salt-minion), so +# service.restart returns as soon as the job is enqueued and the state proceeds to run this script +# while the OLD daemon is still up -- observed at ~7s before MainPID flips. Reading MainPID in that +# window names the outgoing instance, which is still fully connected and has its own ready line, so +# every gate below would pass on the daemon that is about to die. Wait for systemd's job queue for +# the unit to drain first; that is the deterministic "the swap has happened" signal. . /usr/sbin/so-common @@ -76,6 +84,13 @@ else echo "so-salt-minion-wait: INFO file logging unavailable (log_level_logfile='${LOG_LEVEL_LOGFILE:-unset}'); gating on master sockets only" fi +# True while systemd still has a queued or running job for the unit, i.e. an in-flight --no-block +# restart. MainPID still names the outgoing daemon until this drains. The unit name is passed as a +# filter and grepped as well, so this stays correct if an older systemctl ignores the filter. +restart_pending() { + systemctl list-jobs --no-legend salt-minion.service 2>/dev/null | grep -q 'salt-minion\.service' +} + # Emit the pid(s) of the current daemon instance. systemd's MainPID is the parent keepalive process; # its child runs tune_in. Fall back to MainPID when there is no child (--disable-keepalive path). resolve_daemon_pids() { @@ -134,8 +149,16 @@ sleep "$INITIAL_SLEEP" elapsed="$INITIAL_SLEEP" pids="" +announced_pending=0 while [ "$elapsed" -lt "$TIMEOUT" ]; do - if pids=$(resolve_daemon_pids); then + if restart_pending; then + # An in-flight --no-block restart: MainPID still names the outgoing daemon. Evaluating now + # would bless the instance that is about to be torn down. + if [ "$announced_pending" -eq 0 ]; then + echo "so-salt-minion-wait: systemd restart job in flight; waiting for it to drain" + announced_pending=1 + fi + elif pids=$(resolve_daemon_pids); then # shellcheck disable=SC2086 for pid in $pids; do if instance_ready "$pid"; then @@ -149,5 +172,6 @@ while [ "$elapsed" -lt "$TIMEOUT" ]; do done mainpid=$(systemctl show -p MainPID --value salt-minion 2>/dev/null) -echo "salt-minion did not become ready within ${TIMEOUT}s (MainPID=${mainpid:-unknown}, candidate pids='${pids:-none}', log_gate=${USE_LOG_GATE}, socket_gate=${USE_SOCKET_GATE}, log_file=${LOG_FILE})" >&2 +restart_pending && pending=yes || pending=no +echo "salt-minion did not become ready within ${TIMEOUT}s (MainPID=${mainpid:-unknown}, candidate pids='${pids:-none}', restart_job_pending=${pending}, log_gate=${USE_LOG_GATE}, socket_gate=${USE_SOCKET_GATE}, log_file=${LOG_FILE})" >&2 exit 1 From 566f90a0c0e544260ce94349553099a90c357411 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 10 Jul 2026 10:32:25 -0400 Subject: [PATCH 157/219] toggle pg metrics --- salt/soc/defaults.map.jinja | 24 +++++++++++++++++------- salt/soc/merged.map.jinja | 19 +++++++++++++++++++ 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/salt/soc/defaults.map.jinja b/salt/soc/defaults.map.jinja index 2821bb8e5..93fb63efe 100644 --- a/salt/soc/defaults.map.jinja +++ b/salt/soc/defaults.map.jinja @@ -8,6 +8,7 @@ {% from 'docker/docker.map.jinja' import DOCKERMERGED -%} {% set INFLUXDB_TOKEN = salt['pillar.get']('influxdb:token') %} {% import_text 'influxdb/metrics_link.txt' as METRICS_LINK %} +{% from 'telegraf/map.jinja' import TELEGRAFMERGED %} {% for module, application_url in GLOBALS.application_urls.items() %} {% do SOCDEFAULTS.soc.config.server.modules[module].update({'hostUrl': application_url}) %} @@ -24,13 +25,22 @@ {% do SOCDEFAULTS.soc.config.server.modules.elastic.update({'username': GLOBALS.elasticsearch.auth.users.so_elastic_user.user, 'password': GLOBALS.elasticsearch.auth.users.so_elastic_user.pass}) %} -{% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %} -{% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'token': INFLUXDB_TOKEN}) %} -{% for tool in SOCDEFAULTS.soc.config.server.client.tools %} -{% if tool.name == "toolInfluxDb" and METRICS_LINK | length > 0 %} -{% do tool.update({'link': METRICS_LINK}) %} -{% endif %} -{% endfor %} +{% if TELEGRAFMERGED.output == 'POSTGRES' %} +{% for tool in SOCDEFAULTS.soc.config.server.client.tools %} +{% if tool.name == "toolInfluxDb" %} +{% do SOCDEFAULTS.soc.config.server.client.tools.remove(tool) %} +{% endif %} +{% endfor %} + +{% else %} +{% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %} +{% do SOCDEFAULTS.soc.config.server.modules.influxdb.update({'token': INFLUXDB_TOKEN}) %} +{% for tool in SOCDEFAULTS.soc.config.server.client.tools %} +{% if tool.name == "toolInfluxDb" and METRICS_LINK | length > 0 %} +{% do tool.update({'link': METRICS_LINK}) %} +{% endif %} +{% endfor %} +{% endif %} {% do SOCDEFAULTS.soc.config.server.modules.statickeyauth.update({'anonymousCidr': DOCKERMERGED.range, 'apiKey': pillar.sensoroni.config.sensoronikey}) %} diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index cfc0fafbd..a421711e2 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -7,6 +7,11 @@ {% from 'soc/defaults.map.jinja' import SOCDEFAULTS with context %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %} {% from 'manager/map.jinja' import MANAGERMERGED %} +{% from 'telegraf/map.jinja' import TELEGRAFMERGED %} +{%- set PG_ENTRY = salt['pillar.get']('telegraf:postgres_creds:' ~ grains.id, {}) %} +{%- set PG_USER = PG_ENTRY.get('user', '') %} +{%- set PG_PASS = PG_ENTRY.get('pass', '') %} + {% set DOCKER_EXTRA_HOSTS = ELASTICSEARCH_NODES %} {% do DOCKER_EXTRA_HOSTS.append({GLOBALS.influxdb_host:pillar.node_data[GLOBALS.influxdb_host].ip}) %} @@ -75,6 +80,20 @@ {% do SOCMERGED.config.server.update({'airgapEnabled': false}) %} {% endif %} +{# Define the postgresmetrics module if telegraf is setup to only use Postgres #} +{% if TELEGRAFMERGED.output != 'INFLUXDB' and PG_USER and PG_PASS %} +{% do SOCMERGED.config.server.modules.update({ + 'postgresmetrics': { + 'database': 'so_telegraf', + 'host': GLOBALS.manager_ip, + 'password': PG_PASS, + 'port': 5432, + 'sslMode': 'allow', + 'user': PG_USER, + } + }) %} +{% do SOCMERGED.config.server.modules.pop('influxdb') %} +{% endif %} {# Define the Detections custom ruleset that should always be present #} {% set CUSTOM_RULESET = { From 6fc0fd954c7562e94fbf751dbea8bab98564ad9b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 10 Jul 2026 10:35:37 -0400 Subject: [PATCH 158/219] clarify output annotation --- salt/telegraf/soc_telegraf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/telegraf/soc_telegraf.yaml b/salt/telegraf/soc_telegraf.yaml index 4b9a2e3d1..0decdd06a 100644 --- a/salt/telegraf/soc_telegraf.yaml +++ b/salt/telegraf/soc_telegraf.yaml @@ -5,7 +5,7 @@ telegraf: advanced: True helpLink: influxdb output: - description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation. + description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation. When set to BOTH, the grid screen's metrics are pulled from Postgres, and the InfluxDB tool link remains visible. When set to POSTGRES, the InfluxDB tool link is removed. options: - INFLUXDB - POSTGRES From 7d17784e96857e6ea6620364231eba2dcdff4751 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:50:21 -0500 Subject: [PATCH 159/219] status messages for so-elastic-fleet-setup --- salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup | 5 ++++- setup/so-setup | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index 2b677606a..0d7b99e48 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -250,6 +250,7 @@ if ! salt-call state.apply elasticfleet queue=True; then exit 1 fi +printf "\nRunning so-elastic-agent-gen-installers\n" # Generate installers & install Elastic Agent on the node for agent_gen_attempt in {1..3}; do if so-elastic-agent-gen-installers; then @@ -267,4 +268,6 @@ printf "\nApplying elasticfleet.install_agent_grid state\n" if ! salt-call state.apply elasticfleet.install_agent_grid queue=True; then printf "\nFailure(s) in elasticfleet.install_agent_grid state... Exiting...\n" exit 1 -fi \ No newline at end of file +fi + +printf "\nElastic Fleet setup completed successfully\n" \ No newline at end of file diff --git a/setup/so-setup b/setup/so-setup index a44934088..896505ba5 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -793,7 +793,7 @@ if ! [[ -f $install_opt_file ]]; then logCmd "so-soc-restart" title "Setting up Elastic Fleet" logCmd "salt-call state.apply elasticfleet.config" - if ! logCmd so-elastic-fleet-setup; then + if ! so-elastic-fleet-setup; then fail_setup "Failed to run so-elastic-fleet-setup" fi mark_setup_complete From 99e9fc1c3b6be005ebbb31bc053393a61271a240 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:50:55 -0500 Subject: [PATCH 160/219] ES 9.3.7 --- salt/elasticfleet/defaults.yaml | 1 - salt/elasticsearch/defaults.yaml | 2 +- salt/kibana/defaults.yaml | 2 +- salt/manager/tools/sbin/soup | 9 ++++++--- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 022600083..a3132d3f4 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -1,6 +1,5 @@ elasticfleet: enabled: False - patch_version: 9.3.3+build202604082258 # Elastic Agent specific patch release. enable_manager_output: True config: server: diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index f0b01b3ca..14d753c21 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,6 +1,6 @@ elasticsearch: enabled: false - version: 9.3.3 + version: 9.3.7 index_clean: true data_retention_method: DLM vm: diff --git a/salt/kibana/defaults.yaml b/salt/kibana/defaults.yaml index ecf56756b..2cf6fe92f 100644 --- a/salt/kibana/defaults.yaml +++ b/salt/kibana/defaults.yaml @@ -22,7 +22,7 @@ kibana: - default - file migrations: - discardCorruptObjects: "9.3.3" + discardCorruptObjects: "9.3.7" telemetry: enabled: False xpack: diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index e33c27f6f..79ea3967b 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -740,7 +740,6 @@ fix_logstash_0013_lumberjack_pipeline_name() { up_to_3.1.0() { ensure_postgres_local_pillar ensure_postgres_secret - determine_elastic_agent_upgrade elasticsearch_backup_index_templates # Clear existing component template state file. rm -f /opt/so/state/esfleet_component_templates.json @@ -888,6 +887,9 @@ update_kafka_metadata() { } up_to_3.2.0() { + # download 9.3.7 elastic agent packages + determine_elastic_agent_upgrade + fix_logstash_0013_lumberjack_pipeline_name pin_elasticsearch_data_retention_method @@ -901,7 +903,7 @@ post_to_3.2.0() { bootstrap_so_soc_database - # Including agent regen script here since it was missed in post_to_3.1.0 + # Generate 9.3.7 elastic agent installers echo "Regenerating Elastic Agent Installers" /sbin/so-elastic-agent-gen-installers @@ -1174,7 +1176,8 @@ verify_es_version_compatibility() { ["8.18.4"]="8.18.6 8.18.8 9.0.8" ["8.18.6"]="8.18.8 9.0.8" ["8.18.8"]="9.0.8" - ["9.0.8"]="9.3.3" + ["9.0.8"]="9.3.3 9.3.7" + ) # Elasticsearch MUST upgrade through these versions From 5af6c56996ac5287e892ac9780dd972ee9558355 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 10 Jul 2026 14:54:20 -0400 Subject: [PATCH 161/219] so-salt-minion-check: force highstate if none has completed since boot Add a second, independent trigger to the every-5-minute health check: if the host has been up >= 15 minutes (HIGHSTATE_UPTIME_REQ) and no highstate has completed since this boot (lasthighstate mtime older than boot time), run salt-call state.highstate. This recovers a host whose boot highstate (so-boot-highstate.service) failed or was skipped, even while the minion is otherwise healthy and touching state-apply-test. The new path deliberately does not enable highstate, so a soup-disabled highstate is respected and never forced mid-upgrade. A saltutil.running guard plus queue=True prevents stacking across successive cron runs, and a RESTARTED flag suppresses the new block when the existing minion-restart path already queued a highstate. --- .../tools/sbin_jinja/so-salt-minion-check | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/salt/common/tools/sbin_jinja/so-salt-minion-check b/salt/common/tools/sbin_jinja/so-salt-minion-check index 281efbcb7..936ecda9a 100755 --- a/salt/common/tools/sbin_jinja/so-salt-minion-check +++ b/salt/common/tools/sbin_jinja/so-salt-minion-check @@ -18,6 +18,7 @@ QUIET=false UPTIME_REQ=1800 #in seconds, how long the box has to be up before considering restarting salt-minion due to /opt/so/log/salt/state-apply-test not being touched +HIGHSTATE_UPTIME_REQ=900 #in seconds; if the box has been up this long and no highstate has completed since boot, force one CURRENT_TIME=$(date +%s) SYSTEM_START_TIME=$(date -d "$(> "/opt/so/log/salt/so-salt-minion-check" 2>&1 & + RESTARTED=true else log "/opt/so/log/salt/healthcheck-state-apply last touched: `date -d @$LAST_HEALTHCHECK_STATE_APPLY` must be touched by `date -d @$THRESHOLD_DATE` to avoid salt-minion restart" I fi else log "system uptime only $((CURRENT_TIME-SYSTEM_START_TIME)) seconds does not meet $UPTIME_REQ second requirement." I fi + +# If the host has been up long enough but no highstate has completed since this boot, +# force one. This recovers a host whose boot highstate (so-boot-highstate.service) failed +# or was skipped, even while the minion is otherwise healthy (touching state-apply-test). +# We deliberately do NOT enable highstate here: if soup has disabled it during an upgrade, +# Salt will refuse the highstate and we avoid forcing one mid-upgrade. +if ! $RESTARTED && [ $CURRENT_TIME -ge $((SYSTEM_START_TIME+HIGHSTATE_UPTIME_REQ)) ] && [ $LAST_HIGHSTATE_END -lt $SYSTEM_START_TIME ]; then + if salt-call --local saltutil.running 2>/dev/null | grep -q 'state.highstate'; then + log "no highstate has completed since boot, but one is already running; skipping" I + else + log "no highstate has completed since boot after $((CURRENT_TIME-SYSTEM_START_TIME))s uptime; applying highstate" E + nohup bash -c 'salt-call state.highstate -l info queue=True' >> "/opt/so/log/salt/so-salt-minion-check" 2>&1 & + fi +fi From ed533efb7b52bd3b6fadf38277124f3c811896eb Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 10 Jul 2026 15:53:49 -0400 Subject: [PATCH 162/219] so-salt-minion-check: tag and clarify log lines per check With two independent checks now writing to the same log, messages like "system uptime only N seconds does not meet 1800 second requirement" were ambiguous about which check they came from. Prefix every line with a [minion-restart-check] or [boot-highstate-check] tag and reword the uptime, threshold, and healthy messages to say what was evaluated and why it was skipped. Restructure the boot-highstate check from a nested if into an if/elif chain so each outcome (restart already queued, uptime too low, healthy, already running, forcing) logs its own reason instead of silently doing nothing. --- .../tools/sbin_jinja/so-salt-minion-check | 53 +++++++++++-------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/salt/common/tools/sbin_jinja/so-salt-minion-check b/salt/common/tools/sbin_jinja/so-salt-minion-check index 936ecda9a..c78846a5f 100755 --- a/salt/common/tools/sbin_jinja/so-salt-minion-check +++ b/salt/common/tools/sbin_jinja/so-salt-minion-check @@ -79,41 +79,48 @@ log "running so-salt-minion-check" RESTARTED=false +# Check 1 (minion-restart-check): if the minion has stopped applying states (the +# state-apply-test healthcheck file has gone stale), restart the salt-minion service. if [ $CURRENT_TIME -ge $((SYSTEM_START_TIME+$UPTIME_REQ)) ]; then if [ $THRESHOLD_DATE -le $CURRENT_TIME ]; then - log "salt-minion is unable to apply states" E - log "/opt/so/log/salt/healthcheck-state-apply not touched by required date: `date -d @$THRESHOLD_DATE`, last touched: `date -d @$LAST_HEALTHCHECK_STATE_APPLY`" I - log "last highstate completed at `date -d @$LAST_HIGHSTATE_END`" I - log "checking if any jobs are running" I + log "[minion-restart-check] salt-minion is unable to apply states; restarting salt-minion" E + log "[minion-restart-check] state-apply-test not touched by required date `date -d @$THRESHOLD_DATE`, last touched `date -d @$LAST_HEALTHCHECK_STATE_APPLY`" I + log "[minion-restart-check] last highstate completed at `date -d @$LAST_HIGHSTATE_END`" I + log "[minion-restart-check] checking if any jobs are running" I logCmd "salt-call --local saltutil.running" I - log "ensure salt.minion-state-apply-test is enabled" I + log "[minion-restart-check] ensure salt.minion-state-apply-test is enabled" I logCmd "salt-call state.enable salt.minion-state-apply-test" I - log "ensure highstate is enabled" I + log "[minion-restart-check] ensure highstate is enabled" I logCmd "salt-call state.enable highstate" I - log "killing all salt-minion processes" I + log "[minion-restart-check] killing all salt-minion processes" I logCmd "pkill -9 -ef /usr/bin/salt-minion" I - log "starting salt-minion service" I + log "[minion-restart-check] starting salt-minion service" I logCmd "systemctl start salt-minion" I - log "waiting for salt-minion to become ready, then applying highstate in the background (queued)" I + log "[minion-restart-check] waiting for salt-minion to become ready, then applying highstate in the background (queued)" I nohup bash -c '/usr/sbin/so-salt-minion-wait; salt-call state.highstate queue=True' >> "/opt/so/log/salt/so-salt-minion-check" 2>&1 & RESTARTED=true else - log "/opt/so/log/salt/healthcheck-state-apply last touched: `date -d @$LAST_HEALTHCHECK_STATE_APPLY` must be touched by `date -d @$THRESHOLD_DATE` to avoid salt-minion restart" I + log "[minion-restart-check] healthy: state-apply-test last touched `date -d @$LAST_HEALTHCHECK_STATE_APPLY`, must go stale past `date -d @$THRESHOLD_DATE` to trigger a salt-minion restart" I fi else - log "system uptime only $((CURRENT_TIME-SYSTEM_START_TIME)) seconds does not meet $UPTIME_REQ second requirement." I + log "[minion-restart-check] skipped: system uptime $((CURRENT_TIME-SYSTEM_START_TIME))s is below the ${UPTIME_REQ}s minimum required before a salt-minion restart" I fi -# If the host has been up long enough but no highstate has completed since this boot, -# force one. This recovers a host whose boot highstate (so-boot-highstate.service) failed -# or was skipped, even while the minion is otherwise healthy (touching state-apply-test). -# We deliberately do NOT enable highstate here: if soup has disabled it during an upgrade, -# Salt will refuse the highstate and we avoid forcing one mid-upgrade. -if ! $RESTARTED && [ $CURRENT_TIME -ge $((SYSTEM_START_TIME+HIGHSTATE_UPTIME_REQ)) ] && [ $LAST_HIGHSTATE_END -lt $SYSTEM_START_TIME ]; then - if salt-call --local saltutil.running 2>/dev/null | grep -q 'state.highstate'; then - log "no highstate has completed since boot, but one is already running; skipping" I - else - log "no highstate has completed since boot after $((CURRENT_TIME-SYSTEM_START_TIME))s uptime; applying highstate" E - nohup bash -c 'salt-call state.highstate -l info queue=True' >> "/opt/so/log/salt/so-salt-minion-check" 2>&1 & - fi +# Check 2 (boot-highstate-check): if the host has been up long enough but no highstate +# has completed since this boot, force one. This recovers a host whose boot highstate +# (so-boot-highstate.service) failed or was skipped, even while the minion is otherwise +# healthy (touching state-apply-test). We deliberately do NOT enable highstate here: if +# soup has disabled it during an upgrade, Salt will refuse the highstate and we avoid +# forcing one mid-upgrade. +if $RESTARTED; then + log "[boot-highstate-check] skipped: minion-restart-check already queued a highstate this run" I +elif [ $CURRENT_TIME -lt $((SYSTEM_START_TIME+HIGHSTATE_UPTIME_REQ)) ]; then + log "[boot-highstate-check] skipped: system uptime $((CURRENT_TIME-SYSTEM_START_TIME))s is below the ${HIGHSTATE_UPTIME_REQ}s minimum required before forcing a highstate" I +elif [ $LAST_HIGHSTATE_END -ge $SYSTEM_START_TIME ]; then + log "[boot-highstate-check] healthy: a highstate completed at `date -d @$LAST_HIGHSTATE_END`, after this boot at `date -d @$SYSTEM_START_TIME`" I +elif salt-call --local saltutil.running 2>/dev/null | grep -q 'state.highstate'; then + log "[boot-highstate-check] no highstate has completed since boot, but one is already running; skipping" I +else + log "[boot-highstate-check] no highstate has completed since boot after $((CURRENT_TIME-SYSTEM_START_TIME))s uptime; applying highstate" E + nohup bash -c 'salt-call state.highstate -l info queue=True' >> "/opt/so/log/salt/so-salt-minion-check" 2>&1 & fi From 87a56396437f6aacc2a97251de0410fceeca9d07 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 13 Jul 2026 13:05:25 -0500 Subject: [PATCH 163/219] pipeline updates --- .../elastic-defend-endpoints.json | 2 +- .../grid-nodes_general/import-evtx-logs.json | 2 +- ...nse.log-1.25.2 => logs-pfsense.log-1.25.4} | 20 +++++++++---------- ...icata => logs-pfsense.log-1.25.4-suricata} | 0 4 files changed, 12 insertions(+), 12 deletions(-) rename salt/elasticsearch/files/ingest/{logs-pfsense.log-1.25.2 => logs-pfsense.log-1.25.4} (96%) rename salt/elasticsearch/files/ingest/{logs-pfsense.log-1.25.2-suricata => logs-pfsense.log-1.25.4-suricata} (100%) diff --git a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json index c27da26f7..7d64fd1ab 100644 --- a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json +++ b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json @@ -5,7 +5,7 @@ "package": { "name": "endpoint", "title": "Elastic Defend", - "version": "9.3.0", + "version": "9.3.1", "requires_root": true }, "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index 32d210172..be05d59e0 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -29,7 +29,7 @@ "\\.gz$" ], "include_files": [], - "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.15.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.8.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.15.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.15.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.8.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import", + "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.20.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.8.3\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.20.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.20.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.8.3\n- add_fields:\n target: data_stream\n fields:\n dataset: import", "tags": [ "import" ], diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2 b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.4 similarity index 96% rename from salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2 rename to salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.4 index 1ea828514..c01383fde 100644 --- a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2 +++ b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.4 @@ -118,70 +118,70 @@ { "pipeline": { "tag": "pipeline_e16851a7", - "name": "logs-pfsense.log-1.25.2-firewall", + "name": "logs-pfsense.log-1.25.4-firewall", "if": "ctx.event.provider == 'filterlog'" } }, { "pipeline": { "tag": "pipeline_828590b5", - "name": "logs-pfsense.log-1.25.2-openvpn", + "name": "logs-pfsense.log-1.25.4-openvpn", "if": "ctx.event.provider == 'openvpn'" } }, { "pipeline": { "tag": "pipeline_9d37039c", - "name": "logs-pfsense.log-1.25.2-ipsec", + "name": "logs-pfsense.log-1.25.4-ipsec", "if": "ctx.event.provider == 'charon'" } }, { "pipeline": { "tag": "pipeline_ad56bbca", - "name": "logs-pfsense.log-1.25.2-dhcp", + "name": "logs-pfsense.log-1.25.4-dhcp", "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\", \"dnsmasq-dhcp\"].contains(ctx.event.provider)" } }, { "pipeline": { "tag": "pipeline_dd85553d", - "name": "logs-pfsense.log-1.25.2-unbound", + "name": "logs-pfsense.log-1.25.4-unbound", "if": "ctx.event.provider == 'unbound'" } }, { "pipeline": { "tag": "pipeline_720ed255", - "name": "logs-pfsense.log-1.25.2-haproxy", + "name": "logs-pfsense.log-1.25.4-haproxy", "if": "ctx.event.provider == 'haproxy'" } }, { "pipeline": { "tag": "pipeline_456beba5", - "name": "logs-pfsense.log-1.25.2-php-fpm", + "name": "logs-pfsense.log-1.25.4-php-fpm", "if": "ctx.event.provider == 'php-fpm'" } }, { "pipeline": { "tag": "pipeline_a0d89375", - "name": "logs-pfsense.log-1.25.2-squid", + "name": "logs-pfsense.log-1.25.4-squid", "if": "ctx.event.provider == 'squid'" } }, { "pipeline": { "tag": "pipeline_c2f1ed55", - "name": "logs-pfsense.log-1.25.2-snort", + "name": "logs-pfsense.log-1.25.4-snort", "if": "ctx.event.provider == 'snort'" } }, { "pipeline": { "tag":"pipeline_33db1c9e", - "name": "logs-pfsense.log-1.25.2-suricata", + "name": "logs-pfsense.log-1.25.4-suricata", "if": "ctx.event.provider == 'suricata'" } }, diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2-suricata b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.4-suricata similarity index 100% rename from salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2-suricata rename to salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.4-suricata From 2cd889782db10210f463bc6aa30586a0f7b83bf9 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 13 Jul 2026 13:16:12 -0500 Subject: [PATCH 164/219] soup es check for 9.3.7 --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 79ea3967b..a87dfecfc 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1177,7 +1177,7 @@ verify_es_version_compatibility() { ["8.18.6"]="8.18.8 9.0.8" ["8.18.8"]="9.0.8" ["9.0.8"]="9.3.3 9.3.7" - + ["9.3.3"]="9.3.7" ) # Elasticsearch MUST upgrade through these versions From e42f7cd6fca72c23a46fe2940622c36d5e176ebb Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 14 Jul 2026 13:33:57 -0400 Subject: [PATCH 165/219] add restart policy to so-postgres --- salt/postgres/enabled.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/postgres/enabled.sls b/salt/postgres/enabled.sls index 20d256ae8..b0eadd205 100644 --- a/salt/postgres/enabled.sls +++ b/salt/postgres/enabled.sls @@ -19,6 +19,7 @@ include: so-postgres: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-postgres:{{ GLOBALS.so_version }} + - restart_policy: unless-stopped - hostname: so-postgres - networks: - sobridge: From 405dc52587fe417c767e7005cfc177a444cf8296 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 14 Jul 2026 13:34:19 -0400 Subject: [PATCH 166/219] move up restart policy for so-kratos --- salt/kratos/enabled.sls | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/kratos/enabled.sls b/salt/kratos/enabled.sls index 7dd13732e..ab13d759f 100644 --- a/salt/kratos/enabled.sls +++ b/salt/kratos/enabled.sls @@ -15,6 +15,7 @@ include: so-kratos: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kratos:{{ GLOBALS.so_version }} + - restart_policy: unless-stopped - hostname: kratos - name: so-kratos - networks: @@ -51,8 +52,6 @@ so-kratos: - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }} {% endfor %} {% endif %} - # Intentionally unless-stopped -- matches the fleet default. - - restart_policy: unless-stopped - watch: - file: kratosschema - file: kratosconfig From 63d4061500cd7c4576d64a1b0c0d5cf837d92cb4 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 14 Jul 2026 13:52:32 -0400 Subject: [PATCH 167/219] prevent login redirect to any API url --- salt/nginx/etc/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index b7a70da2b..3f74c411c 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -399,7 +399,7 @@ http { error_page 429 = @error429; location @error401 { - if ($request_uri ~* (^/api/.*|^/connect/.*|^/oauth2/.*|^/.*\.map$)) { + if ($request_uri ~* (^.*/api/.*|^/connect/.*|^/oauth2/.*|^/.*\.map$)) { return 401; } From 8b488f92267fcb319f8ee7e2f34c3245270d110e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 14 Jul 2026 17:30:58 -0400 Subject: [PATCH 168/219] soup: make failed upgrades and hotfixes resumable A failed highstate mid-upgrade left /etc/soversion already advanced to the target version (the highstate stamps it from the pillar via the soversionfile state), so a re-run of soup saw INSTALLEDVERSION == NEWVERSION and reported "already running the latest version", stranding the box with post-upgrade steps never run. Introduce /etc/sopostversion, a soup-owned marker (no salt state manages it) that records post-upgrade walk progress. It is seeded from the pre-upgrade version before the highstate, advanced after each post_to_* step, and removed on successful completion. upgrade_check treats a leftover marker as "upgrade not finished" and resumes the remaining post steps instead of bailing. Also fix the hotfix path: /etc/sohotfix was written before the hotfix highstate, so a failed hotfix highstate looked already-applied on re-run. Since no salt state manages /etc/sohotfix, defer its write (update_version) until after the highstate succeeds so it is an honest completion marker. --- salt/manager/tools/sbin/soup | 46 ++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index a87dfecfc..18cf6a732 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -12,7 +12,17 @@ UPDATE_DIR=/tmp/sogh/securityonion DEFAULT_SALT_DIR=/opt/so/saltstack/default INSTALLEDVERSION=$(cat /etc/soversion) -POSTVERSION=$INSTALLEDVERSION +# /etc/sopostversion is a soup-owned marker (no salt state manages it) tracking how +# far the post-upgrade walk has progressed. Its presence means a prior upgrade did +# not finish its post-upgrade steps; its contents are the resume point. It is read +# here before preupgrade_changes mutates INSTALLEDVERSION and before any highstate +# stamps /etc/soversion from the pillar. +POSTVERSION_FILE=/etc/sopostversion +if [ -f "$POSTVERSION_FILE" ]; then + POSTVERSION=$(cat "$POSTVERSION_FILE") +else + POSTVERSION=$INSTALLEDVERSION +fi INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}') BATCHSIZE=5 SOUP_LOG=/root/soup.log @@ -414,6 +424,13 @@ preupgrade_changes() { true } +set_postversion() { + # Persist post-upgrade walk progress so an interrupted upgrade can resume the + # remaining steps on the next soup run (see /etc/sopostversion handling). + POSTVERSION="$1" + echo "$POSTVERSION" > "$POSTVERSION_FILE" +} + postupgrade_changes() { # This function is to add any new pillar items if needed. echo "Running post upgrade processes." @@ -421,6 +438,8 @@ postupgrade_changes() { [[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0 [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0 [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0 + # All applicable post-upgrade steps completed; clear the resume marker. + rm -f "$POSTVERSION_FILE" true } @@ -513,7 +532,7 @@ post_to_3.0.0() { # convert yes/no in suricata pillars to true/false convert_suricata_yes_no - POSTVERSION=3.0.0 + set_postversion 3.0.0 } ### 3.0.0 End ### @@ -776,7 +795,7 @@ post_to_3.1.0() { # Check for unhealthy / unauthorized integration transform jobs and attempt reauthorizations check_transform_health_and_reauthorize || true - POSTVERSION=3.1.0 + set_postversion 3.1.0 } ### 3.1.0 End ### @@ -911,7 +930,7 @@ post_to_3.2.0() { update_kafka_metadata "4.3" - POSTVERSION=3.2.0 + set_postversion 3.2.0 } ### 3.2.0 End ### @@ -1063,6 +1082,14 @@ upgrade_check() { fi [[ -f /etc/sohotfix ]] && CURRENTHOTFIX=$(cat /etc/sohotfix) if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then + # A leftover post-version marker means a previous upgrade to this version + # advanced /etc/soversion (the highstate stamps it from the pillar) but did not + # finish its post-upgrade steps. Resume the upgrade instead of reporting "latest". + if [ -f "$POSTVERSION_FILE" ] && [ "$(cat "$POSTVERSION_FILE")" != "$NEWVERSION" ]; then + echo "A previous upgrade to $NEWVERSION did not complete its post-upgrade steps; resuming." + is_hotfix=false + return 0 + fi echo "Checking to see if there are hotfixes needed" if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then echo "You are already running the latest version of Security Onion." @@ -1814,9 +1841,14 @@ main() { create_local_directories "/opt/so/saltstack/default" apply_hotfix echo "Hotfix applied" - update_version enable_highstate highstate + # Record the hotfix only after the highstate succeeds. /etc/sohotfix is written + # solely by soup (no salt state manages it), so deferring the write means a failed + # hotfix highstate leaves the old hotfix value and re-running soup re-applies it, + # rather than reporting "already latest". The soversion/pillar writes in + # update_version are no-ops here since the version is unchanged for a hotfix. + update_version else echo "" echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION." @@ -1873,6 +1905,10 @@ main() { copy_new_files echo "" create_local_directories "/opt/so/saltstack/default" + # Seed the resume marker before the highstate stamps /etc/soversion to the new + # version, so an interrupted upgrade is detectable as "not finished" on re-run. + # POSTVERSION still holds the pre-upgrade (or prior resume) version here. + [ -f "$POSTVERSION_FILE" ] || echo "$POSTVERSION" > "$POSTVERSION_FILE" update_version echo "" From 618712469e55e6e2d55025de095fa5106ef4b6f1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 15 Jul 2026 09:42:59 -0400 Subject: [PATCH 169/219] soup: clearly report incomplete upgrades on trap exit When soup fails via the EXIT trap after it has begun modifying the system, print a prominent UPGRADE INCOMPLETE banner instructing the user to run soup again to resume and complete the update. Gated on a new SOUP_UPGRADE_STARTED flag set at the start of the hotfix and upgrade branches, so pre-flight gate failures (ES compatibility, disk, network) that abort before any changes are made do not show it. --- salt/manager/tools/sbin/soup | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 18cf6a732..4fcdfebf9 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -33,6 +33,10 @@ NOTIFYCUSTOMELASTICCONFIG=false TOPFILE=/opt/so/saltstack/default/salt/top.sls BACKUPTOPFILE=/opt/so/saltstack/default/salt/top.sls.backup SALTUPGRADED=false +# Set true once soup begins modifying the system (past the pre-flight checks), so the +# EXIT trap can tell the user the update did not finish and must be re-run. Only the +# pre-flight gates (ES compatibility, disk, network) fail before this is set. +SOUP_UPGRADE_STARTED=false SALT_CLOUD_INSTALLED=false SALT_CLOUD_CONFIGURED=false # Check if salt-cloud is installed @@ -133,6 +137,28 @@ check_err() { echo "SOUP XTRACE debug log (if enabled) at $SOUP_DEBUG_LOG. Re-run soup with SOUP_DEBUG=1 to create $SOUP_DEBUG_LOG" + # If soup had already started modifying the system, make it unmistakable that the + # update is incomplete and must be re-run. soup is resumable: a version upgrade + # picks up from the /etc/sopostversion marker, and a hotfix re-applies because + # /etc/sohotfix is only advanced after a successful highstate. + if [[ "$SOUP_UPGRADE_STARTED" == "true" ]]; then + echo "" + echo "==============================================================================" + echo " UPGRADE INCOMPLETE" + echo "==============================================================================" + echo " This soup run did NOT finish. Your Security Onion installation may be in a" + echo " partially-updated state and is not yet fully upgraded." + echo "" + echo " Review the error above and $SOUP_LOG, resolve the underlying problem, then" + echo " run soup again to resume and complete the update:" + echo "" + echo " sudo soup" + echo "" + echo " soup is resumable -- re-running it continues from where this run stopped." + echo "==============================================================================" + echo "" + fi + exit $exit_code fi @@ -1831,6 +1857,7 @@ main() { fi if [ "$is_hotfix" == "true" ]; then + SOUP_UPGRADE_STARTED=true echo "Applying $HOTFIXVERSION hotfix" # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars if [[ ! "$MINION_ROLE" == "import" ]]; then @@ -1850,6 +1877,7 @@ main() { # update_version are no-ops here since the version is unchanged for a hotfix. update_version else + SOUP_UPGRADE_STARTED=true echo "" echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION." echo "" From fee62ab976cfabe96390d752bcf4b42dd6aed337 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 15 Jul 2026 10:49:27 -0400 Subject: [PATCH 170/219] change restart_policy to unless-stopped --- salt/strelka/backend/enabled.sls | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/salt/strelka/backend/enabled.sls b/salt/strelka/backend/enabled.sls index 8c71bdf68..e62349f70 100644 --- a/salt/strelka/backend/enabled.sls +++ b/salt/strelka/backend/enabled.sls @@ -15,6 +15,7 @@ include: strelka_backend: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-strelka-backend:{{ GLOBALS.so_version }} + - restart_policy: unless-stopped - binds: - /opt/so/conf/strelka/backend/:/etc/strelka/:ro - /opt/so/conf/strelka/rules/compiled/:/etc/yara/:ro @@ -47,11 +48,6 @@ strelka_backend: - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }} {% endfor %} {% endif %} - # Intentionally `on-failure` (not unless-stopped) -- strelka backend shuts - # down cleanly during rule reloads and we do not want those clean exits to - # trigger an auto-restart. Do not homogenize; see the container - # auto-restart section of the plan. - - restart_policy: on-failure - watch: - file: strelkasensorcompiledrules - file: backend_backend_config From 5178d5fd0e91ec9a0cd48a4361313fb4c03fa9f8 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 15 Jul 2026 11:04:51 -0400 Subject: [PATCH 171/219] ensure restart_policy is below image --- salt/hydra/enabled.sls | 3 +-- salt/registry/enabled.sls | 7 +++---- salt/suricata/enabled.sls | 2 +- salt/zeek/enabled.sls | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/salt/hydra/enabled.sls b/salt/hydra/enabled.sls index b19f9a80d..74106f550 100644 --- a/salt/hydra/enabled.sls +++ b/salt/hydra/enabled.sls @@ -22,6 +22,7 @@ include: so-hydra: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-hydra:{{ GLOBALS.so_version }} + - restart_policy: unless-stopped - hostname: hydra - name: so-hydra - networks: @@ -58,8 +59,6 @@ so-hydra: - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }} {% endfor %} {% endif %} - # Intentionally unless-stopped -- matches the fleet default. - - restart_policy: unless-stopped - watch: - file: hydraconfig - require: diff --git a/salt/registry/enabled.sls b/salt/registry/enabled.sls index 7b039b313..b2bfbc56d 100644 --- a/salt/registry/enabled.sls +++ b/salt/registry/enabled.sls @@ -17,14 +17,13 @@ include: so-dockerregistry: docker_container.running: - image: ghcr.io/security-onion-solutions/registry:3.1.1 + # Intentionally `always`-- registry is critical and must + # come back up even if it was manually stopped. + - restart_policy: always - hostname: so-registry - networks: - sobridge: - ipv4_address: {{ DOCKERMERGED.containers['so-dockerregistry'].ip }} - # Intentionally `always` (not unless-stopped) -- registry is critical infra - # and must come back up even if it was manually stopped. Do not homogenize - # to unless-stopped; see the container auto-restart section of the plan. - - restart_policy: always - port_bindings: {% for BINDING in DOCKERMERGED.containers['so-dockerregistry'].port_bindings %} - {{ BINDING }} diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index 53f367971..d9206798e 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -17,8 +17,8 @@ include: so-suricata: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-suricata:{{ GLOBALS.so_version }} - - privileged: True - restart_policy: unless-stopped + - privileged: True - environment: - INTERFACE={{ GLOBALS.sensor.interface }} {% if DOCKERMERGED.containers['so-suricata'].extra_env %} diff --git a/salt/zeek/enabled.sls b/salt/zeek/enabled.sls index 355e555b3..ec01693d7 100644 --- a/salt/zeek/enabled.sls +++ b/salt/zeek/enabled.sls @@ -16,9 +16,9 @@ include: so-zeek: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-zeek:{{ GLOBALS.so_version }} + - restart_policy: unless-stopped - start: True - privileged: True - - restart_policy: unless-stopped {% if DOCKERMERGED.containers['so-zeek'].ulimits %} - ulimits: {% for ULIMIT in DOCKERMERGED.containers['so-zeek'].ulimits %} From be7d8a2aa77d06ef5773b27ac2fba3fe05addaf9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 15 Jul 2026 11:35:04 -0400 Subject: [PATCH 172/219] soup: make partial-upgrade state clear and avoid re-running completed upgrades After a partial upgrade, /etc/soversion already reads the target version, so soup's startup line "Found that Security Onion X is currently installed" made it look finished even as soup resumed. When a resume marker is present and differs from the installed version, print an explicit NOTE that the grid is only partially upgraded and this run will resume and complete it. Also clear any stale resume marker in the already-latest path so a successfully completed upgrade is never mistaken for a partial one and re-run on a later invocation (the marker is normally removed at the end of postupgrade_changes; this is a belt-and-suspenders guard). --- salt/manager/tools/sbin/soup | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 4fcdfebf9..1200fc423 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1118,6 +1118,10 @@ upgrade_check() { fi echo "Checking to see if there are hotfixes needed" if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then + # Reaching here means we are at the target version and NOT resuming (the resume + # check above returned otherwise). Clear any stale resume marker so a completed + # upgrade is never mistaken for a partial one and re-run on a later invocation. + rm -f "$POSTVERSION_FILE" echo "You are already running the latest version of Security Onion." exit 0 else @@ -1813,6 +1817,15 @@ main() { set_minionid MINION_ROLE=$(lookup_role) echo "Found that Security Onion $INSTALLEDVERSION is currently installed." + # /etc/soversion is stamped to the target version before the upgrade fully + # completes, so a lingering resume marker means this grid is only partially + # upgraded even though the line above shows the target version. Make that explicit + # so it is not mistaken for a finished upgrade. + if [ -f "$POSTVERSION_FILE" ] && [ "$(cat "$POSTVERSION_FILE")" != "$INSTALLEDVERSION" ]; then + echo "" + echo "NOTE: A previous upgrade to $INSTALLEDVERSION did not finish. This grid is" + echo " partially upgraded and this soup run will resume and complete it." + fi echo "" check_minimum_version From bd70dd53fb8e9e3acdd68bd7c023327b57f5e270 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 15 Jul 2026 12:00:11 -0400 Subject: [PATCH 173/219] soup: add cluster-health and Fleet Server pre-flight checks Before making any changes, verify the grid is in a good state: - check_cluster_health: waits for Elasticsearch to reach at least 'yellow' (blocks only on red/unreachable, since yellow is normal), modeled on the wait in so-elasticsearch-roles-load. - check_fleet_server: confirms the Fleet Server status API returns HTTP 200, modeled on the wait_for_so-elastic-fleet state in elasticfleet/enabled.sls. Both run alongside the existing check_pillar_items (manager pillar render) and verify_es_version_compatibility, before soup modifies anything, so a failure exits cleanly with an actionable message and no partial changes. Valid on all manager roles soup runs on (eval/standalone/manager/managerhype/managersearch/ import), which all run Elasticsearch and the Fleet Server. --- salt/manager/tools/sbin/soup | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 1200fc423..37e550db2 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -327,6 +327,31 @@ check_pillar_items() { fi } +check_cluster_health() { + echo "Checking Elasticsearch cluster health." + # Block only if the cluster cannot reach at least 'yellow' status (i.e. it is red or + # unreachable); 'yellow' is normal for many Security Onion deployments. Modeled on the + # wait used in so-elasticsearch-roles-load. + if so-elasticsearch-query "_cluster/health?wait_for_status=yellow&timeout=120s" --fail > /dev/null 2>&1; then + printf "\nThe Elasticsearch cluster is healthy. We can proceed with SOUP.\n\n" + else + printf "\nThe Elasticsearch cluster is not healthy (it did not reach at least 'yellow' status). Please resolve the cluster health issue before running SOUP again.\n\n" + exit 0 + fi +} + +check_fleet_server() { + echo "Checking that Elastic Fleet Server is responding." + # Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls, + # which waits for HTTP 200 from the Fleet Server status API. + if curl -sk --fail --retry 3 --retry-delay 10 --max-time 30 "https://localhost:8220/api/status" > /dev/null 2>&1; then + printf "\nElastic Fleet Server is responding. We can proceed with SOUP.\n\n" + else + printf "\nElastic Fleet Server is not responding at https://localhost:8220/api/status. Please ensure Elastic Fleet is healthy before running SOUP again.\n\n" + exit 0 + fi +} + check_saltmaster_status() { set +e echo "Waiting on the Salt Master service to be ready." @@ -1854,6 +1879,12 @@ main() { echo "Verifying Elasticsearch version compatibility across the grid before upgrading." verify_es_version_compatibility + # Pre-flight health checks: confirm the grid is in a good state before we change + # anything. These run before any modifications, so a failure exits cleanly and the + # operator can fix the issue and re-run soup. + check_cluster_health + check_fleet_server + echo "Checking for Salt Master and Minion updates." upgrade_check_salt set -e From 186bf86e996f17f00d4af0eb7292888881f7cba3 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 15 Jul 2026 12:14:18 -0400 Subject: [PATCH 174/219] soup: require green Elasticsearch cluster before upgrading Change the pre-flight cluster-health gate to wait_for_status=green instead of yellow, so soup only proceeds when the cluster is fully green. --- salt/manager/tools/sbin/soup | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 37e550db2..ec2dcaed5 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -329,13 +329,12 @@ check_pillar_items() { check_cluster_health() { echo "Checking Elasticsearch cluster health." - # Block only if the cluster cannot reach at least 'yellow' status (i.e. it is red or - # unreachable); 'yellow' is normal for many Security Onion deployments. Modeled on the - # wait used in so-elasticsearch-roles-load. - if so-elasticsearch-query "_cluster/health?wait_for_status=yellow&timeout=120s" --fail > /dev/null 2>&1; then - printf "\nThe Elasticsearch cluster is healthy. We can proceed with SOUP.\n\n" + # Require a 'green' cluster before upgrading; anything less (yellow, red, or + # unreachable) blocks. Modeled on the wait used in so-elasticsearch-roles-load. + if so-elasticsearch-query "_cluster/health?wait_for_status=green&timeout=120s" --fail > /dev/null 2>&1; then + printf "\nThe Elasticsearch cluster is healthy (green). We can proceed with SOUP.\n\n" else - printf "\nThe Elasticsearch cluster is not healthy (it did not reach at least 'yellow' status). Please resolve the cluster health issue before running SOUP again.\n\n" + printf "\nThe Elasticsearch cluster is not green. Please resolve the cluster health issue so the cluster is green before running SOUP again.\n\n" exit 0 fi } From 376607d292852ab238a2a6fee1c4dffa6bae571c Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 15 Jul 2026 14:52:27 -0400 Subject: [PATCH 175/219] so-status: show container status while system is starting Containers now start on boot via restart_policy unless-stopped, so a highstate is no longer required to bring them up. Gather and display the container table even when no highstate has completed since reboot, while still warning the user. The exit code / JSON status_code stays 2 in that state so SOC's Grid continues to show the restarting message unchanged. --- salt/common/tools/sbin/so-status | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/salt/common/tools/sbin/so-status b/salt/common/tools/sbin/so-status index f4abd8aa3..b50cdff14 100755 --- a/salt/common/tools/sbin/so-status +++ b/salt/common/tools/sbin/so-status @@ -74,13 +74,13 @@ def output(options, console, code, data): summary = { "status_code": code, "containers": data } print(json.dumps(summary)) elif "-q" not in options: - if code == 2: - console.print(" [bold yellow]:hourglass: [bold white]System appears to be starting. No highstate has completed since the system was restarted.") - elif code == 99: + if code == 99: console.print(" [bold red]:exclamation: [bold white]Installation does not appear to be complete. A highstate has not fully completed.") elif code == 100: console.print(" [bold red]:exclamation: [bold white]Installation encountered errors.") else: + if code == 2: + console.print(" [bold yellow]:hourglass: [bold white]System appears to be starting. No highstate has completed since the system was restarted. Container status is shown below.") table = Table(title = "Security Onion Status", show_edge = False, safe_box = True, box = box.MINIMAL) table.add_column("Container", justify="right", style="white", no_wrap=True) table.add_column("Status", justify="left", style="green", no_wrap=True) @@ -154,8 +154,14 @@ def check_status(options, console): code = check_installation_status(options, console) if code == 0: code = check_system_status(options, console) - if code == 0: - code, container_list = check_container_status(options, console) + # Containers now start on boot without a highstate, so gather/display their + # status even when the system is still "starting" (code 2). Keep the starting + # code as the exit/status_code so SOC keeps showing the "restarting" message + # on the Grid until a highstate completes. + if code == 0 or code == 2: + container_code, container_list = check_container_status(options, console) + if code == 0: + code = container_code output(options, console, code, container_list) return code @@ -180,4 +186,3 @@ def main(): if __name__ == "__main__": main() - From 23c74f172764758bb2478e50565d0dd9328250fb Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 15 Jul 2026 15:24:33 -0400 Subject: [PATCH 176/219] remove installation of pyinotify --- salt/salt/master.sls | 1 - salt/salt/master/pyinotify.sls | 20 ------------------ .../pyinotify/pyinotify-0.9.6.tar.gz | Bin 60998 -> 0 bytes 3 files changed, 21 deletions(-) delete mode 100644 salt/salt/master/pyinotify.sls delete mode 100644 salt/salt/module_packages/pyinotify/pyinotify-0.9.6.tar.gz diff --git a/salt/salt/master.sls b/salt/salt/master.sls index bb39f1395..2c47a95c6 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -15,7 +15,6 @@ include: - salt.minion - - salt.master.pyinotify - salt.master.boot_mine_update {% if 'vrt' in salt['pillar.get']('features', []) %} - salt.cloud diff --git a/salt/salt/master/pyinotify.sls b/salt/salt/master/pyinotify.sls deleted file mode 100644 index 8aa2f1d53..000000000 --- a/salt/salt/master/pyinotify.sls +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -pyinotify_module_package: - file.recurse: - - name: /opt/so/conf/salt/module_packages/pyinotify - - source: salt://salt/module_packages/pyinotify - - clean: True - - makedirs: True - -pyinotify_python_module_install: - cmd.run: - - name: /opt/saltstack/salt/bin/python3.10 -m pip install pyinotify --no-index --find-links=/opt/so/conf/salt/module_packages/pyinotify/ --upgrade - - onchanges: - - file: pyinotify_module_package - - failhard: True - - watch_in: - - service: salt_minion_service diff --git a/salt/salt/module_packages/pyinotify/pyinotify-0.9.6.tar.gz b/salt/salt/module_packages/pyinotify/pyinotify-0.9.6.tar.gz deleted file mode 100644 index 3150ad360fcd785b11c1d67e734fe906e973a782..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 60998 zcmV(vKMg;E_NYboKMvJYT;NOD>+h6?#e>?E^@Zo*@+%VvZ!To#p)cV8k-LK%k2RjeG5<6ek|2O~J z{PXs#$hQYcx*ex)1uuWAwJGckAhzTSk)&mukHTRr%1p=tQ1~dDF7o8+x)hyZSM2TV z+~3**EG{nML0FVYoQkJmUM6WGzANIv_tD@VWgJd|EWdix+JySg<9w17MS?99;yTXb z!9rZ+A+#3t#3+wrk&VRgI?S))9yT4O3o(uJ0xD#KG6cx!l?cTU`T^jeY0x*22MwMe9;DDxO5{|_*%c)U+v;lYG#}jdJ{^au2 z;foV-dLf>_IRESE@yW5+IJ|)08$I#r^zz5^mzM%cyf}Pz`I9(*A`YMZB>r;x?6@aR z{`>ihlZy*+{sMrVK7D?6dIC>RpBP!^fv*rw5fBw^p(;t4k6hEGy z9iPC%$0yLc!^dYQ+!ze)=1ja%p`p56QlOB@$=+qxC&( z*D@TI`@+LL1i3XF17;AjBcf)v!o2|Rw67UBlPhrmw0euuARxb;Yhe-h& z7qC@mKnMaEAatxdg&!@|&daozO>rZ~(Q`tne}$t7ptxDL7p86ce8FwO0h$>gf}x1F z45kR2Dp5QV{XR@h+3$A%V~l#LpoDD~#=q~UVIEFYp?z^Q%X2`KUMr?}xtPYLG$O3@ zsupmZ=i)%LC7e42;(2QrJ;0@ZpCy1$MF@y9hJ{sa?gjVS8jL$Zfj2g6v(wMcR!h6tD;yGtz?8BfqVntMw$3dboP4Uo}40D;uY&=f#7COxrGQu1lPmcxKM4D z*LfV`EukJ{`W1`;3hQ5GJ~O{Y!?YZ$A27@`(*O~#8X#ewhYR&%oLya6O~K}Umz1qh z4mZFMka;}hO)OtWumUl^ji6|mOk({?{)S5t{*B^s8S3`of%=~7Z(*@Whne{eXndMy z!x&FOnJ;)|sj1N{9hO-(*4@bCXf}*ne8}WP(k$=m(Ec$L6CF$$A09tg8l;C@Sn z*Wx-XB%n@L3}(p~@BjQEgV5E6k&eF^6*jptiMxfsWcL}5&TTLF!Q zvvJ7>I-5*^rWz#W0!gPBu{x0`m-&pT(h2M|B1L4uF&0Id3<*t*qj!Rb!EPXqGg>=> zfn>_tFi*k(G{4AZIb#)^2&6ZNN=oU>xWN0^57=i(TEK)vLF?%J>GQ+OQ@&sOPtT7} zV3be7aS`i#;FP=ZI7ty1Jj==_*({CLzXj&kF%nX0!Gy$4(6QM#O4}vURU-ts;zF?Q z0Y=dqM+(9yL8=)DZPmtrJG-$yHtw47KBI!K@^A{2pgW{PAn|gzpn1u~9Lo?rKSYFK zaT@JHuRsw1)tS&W5~gzN0xC>_Oyt|wY2o_<>F^PS>q5Qk4RnH@hF|FON&0%Qd5Ax_J>J z4Gttaa7F;<2xyWLh#kp5;qC;cCUQFMAMM8O%T8a&a-XX6dfKnzvQJl-zX5G};RRPl zSKkY2;ee~!HSP76oxt9?9#zc?N)8U1E7aGcZzNssFzMw1kksbFab8y?JEf%A0V=|Ao0mGYl)~I@2vuHYM8<`Zn^;=yUT}XhEX$s>ZtQcWzh3z)fDPV zvbe@{P3OnO1rq!+d<>_&%#b2RUU!TXG4G|f;do|kd+rv#=dG%*Y%|R2u2A#q(*@p; z@lG{>`i&>onIY2^tLm~Xs;L$AH;P~oe%}Ib^Xh9$k}~IX3DIG3q18+fdyK*`)r{m;XQeX4`O+aS-@u=7NvV@^hxR?L35iUEBOO^vZs+}GfNUq(S)oR+XCc*Qw@fvinZee87Ba4VbQOp1rGCnze8 z^S;f?lg=YS&0F%1SQZe9AJf_mc#g=NLE&9?Mg=;0>R53@IWZm{%LG?{vaqh*IfpA?|c&!SKsCejAVZ*Bkgb ziHjn zs~=)}5VT&zv>VMCZyVH^0goT~O#SycBBFg^Uj^T@CQ8!5`z(mcVKJu}ku7A%fVK*H zU}6}ih$IHF$Vo6=*Hx_5&jx>kVd7VmKOw#f{+12|HAQ{0#y^RH?Q$5|0qFCl^I#Awrg9W`0JRK<4`DDw5=N(AlPr73BWhaSzR|Z9F?&@&o?d1UolHSzI#2N||v?F|jDK`_2-i~D37*tB@QY@>&B?dQspGE!JUu@?ee#n9fQ-G83IJ8E0^soS^2O=n zeepCdL)aoA+Q6k()egvFmHJ0#=NBjaS1(R4Pxi$tL@K1*9I5`$dG(<=l^Q@kJD2ZY zrgInU5|Fx51?c(nlV{d60lr8pFt|+39RKy?xQ}K?&OE<`okObk|E|onDc=C>^1K2H z_R&wZfUsJkK`&v0?q@X%T;&! zv$LO$PhaedECrIBW{eOQ-r%HFElIs8O^334Av!>|GzdlD4!DW)G#-n+;6bo^zv~|v z)mU|JoIktlKRG|6VMZD0`^ml&nl2U-pww=t;ie|)|4fr&9^*X;H`)zg;#tn@2tMV; zXy8jA;gpnU?o0QH>`~!wPkRuRA)fsA(HZQFv(smPk!#QQ0^Z$MfM0U51;#sdc%6(R zpz%aF0v;0EUM%Sv+4&6O813+|R|?T!m(>Z-tG8^nYUcOu1AhsfQB;rM;l*G2hsUtd zfOl1fx+rA8S%c{h$4G@NMEnkTJ+e)v4Yy=V`3ru}?~rM-dO9v>v++60hD10=BD@Nb z5WzX|Psv67<~)CPa`EFi9OrX9BXEyLd_lvR8OD_@F3Y-54$sc|WIYSl^5HmvgGvhx zTzwM7q>@If9%)EylOhb*uC4%+wJ3{~(cl^2gk-9Y=d}%xN6FPJCsFoRSZpB))CSUy ztcvC#k}9#F?L5mxiuTV$N=bm&F*I{7dIES8ED9u9M&+sO^=uNRTcbRIxqvAt<98)2 zGnvU0(qem@WjC{_0;nb(20o9+Kyu^oQRl<-+LN=xANogda@b1u0w}&;+UL*v`23>1 zFMiRxqpj(iwtBQLcHT)hhut0<&3N>YQ=5sKy4rhgZABcpZ4+rzSNk5-_9-K$vPH?X z-OArmWu%De2%;Jwl6GlZKz4U}Yy~lQz|??1(xq*Vm;tf3<4yq+KHl^xdZq2wy|?2} zpP_kbAY7ie{no#&g@B}vI{`$aRF`0PhwIl+IZl11BC7TGYU)cmq^3U63eGCtQ}r8( zk6Y*xHXEDQgi71({iRuagmW1PT}rJc?nxbT6T!REu{u-H6t$N;BJKr zYgM=t_v}u@jdXywymW)x%lqw)zq}q1u+Faf)kv3c4p4&8@$^Ggi^IYY1cJmYmfILi zDAo@&=|cN!3MiY6-C8#Bn#aX#T(X@ml*zMcj!*(Ag`z>BT>>|i6jIWKt`LKG7|sea zMz1WA)hN+rLtr4PLvfYB8#2YON;9#```mF)X31|{nWdCUn+d4Oq>D1Ioxn*!Vy?n$ zd}Tv&r*LdiXh;5=Apo5f)KGwSQYBp7xCZ)O??GQ&NM-KWVyXZsy1yQty*#;iJt^Ms zEU9gQ1`(Ela7R67apDEgR&m)zecN0Rc$OAg1RqW!+orbsHVLI6dG%|nzB~Mm$g^>H zHSD3}afnp7UEHVF0=#AlrKtld2-z&|miFsti%C2gpfrrJ8f+q_&k`h6l-dXl+a(Q3 zqP>Xy8AFKz3m~u13c>13&+cefk;F&bu#m>`QiwN$qmh$i=4666Z8FG#8fBbZJ5vFR zLOTv@Jvy|{c$-ACa7=4g*aHfg3THC9jpss3HbpT6!b(xbX`~n40Wesw&{r}7xFW2y zMQuhxiie>)@2>b%iB3^Lq24y^Kq!JGu?B51oV0d|O3ZOvUI8t`f%fdD@;$2TH z_QgB#UI1HRH=!ta78@5R5U-u(*KGs6jXg0x8+PeyQv%D|^^N@;RdofzOj58j(+nL` zQOqBcXjWcz#)S@7xL6c0!))BoX8nwjNomK;-~txc=%s*Yu$T?lQzVk8Z;3h(_jbPl z{U4SkN~g7Ys1P0U8%$C%5R!Gtq#0EX6*Nol_mu$Lfbo&8Eax4wii!uVrKe-vaCP6= zhm%M5HB?wbqvE*xDSZC}%0rT&#QV}{TQ3!xf31BHAO>%@ow2a8M(b1xa-@@roIZ_E z53wh1;>Cg5>9EaT@4WHGo$v%Z_Kf{@jSB*r8-sa(T?Sb?9!G+u@k*&Zf>L9-f$OKd;^^ncMAL;dk4(Q>8$LJ z$lxIWZh0J$`Co^%d@D#hxAqi$X=ZGE*FI?9vNpF(UFRWpyDfJMaJNNm&z-ivThX># zl^w@u&~gJwS2Fm9Py7Al(285M<#h8=yIZyWu@4;jlr(8D@i6f#gD+o%bHa}u#+(&T zuQ{K`9Yg5r8e z^7eyXB19uqmX1-D_|j^w9+ZqiQ6s0Q*fP*;$V&jm@7S2xE6y&PuHPj2_zbUOn+eLu z-HLaAkIOStYe7O^zR6@(x;|ZezzZ_Mak{Z7dzr^5UlApYocxN~Kkt~ECnB-tJZkUb zPIjk7qJG}q-YecI@KWx2?=`jzFNr(Kahj$bY_|KWF30EDcZjq+8_<$0iU+f+j-HtA zN=%}SJ8?d0-Oudl(tWUC)msYUJ??1s>+uo!TXHFlj1;vG?CNMXc2QC|z#u53lphpv zuB2eJTLbZ9HivI{kM}px8ACdGWF$bBi?VUNnuQb!2D#l-xo;xZU8qe>+g9NV;LlSL zk4CV+daT_iXsrB6a$rE>x1q!VNSja*$0|W|@OjMU9C|xocP4R&p=QQtnm3Y0+gd`Z zj1i>dX#$-M32h)WB}lwH?@Hnkl?c;JG9RXk#?EdYi4I!SH{;VS=9FcrrxuUDWtdys zV(=EN_UMFGdHXGvnaP6my%7ZtVl50Z4yIeeZt3Ik=qEs+p8|6(L799S^fnF0^KfBq z;hN#7CM8-$ol)DJe|rj#vJ{P7QkE{jJhrh!n+Qq9Kc;Sx5Bq+7Qk*G9%i4aZ22hL_ z!XjcK@qIo>kYJ|UQ9|h2xtW+U%kV(s2ClAUkMsgx+PwlTNtMl?Yq_Q_OAK>^=X_Tl_pF1l)po{u_#Uai0K&tUcM50LBTgw|POa*yw+s0;ef~Ijk zV$mG1P)SsHkYJ6c37cb!aX7p&Hga_W2A8mK!Y@4M;48Fu)(A3DDK2zCq`-ow=tPF{hOVWu#uqn@a1nN@AbPuuIS{V-E7KUq8`qTsaS4iUk zxP(z0n>eu_b^|pi(keYWe0n0RKpFhEX9_(ze0qBJQ@_64-GD7H=V)Kc5oA<79mK_7 zaGGr?C@gU#VHo z9`SX@@n&QPB93Ik&BfGM9Y(b_u!*Ho5$)lv>RYl`kb=7qDC?J74|nJ~=zNymq}e=` zWP80V5!tJ09QBFNJ&;AEeyC&pGWsNgvKwwqBh^OB@s86(*H=`rt#xgzkNR2ZY`m1i zqJHhXyR-A{yA5$icqcw2m>fByU=PVqsX8&Mu>@EhybexnVwSB$&=F@ig?v@G|#i5%da`f>~9V_gWQLu00 zr9dtU4RT3y(_AbHd*gJRtEa+uOCvLk9cMUI>mY~2Az*gf$et0U+GBv?i)lQ>7@FkI zh9x^>>O%2P5igqPNmA*Nqb1r^PXs}rkrCViYG%=8ubdDzW+skR;o~H>3=jU6TT(TFJ~&fUR7D&!)+p{#dwO-1$5+Y}ASy~5aV?XgPI~NbkimZ#&>^;+ zB8r2bdNou#sPI%@QGO@Cd!;9D`;zNJy*L*9g?~#Xu%p;r;&giyny^dT^w$a`1H!2U zKw$BjfSB-LW4Kpm(`EXzBQ7Ej6VL*{1B=e1-AUR7oPdW?%gR`Y;f>Mw_o7ZyzN^0o zv5WoTEXRO(DjZ_~*No*LN5ZW5@7T_3uc4lX{>Jm%f;uzZL%oZ+PV%V_UYTVfEB%+NKy=lI1ymCwoyy}HS+L9$_nKPE$U#0z{5F!ZzuO4tK{ zXz8ur0T||=oFG_OM7P^xghv7+vO-eqo-|QFYkI7oAzeDM43Egz7~=}m4i^?u&q%P; z42O@ojuhgYdh)kpcX9*Gj@v`3S2hIgqmah%*TnKOFlPF#{#2uwKzC(XJ^f0PDxXWU zx&9~_qxZMfXw4pk`tybnX@C7z)dCLXw6Hj1j5i5ro!oWJ21!!5T@u^dw~#k#S=0cl z$Rlfe+2@E!_|+a$PYt_9Y0*IHVYC8G17do5dgC8>E3q1gyEX^4D`t_w z)`aH?Mrz5JSOG%f1~<{xRBT7P+tJ>3c~{C0qg|o)yQLfOL@0ZFhVKFj(~E}2Rta!f zB!h7rv)sxT2`VM<6q>pKkZ6`UsV<0l$yuF>*Jpx}0hwQ~fk#ID1KLj2=B$P8Q+P17 z6NpQ(w{|Up?s)Yr;+YK)`~r`}PPcZ_Nc1faY*IjJr||~lUK?@9zGaltDnK45Md^dx zU%H=?G>t2G#5oyN>`^mGjmRwOPz&8=mZLWCOr3d2b=7~6&PG7HB$L?~X$@X%5^%9F zaKy3hw_5AjCu4McTvN(j5Ud)uSxoX5=b^41-lq1uqI1Hhk;v*bnx$nY8^8mjAzNC5 z>}cG5w{A}>4T<;l#-r6Xi+G)hASoLu#-!z$bO0~I8{13&qc#fMgiMDVSe7B~nPt;U zU}ARG+3+s;9k8V`U$!09SUm(O>(;T?Dnf48Vi$v+Y*a`vWVJ}DZ6VJ)#XkJE(Fn_s zhzRwLaL+jE%28t5?XJbW8{)5)P&7KNHHpQi;Z)k#5`SVSG^61Xoo|RbXK%AHYm2+} z#MnoXSlAfWwP-2B{zzuW!k1`UTGhP}kO~RbZQ{sYn}YlREkQLu*7MksTdq-0{9^1b z{DyvAwt|=Ew_HDaz4RhRF}gW0PWD&Q#0LJIR#dU8AHAZmmb!a-U5UQ%r`(Da0`wlC z4zi|cUL!|*2kkUorK`lv|m{K1uY+SjSat<5G^+I$F2l9zO*i z>2j1(7@`+zGfyma&yZ@yv3_jAoF=Hl}3^5w<8moE$G845Q_K?6p~JC+5) zl;E1Ovwa~tPfq{)sb!$SVs#mwR{UodEtlpL!wvOLg%_T~B%ja#&Vfu-z~{&=LpQF_ zzl@|rjInYnn}l!bWO{;?=`f)^9Xh99hmjR54!1CGF!Wt#mme~87$*5}Hjy!i%=N&w zIhdM+G2%XZYZjGVPFMq9SeLW86Ll$orb9~LdBsyFLl}^9RpuAl23BVFZhKV7yakqR zhcb^IkZsls&ptUvIrfaCov$SoED4;%Iq1sp8_)Z_WJJnEM^(P|BuoRHe7rc&sk~?v82|Pd8f3QM(?j zQw-b*8^WPi_|)p;LF&pkdNH&0s+W3B*aa^62aKRPk$N`<9w|(euZGl<1zy{$ zy>ICfsysh(ot$uEV#-wX*Ups*-3dAXEG*D zN-LIJna_%HIi$q|Ry{z2A-F^&8YlZGvVpf`L))CwlB_!$;SJ()ju(=``HX!F8%~dj zRZ@$><_1n7XofePAr;2c1eg&gFW6|h;2j+R|7a?@oa3bc~SV4YGR1ykN2>-bFd=uT-8>j+C;-4MHImjLE30?QWfxuo^&nX)5WP?umriQKYoKt9ioK-#eaM2DXt-Ug04 zoahi7Q{N`(Y}3B;d>e^X#fivn*mj@V>YcK3wnyP4886!0(Mp&43S&BREMLN*hsknr ztSgT5@vUmOlZ+}Y)YY$)QE-sTZI^O$AjgWFe%CThl=GlBns-aFY6p=8O?nzRYcRIZ zA$!11WOjPa7~KxHZ^XH}5a19B$6y1eA@%7g?VSZIKT#6?oykxs_a^oGjX74&Q zbTyL2R9e}e%WbTk4&+hq`KLu`v0F>zhjM;8EQqITuIn2ls)kj2e#dCM`_d29I)>yr zJRQeqj>`J!P+%gcG36HUT@*Z1GBoHU+YRU7-o%SJI!gew{AxC#oyXFQ21s&`Yb@iB zP4*peZnJTv-UU;4KG`cvf>!HpMYpn2u&mk_p1X)#LBeqM{xE(r-N373F7(N?TyO+T zim4-ARDg(9M~iG-*frR3%7r5g7~8UZler=hAC4r_O-$+$2fMfR)@8Md4d`#!1@fc; zX!`uxc;3G&NLZLEH)ai}ZuzGaWCkHOOq4nnmfKvI7L%luo6=kcnlx7~I*;Qya#?uQ zJ$-p`$!V;-&e%JVa8~u?%1x%WvD)6wFlec*)xG6DH3_!X@Nz23Y}(f-NTopb?Qy0S zA67@yY@$bxLcZ!&vN94$nKW8lVpi=Ltob-eYn6SfHc*0!QNTVLG)bE{em5N_!vslT z4~PmcmdTdUtSOV(wxnfd&vJlbVX^COz<@OAqfjZzMTcZ`L4A=)r?YBe0uK;oPa9X# zr`AorHP^n+gcNc(HEJC$`rg*BS$(rP(ir7gec9xajG+?SttH-A#`9a-#6I&R1 zT|jFhyiHLv1k|9PCRd*mB_^=b^_U18)oeOwbw{3mR*XVrTS>Punxz3wjkB2{>j^%E z}kK_bO5evp^P9RAS7 z+l3<7dTxw%K0xe20(^24?saZ)pP=Jxq)m>AZgZZin&Xz(h%`~YEH_@KZ3S7igV49T ztK=ffw*IjkO2s`@-!SHED4hdE0K(wACJ?pn`%7C~UFiNrs`i1}BuGIiycXQf%~Kpn z4sb*X=N_(m%5cM^?~8*Pjk#kS{{za>|5Ab!ZIJvCX|@oZCtbM_f!W|O&BwiV=HA^et{(njGhZuDlr z2s5?LnfT&>61?(QJ@F;K#^)b#=rDSCTr7*3hjVQYNDI&`>}jzK|1-~wo^dO;6i;in zv5cE@%Df$oXT^2L2(6BgS~(q|Dao?)hXF*1XVk`2nlR!>Lc}7cUd$?1uNEqbwuM`D z7Q_`N2tBNuj1hUVT}hM8j#&{eEoL`ZGQ~2nE`Hp0TWi9$v|wrXS2@`iu&I}#xg#J0 z&_@aJ5c-0TL)e!&P%<##6kxZ2fvUhz#!s_OAmY#E=bJ@RWcSem5$2K%5%Hf+w_2<$ zICgK%3(TgB?&|2W8{+N?&fr1kqwaV)@vxEKQ?d6eMj?9P<qA=PlC1iJePdfMQ2uJ)h}L$+zDUBW3IGET>6?Pv1ibT@q0 zuO&mtwVm+Y3gUO-!H&0$oLYqE`3a>FEMhA)J=u#s*b(0ySn%#X*jWm(eJ7fvcOnFY zz5%M@HUR2rFn8cr*KG&kx-EVC;NHrnuCX<*A%xU35AM)|F7?1`Qx$o1uvTLcwCA-& z)ttuYNmq8G>u!VB?Gr9{nMey=s|~Xq#%`)_KFKwad1?QyACY$~y-PY6toC44tlIc( zcMV&OmEmT)Zn)X+DBA3!nx;Q4Vr;w3mL}t+(O;w4WU6j`p!k-=Nd;zLT3GjCmF`mj z_VqXE@W=;V(bw1{GQ_%2QZ$c3I?@J#Q$6Ckm;$Gx8Z-`1&W?v-OebPGYb%1L^`0}4j-rOJ znotb}8Xw%dzaBoVh{{6$x>GSg+GaT?ocB|nCKHq0#J|Dt7(lU&vk)i%>=2%AV|hE8 zl!PAcZ0$WL;9tOfcjVzh#y+hsh!W@LjidjM%MZ=bI43 zvJ<|l<9JoQd%gPxzS10ZZvm|!kz5CGD$dh83NO=M#ApiA_E83K8GN=U;k1Jz?yZji zYq5c(lvO~DHA1cF*c-JG*7RE$UF}ncSL5T{ECmV!S`fW@e-26-3gUl{d&r)P&?#8=RMI9-Nf91w6xK0+XL zT{4F!ig6rImt4Pe5A0Yw5W|;iJn@~iE=?Qt^5!+fy~cTPMkgX?lK-o!Hm)4%1XX_Et0L$QbH#?W_=c@;x9gx#`# zTfn@d=1T~pB_^h$sAGrGdycq4%B@)qbd_KbRL3KQVc`@&NrsIRgAXbaGg&HNKyltt zed?)d&w)2MeP(xp<1Qpq4$X#o@F7{K8tV|gVQ12=`;xG696hTKE=G_JL}zQaC$@I& zhAgd|SBK8W-^uR@UOQ8C!FJewNKK5GAT8)otq;>EQ#yGvpGBeXHH`V-#{{fdCl2Ff zccJl`_8RjqQW}r@$mVe>#h0WTr~&P1#_SHCp{tT1guHHA_p~|qUKvAMS$N7XBd^B; zbpWj%9A&^26+@G2aayybhpLcowX#WgMrO1y=J7qg$7b4_QUhS{IDx38sM}mc%p%g# zmial+I#vsZ57{y}w1SXJ>k-S5pi#qgm@ZXp>u@BV;-g;l%m+pZgz9`(aF;AI2&rr- zD=43wRylKWX4F&iDQI%2O$24Xe*lIEj&>9$C>QfgnNOy-_BYk;B#zMpR=m zq7S{@r2o22>#|h_4lXQI2s#`ZFxf+3vw0KS;t6Qat{c!j2D` zE?S-jBq^jBmdY9sRIG~31jDdthMCGlyF)8xhO{Cpr9O)5s));UHgUTrn9j6%EY*It zjBqhLpjmDvGXstTJN)zf~dWYa^HPV=HmZae+Ic{sa(V}X5 zS0&CW!TWTaMs)=RnpHD&>QuG1CbhbZrmV(4e`cnv%DC0V_jVkSu91MmZRM_WZFsj$ zcLCb6V{NDRt<~IXLqs`!hdXzAs~TFulu6N$%AgcZ`hU*2rKnfQ7wj`G*we0!~fG zkh`+~@B+S76*uzQ^=TZPa(F}>Ax@`tSLu{?1atR|_&1k=H^-l7IP3H`=3Wl=k;lMg|U8|&R(IW32#;dCfaqgg8PomJVpTh%t%eNdSpxMpDERuVLFP)O2g zj$`+BsghQ@94qT*aDHd$G#uW*SvfsDZGU)1p8PcqZ6E*nhDY(H+cX3x(-2X@toW<+ z=}ttNFWu|4y2m{c-^j__cYeVa4bAIdZ7{}$DeTtfM(WoSOgwt>!dGYvX`Bv|x7~_0 zwBI3PThScCksX3{al9CGgbzik)!#lIITuDz$AZ*dRY!Au^?h5Ci)pR;ym$vSt5>zF zTI15mFfB{m;NVDjhu3GoB)sI|ESF}>*&YG70vOQ=(?)HDWlnioa#&z8%hqWaOox#F z%vcg^`Tol=BEb?O&Yo8v;i;%F2kz`ov*|b8_W~`J*G1W8gXoMAD-sVRNltx@8qxOEorDlp!pZ#l=z5 zeF_S3oz2m+2@VA8Frcx~1_V<{(mBt+;HA4WrvK4b6%;Z)|2667}U~3g{BabD+{(blJg; zLJ3hSM~*D{wI#8_d={-W@3~cifqp*0m}LxiUXK;!{0kj}>A zp7dIGpT%v=HW3Z-Fs*KUKCBj{<;2p0m1nmuIW|rI{{j^&#+wPt&|#^AsfvHIK;}gu_d@ zM^tJ&3d{`D6#UNWem(CWCAm!4)z%Z@@+kwAT-L<@Qja>;%yp3sZtLlDVdy^iF~qp_ zo@wM8(Z-2ycU!e%__Ndwa4uB|YD`vjMMstL&^=5b4)2Hqf9XT8Pn(6H-oZd~$l5H} zK(7dp?p5GcvMy|K$&=lBiR~{L5Ngqg8nW0$J6VEn#?#0qfh_?QTMg1h5TBezVFnVm|`QjL@Jl&=d@6M)s9j;h0?}x|I`O3h9 z2fYTfQQz{@Yk_L+R$AaQX0-qhKClI&6vhYEL%b6yYciDHSzxNz zhlU`h(kZAX8cmRtsukE1%*iLIG<|=md_iA-y(}t?XLhH>%rp;_Nk>c)BwLLor7y&n zqk=sHgjrEqY0PDUb=x|u(VEGM$0yl!7EBh}qDY42R1u+28K@)66}HUE=)@~(JULhU z;)zV@tDsTqCR3y1TSzfn9Xs60-^sfvWh$d#k|LR%6_vHHS*^=TCSOTpB~L>G5zboAs^Dj|>N-{wys<{G?VjuUOoMaG z7rM)2=6rwFU*t|nTP0enn*K|?7-V4{ohmhQ9U*^#VT)u^2}#iunCGM9(=tQL+s;vz zmU%YbI;s)KS#*PT;GhUVg#|XK;bz<=*c3-d{At0pXV8ZLZK)6aUYbyqficrw82Nc; z$ZyI*q7rT5>Wb1a40xU`I8-~K88&ODK;{@hj}19wGT^KXH2FD-R%Osp#tyTzVp7{cqSwE+Y=a$s*(GN!;b?uK$}+&cEoEbrvl*2sEo7a&K8Nj&xT76MB@*^qDACHwgWb*Rwkdm_~}N+qE^xn$hAr+{_Q z)X2P$K~9n8lne`{=gAA#lOxfg(sbu_jCrgahed)hs`VWe{PBIKdUm)!6S5&Lnw_dz zzL`*sbJahSQ`JRmkNk@tRGs157l)%pha;rW|K~UziNj?8Yn_h%bsUZ?9~=7T_8v!w ze9bLup`DhE&h0DM_3CweU?_g3imX(Lpps3srGbIT)-+weNBSnL>5zIC-0tqO5X~~Q zrguk;ya^~=3Y{RcxRVPJhg~CK5ba+j<*0FuVvi%#4Z*i{eWIk{X2mgTZ91A(MbR`s zN8@o0?59KNv;1FK!bbT#lI02)?EY^_Q4 z%1Nt<0&$D?yKm~mo{hnjaTqYYD7TSwe4Jr)@kt!&kZqE87%W(?uf#QGs*jLYkrXQw zqmH|WK~bexQOjBNCKzZn#+y?`Pkn_#k9Hx$0awu3rf~LuQe5B(fIA3)QQo7J zm_q4tRjPdvzl(=6SC-4t6%%N!3blsSTPkB&L;A6z!lYj)^yp(uJ(6VPAH-KT0ds=6VlW0a z%p!P5L25}Vu(z2?#7Fd;1PI`P^_pk-&F>tjqeuDJ(s&FFOxI#RrJE0p#0UW>Ix}HI zEB~%$Fm2^W+iI{aHw9<>HDTEGGyyIDmU0q=Q|4`kVG%Z6>}&J>rU0xIyI;+28IQ*p5m%+wV$qRG(S?o8r9WnD=j`xY z0Nr#4Zf2S&W)+~$G!;73l$EJShTyeWzLvOUc^XEUizS+H0hbj?Mz?TC zhCcbNk0@97HNVvzvc^$fa&oKi);!8aR0p>la`HLzf=F`^Qs(|kQm<<=u4TYTmQLCS z<>zTmI{WX#PAynBWGyEOY~ptz8pT2}n(4kqAkzhH6w*?0ky)YQx%;w4ix1_l?yHr9 z8}js==}M2uQw3$ifgI!MIl0$YCCn=o(vD!86w@jLfpr#}u|kJiCTlS`bzCR!1`V3( zWmbz#Dr}s;acrroKS~{3IgClxr&Dqu=hl5|O+sJmvJP~ce*n}gcL1jSmb351*Ko$5 z4t0R;Yb)4Fdv*f053aPZoPTRX>pekQO3GO?Ut%_bM-9zOIRb%8B7G_Z>yyg0E52MI zQ&&Yzflb{d*)q)C(idrxZKL{boT#h};-m2RXTaj#`qkmeVq1=FIkJQf5?$)+)Mozc`s9&+pVi*Z&K zeRT8cVEWIVOxxL$c|P&C+#L8FmG^yE-8vW6cKmKQo<(t=gP~w?%(&TY%({nOpPby7 zeLu}ERL%5+3v(@Jvi>L6fUxnY)bw+C`pBqJ|hhi%2R~)63F`b;`eR1fRWWXb3*MWg-P&RL6F0U9@ zX(&$U3l#>*Sw4=I0=&1zfjynIMV4GA@TJuonTvqt-!&|$tVY8h-k`S9Sz}B;Vv{!# zZ`PY}`5etZ=3X2vN?d+Gd&5gch(3)6v;(TRM_P(h;)+oLwGbGcL4I`$xFf(g=FS7R zFwC*##FG^0gQ0uAMg)DwA4~;Mxu{F!QjMWvpA~01(Y;5=z{0QF7?pfgKy(Znvrb5K zFVyiAx95zy*J&TLKLCOqdig3Yu^)`}KG^l&kA~CjcE!&P(1_}TGR8Yy<9X>ui*ExG z(H1Cm?6mvs?i;A>zSI9}X4#pe?^eyxr;Hnq-MdX|PEQ}Tm&KPp;Un>c54m$y zI8^Ezp9bYDQ{q-h`4~!^~=YPY9jO0gFNgjHO(B5uCzg z>=X6MLf{Gx!E8Vdc11kcnrv&?{5HWUw(st?YRu7bG54+1fuR^<*XQIFGO5ch$Tb{1_F-MOkOw z$@O$Z405ZS;-Kg`@<74MXQxN!$0v?YRMS*6=BD6b+MPq;J;!sUnkE@TJ8v(^fP;gg z^m9q7VGy5!4{P;}o8mH0CTP$jt%@*jV?v^J*n4GMOhX_aTMau8Sx zVp$r)eDGK2M%iq4mq+xfnobv`S``U!MUEdm8BzxABX-+KE!JQo6j}9xdrf{s2_c?qV0$rV zWSnv<(v-C+s>ML#df>QNov$52ZV}?nIeYd|+pmvt!V_G-65fh95ni3pY7UbuX?S+m z){&_^?Tuq%TALh#g4f{<#KEQbp=5~&L^hSKWNRc)m=ssQ>Q18NTLWWf-7&0|tW4r4`r2uf@cumY7zC1i1K8PmrRCH(D?dnPH>4>u9R5$GVPQl=hiffy%RZTyE7uY>IOpjQDyq7fJhLb0w2AWz z_8mjo!GS79NUZfF32@55A1CBw0%LswCo^KAz`2xkpx*%(g^u8Y&(I%AISdSC$evtA zM5J}sjwB|MNUd$keQLy1tI(3Uz{2<^=U^QalO4ZsBEi1RW z9si(`cED>QP`^0^E)0Iukifw9i~7!Ygm+`ASiU4E6|_Ai^zpw_v};Bs&pC!r5qBDT zj5)+88f?cVyNw~~nx1XmIHeZ_XM|^tihPM2X`w}0P?)9;3De1BQQZ_>a(qM1)QZmI zZ=k~WgAes#v90FFF498qwm%+wLEzm17n7mT$rFiJf!rL5L={9JL)ybDfJzVKTcfg3&#; zYLisdGg+KhA(Fx3m=@L4MjG?oi}rLbnPKmW8qYkvrDRTyrD1c4!qgN}+NFH7b9>^Wc)Z>? zUhpyGZo^domyR0~>$41~!iPiu&G25&qc^Rphw({6;A?aXCcdJhBUHf&2a0c&-lW+) ztqzH$u9WOG&lO7i9zB<0*hqR(<+)h4opD=)7O^T5a_g9w&D2_Tcb~4Vq4uja3xP+_K?V0R=I^)5Pq#&gc~;B>j$sE2SZ?q8+}((97bV()+_Zfnv`!HN7y0;UciFHwsi8=)5H@WRY{cUKn`g03yc*a$?MoQmrqJ%yg(9Yt|v*4 z{*3$DAI#8B(%|f{(4j2?CkYU22#N}U1Q+pyc}&mAH<$fbVz!R0WPcJd!?lU(s4rjrJ|$KlWlFKkJdUL zMv>CK`-FS}FEhO1>PE1kT6FGlUm13V9o(XG3w3bk< zr3s=+F|bbDm7%xq56o-R?bW2GJL_x&VrJgy*1U%Hl&fp0PI;;hcyi%tIZze^2L|kc zyQ&9zc^lXqH6aI0tAF5ahAMx7H-X9=Ru|eF^G4jkXi-$6vVG?c-n$xS180{;89lQ$ zAx10VhqV(%5Zr~Y5v_CC*I4e2l^kh}gQ5-BhB7o&Yssw2;8n7}>gV5QNnBD+p}NZ6 z{4X(4K2)X(ouaF5HBI!q$IidUH|+Oo2#EJMp!XE7t>t?AlCzd{FF+##s%U$azgM&H z-UM>8o#tUyW_f|8We8oohu`n(V;-GG?D@;xq$t2JP-J^$yaGwq%+lF zl5HUcX<}U@ti+aV&CJzopw*C7($kgnOA9U9o%jfyAEdr;5L-4|4Pl^6JK~U7S6b78 zVs-p|MigC#v}{uCc-^c#Ag7ny))+crkR8jU$4N=(U;8S4oSD)J>m+lWpnrDU1xw73 z+aiZXv6Z!Out+lAg^+fn^@dU^qQelCDe?e&mQc)4kr`O3L;>}ZaKdV(t zMk&!jS}$$aiv&mLi#vw_EG8JFv5+yZUqx*9} zJ2o{D*g6=%0QX*mVTe?b<`4y0G#gvS2T96Q8H!UX>!%y9>ls~@6k>#T5XsXDMPb?s zFB7*-GzxV&+qvUn1IVpUjRez$-!s|xI+lIFN=4|d1ykiq!@J6b z+`L{ezEW~sm7B9xOff^KCS56#M|LY>z8QUBJ@K=S4Y&#pjRuiXo5B!L3YMxoVqMX> z=Eq5zt?>Q|DG3k{hnkhB_LuTe6$yr$R@2WAP>Di^3blMn$yYRA*mbt7>7NZ~5aNZqbjC7i(d`vq2nF0)^oL=}D3BPhpr z@eALo$b$qy@T(KZ+}k9z^8Sh*qfWhH#~DQamZk4#w~kCb-Q)2%o;X=>DZL-fu`&PC zOd*`zPK~aexcGMl-znRU_}w&2D_SD5FZ5a{KMG0{?^`pibu4_%^+S#Zde-)pQiBoAokF}iKDZP0r110Sbvv(Se}0Sc zuGT_zm%^f)a!FIdr+V2hr4U#;LyZCJt4g)1$qU6)$9-Dg$a3JD;w&yPQb)KDTYKaq zU!YS5W?M}t=J3jUcT<`-N}XA|nFw?b)7D;13dFu67J=V@DCm&0XkxrX9{fYB;tS5D z_S`6grF|)55R8RE@fp;uWbqZqM|6PSrKnh4HjZIcBe#tqX^4WC)Ve0tYy9i@s!(i~ ze=qwEb#7^g!BW@$YEbfitApYjZAE%ao{%WZv?VAm0ErZk_rq@tZi=1xQf92q0knTG;+^v9k@uXo=BjuVPS^^UCW zvdsuSOx|@~WAi-(@}@pT9u4(?^M;TQXa_ruwJDJ&4Fn}^>!Nq84Xt&>qubYodatq4 z8`GnPi=KC&oD^Fq%PHYJ&83t!vn=#BqR|@Zs1b)Y;Io(11262ZP1uX~^XNSb<)mVvPUr!mx8A3Wwz`w9v?= z;aMEEhA2!Kpem*fw``rf=);9t#m7svG}GIbB!Zi#Tz30+6mC`WQ?4W4l5(8W&mu%x zM~37}FRBR5m1IbfQo(4gCX1^pG;-E#9vN$R+e*0#L(c6QHtZ7#HR$ePv*Yf^fW_aM{C z46c^zlZJ!#)w!?ZwiAzs{9QPYTDN1X)?lR?-AmTsn*6~MlCNf%jQSs>{ATIIM|yy| zx^ssC?>BSc#BYVHsBTWiq4;xm*(8h34{y(-E1?YZulK=B_iAQ)O(wf>cYMeWp_5af zVFw-+q%dKS)m@~0xADX%MBR|L)a2WpM6_Uy&{k$XwOz%1Agj7j>rUQk+EI>sj%WSA zw5zf~++s@zcZl6p+`of4m1P2aU*&EE|DqPQAxevUZ`7i-oHp)JE*dm4*Z8Z$JzB%{ zk_y&3Mv|$19}zV7t$|CP_Lo?8$RZ>=buCDv)rNF0q}zfaIV^)j_exe~0NQZa;xovq zu3*cF>tw(xg<*#7pi&SpC2K?{0r#t8ZyRuIUxP@yt^$FCH*7meMseJ8VV8B;DXy)`?48gf_t)iLGD zg{isC^nXzN6}1e{C9P4v5a#qYu0<8GYSU!<`qY`q>*ZL=T1(Rk$nA9Jmn)?nGx8cNwJv7@f#O#ON`VbPQlxesxq5~9_@A2xnyDI)V=G-6v(Z4HlWp6TZHh19bStamToxzitO;F z@&t2@de>$CoD(m3zZzD$kz-rUttn1TwWPT;+3)pPG~`@9j4%7+aO#_`#}H}!vYBm7 zU*Uf|nyWen*V!kuy?t)O{xDvq1(dpzGE3Vx z7TT4h>^FYIwq0-|7$vOZE~%y9+V02D*H!=_!f%wQm6xBQgv#t1d2)3X=f+OROEF9d zajdKHI%ILHc{C@x=>$_M3>p1m&-@$8oxcDbk}QwOl2b$BRL=8^#mJN?1DdkVb~(=! zMkam)oCLKi-rjy2=G*x!-5zCG0CU|Y{ML0`MxnCdpG*x^)V~UyWBW04&kIj z^`Q|MoBe6-yK?O>Eo7q^I8J0$W)#Zui4K9}-2S@wfoQjDxiG%qpTbV!kS4?IS_R@D zmkrY)=cs#~P>*VQu_+FVVm2XyiQOi2$Pi@`LJQWgk6VonRN}o=$_|h(hLb6W#G@lq z<^EL|L3-U2Z7kdF*4q|ZQ&puO7?l4;ISO|wo$OVcTrnvdPw}>LJr$`wo(EKJOBMMBhUAekd0vF`a|*-A&be+_ z9_g@K)CB}BR_0Y@&cSyKqY2_y7+{URz!ClBS8J{lM71heN{_-|CWBcSyY6*lVQd7p zMO~CUoMUv(E-Lg|$La}5nsh60YT?9w19iWtPyS?8YO^g-o9bA5&W>g&$Keup&rJcq zi(}wTn|!X-Y#=h>#;kVi2&T2|JRwmBTY0bD&UBW;*J?XD7HC5=gs9eTWdMC@J3&O8 zc%zjzj%dfO(T2S~IT7)4LH!P7;|M$81R{kE;~ff&=)$Mv2%C)~=cQM;TYH3tT=Bz> zk~Irx?e&Y=2K%D@^UmG7uXiSXpl_JkaCoB&+^s2r^P=DHR^MOcah#eGd({$)7-gQO z$lbbbzzXXE_o@ZtM$<*^SBnf6q1BHEHI3jZssc45nPHSfUEp3_FO$?vR!tWYa+B00 z9!}c7S`o6G<@W=iQGj^b+!wGFDL>35iUW(b4 z%z56Dj!m&00S&f%p1t)U5E1rVXE)mk9tOKinU5o7Mh^9^fRZn2TH;ouBX41+3Ihfv zL93+#rmpB?czK@DHeEn;!tqRUBF7_uQUpM4ecdMN%z1oxcGf@n>&dgri?)iY=66_m z5~HCnT?r;sj-E&oHAwR=8qS~p%+Z32X*^8GFo~sb()l)mj*heGRHivKO+Gq+R=*A2 zxhJ-eG@sRT7@pPlo-jXnUO7$|;j)CC!GUP$xc3{sccOK|G_@-k5^EagsyzevH9wI` zTz)K$&YwO%ygYq;dUksGQ~&AtF;m3V$4Yvlw=$3yQ6JBhzlk3k*Ga9NB=DC-K0bc4R^i#xbk#A|j$F5a*g{;vGolb`7undgOo zqqNTk+#yg0^l#THMx;6e?rdzm#puvmTQHbGR>bgw!(bh3P)wA52{XUZMEb#Mmj@fB z>UBJx9&G%Tf71!wXl^*?hPA%vLz=cKHVEubLk+f{xQ`jJ0!Y8AWCZ6jz`XI8C}Em+3TMnhxT!F;aQc90o?Tr_Z$+J%l5(@_jF~(cD9=bxK*bq=Nc+?8Pr?Ll7O4et`NP&% zf8am8I9ofxzXuPtr$7;A>E1R<(ox;?Dcab9zXuQQ;pgtd`#bh0{oQ@I_toy+y?b}> z-Mja27s~DJJh=PdE3xxACSZor1tGo)qe+s^07EvETlHR!MSuR2{GB}i>G=HUAf7IO zvuGV(T;ebIszPqH;QPK9q8Zl~miW{E>Ts2=*xI@-Cu8`y%ERgP0g5xW-iobE{I3qf z>6)XD24*b{Wxw^Od?L2;ktr(QI!%odAVmW4b4P5EoT&Zv|7`>OT>RYn)1TBMfIO^y znzhBxU4BRuAlzo};-P?sfB9AX{2#5azWgmY|2kXmCp!NRAKq^~|9D(>_wL`lxASoK z!Na?7{&#nF?|voje>wlZ|N56RyxqgGov!)Q;{RR1_8$Jfzq|JZ|9_UhP0ef>10R$Y zt7UI(wl>jbc%M?xEzdwVPi~U!OH)w_nstFzi}+A;DYzUdv2eu0%1159Hkb<9Bt^V; zpGu<@#RUT&qs$j%!bJS_qs&@wYT8Cfl3pg=wM?L_0jo?jp2Z8_D;q~Bi-g;9e?t}t zRD7yd9Y`L*a*$*v+5nPMG3SbxRHN#)EDqWX(cz_K)oyDZaRwvD><&c`HVMwGK+C0u zyI(HOf2sQ)c`TWD@b;L0t^5D(-p)M_|KEMM^Y9D)|19|b&)c&i-yS6CcAUNyy!?dp zUy>3Kz6cR`yMU#S@w;-j2mkEhKdBf_BC7XzoDFXreWi@KN`gf;=yXzJlRaq!C}t@0 zxL^mQBU3D$K~N}F%toW+op=we-MkOPi&-k(L!gs$TQeZ?|R}n&A8YL9`uZw-CUkVY@?qnikh31uetN|6$+i!zUU@0 z@e=HWBf=q|~y<8UbH- zr9e;WZ%}d@=2ve~hjZ88qmLdb$u7D7Y#d327-hsIsr$Z|Bpg{u8Res!$9Yn~SXfDikuhKWU|~Y;nbC52 z4gRhMUqSpWZREUk_kJNL|H$+oo{$eE{MMBJKDhV5qyP5q-n;vS{`)-qzoK&luCzn9 zNjnv(Itgky`b}|ar-daOAt$$9M|tNdQ%o{EP-aElV7W}7MGh|PQp_fe4seXw(|f~HqpWz;;SPr_m86iUb_(-m)SD;!1*FI{)BAy7vpg_i{1kUE6l z=VQ57p8sJM!pAWF2pq8D{NH_WcgN@dcJJN$a{fQ_`By8gcVYc`cYH==;6FH*N_vP9 z?q?~VOnRXe)X4MdCHH8e>yGM%@-5$aKH!(IKsXJ z1sTj>SM=I>i?Epw+DhK%DI((a8I>bVH3Zm0{42Lq(_SDLwmSYG3#l1=6 z`HsCgf%RjrEsnOj^KN%%hrS|7d0^y|^pl24>A+|z6bnJWw&^8MBK`gmcsp@Fi_H%I)sDVdomD0D3D!Ngb}B` zOUDZWrXErNA=sfD>qo+2x(~o~kWQob(>PB6a|dgvNsOf;8koWrnUeLb>>S1M8dYhv z%DK4>P565$Syt>?72_tM$Ko4?X(t-ov}QU+BNj@mHbnSo_P19wZrVN98ee zdX9XH^nqfIrB6piS;%$NCh>ft{tFXgk!KR;1jZ9|dQQhoJ3dh!dX5B^7t;6`9rP+0ToY=~{ZFNhiZnZtDz9j3DR0Dg2!oL;xR8GQy9Dktwts6A1mpplg z#UdR#XoUgw#4RH0YTLuQ>An?$;l|mQP7g{ZpSRGDoRcqtu`lvJiU0jTFrS?Nhwr;S z{=fV1i~Y~%=KrPSukfNo6|4?Lb*tp(BTm?n5~J_!kK&qhfKn7bHYpDp)s<2!d8t4= zAss=J((=~Yx=ry4nfSLD@X%z30%8~L5el}YX~P?|G*vWR;42@0F5A8pPEWMO)+2!? z{$-rDt9_HQg~pzpPE3y)g7cVgJ|xS`0`U{j+QTrlz;JXFT#X4;mcHOf{l}AOx$ygg zfkqY;C`vGjV0tR$FG+A}s>919hp}=R&By}7vJrFv@XV)rRr9B=irt3cwk#MJoGnk- zt=?|AvMBQTzR+uH^fd;tl#e5-Fwc}uJ?XV)VIDSLvdUDUtE8B``^8ty$b%n|AqelT=qX` z`!fiO>sE_GjUxWu6?@{YxF_z52jZdlR{Xoz-DyP_KCaAWaB7`>`5*9@#I{?&lW*MH zLve1U0NVSH{KpZxdVQh6{@C=Nvi~lxk@<+D+Y16#(SN%SeErXZ2YdIwod3^5|6R&; z{bC3HPi6<++N}7(*UKZk1twX9YdY=-09Zh98xwITq(ezVwSdEYlV|o9!NNZ|{*9TvV8&=OiaZ z^-ZqRP3Sgmdru$+6jdEQ5S8tc0=11Xg7aWj%G^D9NI59EJqo9vU2)Gz3);$kboW}} zJNn#;vMuPx7G*bWziE=QGUCpubDT57O5R|mr&Z>Q{i;3e@+RvA$Mx2;R@>bcH`_3i z6tljE)3R@w){+q`{1eR4O#2ptanjJ<+v_RqCw~3iI;F3tPv49G;rKrafdolE$^vw? z{nvfp{_~6f|L4d5ywb@(VsjYUh0&U^6qxPN3+1-*=s@fR_Z8-*aFba!UdvbID4V4v z;BgkBc^bnDict$#oXV^p9+rJOu$GSaLsa*8R%B_sr1UojVwV-0SxVYLt;_7GUU*$R zrdVYa{vVh(l2Wmef*JF8A zYg?IO8qU)~+7hdUCd(^x@L-bwKZjO1$XIdOLNda!Hb+}LKc%#GQu%uE+ejkO zNlFn?jx0N*BT-D48Kt0fAJ{s4@*d-$%Hu^hz}};rJ1nJX&1A}XMJUNhA^tYwv;qK) zEqZQZmB&j7572n$ZI&Q$g&2dJUJ!!*v`y=CFn0^dr7aOV&6tXbOQS@aARUu6BeSCkX-)D_N9@(Z;~XzK1xdhbDQ5%e z$qch?hlM(~crCn@-EF36P@AL38tjm_ z+(^f&SjVVxfS=Shj5V4~l+lHjg#_X(xrwnEbdXF@yOER#$i9mG{}=xMbJ2gOF7)#DsWqQ2i0C_$acifT z6brMb&I&djz5+Z<;dcSMCaE&o1gwFmAqw*d-LYqY#ak$5j*&-Gb#g=wETIjrFgFgl zI#2LMi)2_|Qb6Z%Fi|Ocq-8555Jme>*&n# zKcAcbqid+JqA%Bb05SJ=8ew|8avn5Kxxv3*{j(qx39G?2ZKOdZtkJ+{Dkd zB7!0rqo&8oY{rQa`h@BZ>gok)Tmbg#PibyCwx2r(GNroRp1_czJqe0`wO0ECP#9k9 zTBtLsAx?La(rKz0FsfPcO``X3N@tXAZV(nQZO#l9PTMl#Z|egh1?X*6_yL2fyxVsB z){z4m0(;8U{a4gZtxf-Z#0{`g|8u{h|Jm8w`GWsHGyaz=jUuK=!P@^pj0fH%BQY03 zR@_=$vsymW+T7d}Pnj#;-KQ8l+$BwUQm>F9L+)zKtiMT=__r)cJMiNQ$hUcfF~{2$ z`@$^Q*lAui@Pv}+bu>)n_$|HETPUTgDu--(l2kTCHO{7m{*e!Nf~Y}pvkG;23I6b> zJ41V-bMp6D@-`gfRoXaP=rzH!HKv>Ab-O|)d-VV+P&ywy1xlN$mR9isEj+ zS!<`9p9%jveoDWY@fYnL%o_Z~ z80M~$NuQx7QSLI#X#fJ6doRGM39R>l-HfoiG2bXjF96&gu)X!y!x#39Q08Z zL0vO{yRA=|1I9{L!{x*`!p%hE7FUUAf1y+fE^OO#>kdGe@iET3e4Yxms{%O68{^g!-ri1omPMb8cCptqRde zk&bW&(b)=95`s=u+UE#kC+4BGbH~X|EZTAz+kqG6lc%(1HSvhbN(ZB?+og#-XrIqe zSV|}#+G725`!@*tx!m@mE<)#smBsrqcdT|?|8j$W4*idI{`oEZzxVF%?Nt2#9(=j~ ze|P;4T5|BRtAv)J;{vf2P~$C-)qLrN61ok@jKB$7q84BY6urnCz(Nl|{Wn|zz< z&~jtArAQUT*DI`sj{d_}2!h5pz_oDHISTYfwuB~WDw zEuD+KPcRpItL9>F>0IntbFpX5#hx=4ivCzSA7t`wO^7ssZ_Te!j*ipATUXO7mheH& z+3rS9Y9n{MKHaq2P-joq*{jsqb06qmh+TqW9LLj+$`;DzRDX#VgDlLWQ#7{BXVbEp zL3GzQPqnAdc`<wf@K5|Fbkfv%ud_|F`qt{@sfHZ}*G+|7W`Yx4zy& z!%vheidi|@`WBy5)qM4){Aiqvg3g#@>u<`y^fME=;^!IK_qAJgqGNjT|Nnpbq1>$n-rU?jGObowo2AMxDWW8&it^fp7n2+Ptkqh=zkRg2vPOF9Cp3^j zD#?Dq@r;7Hud#5WBiZK(3Fx2yY#eQW>Nakj|8MufU3mQ^|KIO`|8f0~fLy`PYYEGe`G{*KVU*qVy%Xm>l>+kPDc!J9x} zzmBrB%oE@e*~aY>*ubz5e}S830nF|Vetr+!)NC>i;%J5@$I#d_7Z0O(A&`nl`fCF< z<9@%gx(31VoE#sRNR6Ju5c`+$!IQ^pnp;F4`x;^;quovVSZI~*=xt}9XKxE zkT20@hh)>uZ9He|zCv-3$V@Wlf_+4GY8nJ;di^r}!~h8-!;fo6Dvqb9DLD-vzsmd1R?Fq~ve3CgY$SiEVd7gTp? zs%AYDa{sVcRj-M)YHCklcsBBNVL>bxJ>d^5YDcvT3mxX0ora^kFFvhG;9LavPn4xM#FYxUl5gU^B>N&i@~K@4nu)mF0=n zzs09O%i|%LqM(Z%JIxcNEz5GEZrk!$a=G)leE5MRD4|ROEPynviZkysFL1u|e?QH< z#aXv~#Rf?^PO5uOR254kHuin5y{^B-^ASg{D-033x?-0x^!w*+gxTU@&=uyBquq+D zq=WwLJ%gt*Mc*Y77t>0sYz71TVlZG&RP0PD=6k9PE45bWd0Jk{0-?T%Gs%(t%FRcO zL&w>~Ws9cSEE8*vDMe*)Tq8u6=0k-NE$rkIik6CrVm;i2{~zqHqgc?B4 z#}DHJfkqhjqwiln`*4WR2uRw>*c99yryt}DQczfm1h`V=^Qm-Hlyp*U37a{oX!mdz zAM74JSl7j>99e!G_EhzuvQ<&O7vyW+i6;f_>dwfJRLgk^i0hN2EN`aKlk6js6rh3l zF&^W`N2?5__v>OwK;($NfO@J?F%y!zt}95Wc2kvBg41gW;rbOiqo(fQVe|dy_Ey<{ zKz2dWQ^nVPpw)AdoUZ5dgm&y~^xzxd5@KA`jtP8$o<7gs!>OM}-!H2KNC$Sdz-P1M z>XH5{&eMhI-Qn8a*}G!j-WAewk@WY-qCzu_Sr(Plr_TB2b>UFO^9zb@lh9I`3bu{n zSSK(v*fc;8lW62##?OHSWeFzuff3>&&G?IWMPEF+f=xf&a*vFzQ%ptXfRW7MdcmY* zIOD%!(^?el+hMEYf}~bVvW6YFnij($o~B>3CmO5*ZTb{6MT-LaVTb{qK%%KH9CIA1 zF2io>9f1J=K#hlR;Vp#S)440}MWJbZRlpddH?XzQ8o3*l@rwU`G+d^OVuk#m>|!`z z2f+4ivx^@e?kJ8)yf8O1{zuiy|0|z#=K+v^tk%rBwC{l&GmybHvgmN9m zA*Ubb2`3ZmREC{jtyVGX2#K2?7o)vtHr(U;ewX@ly_ik)vI2cN+~01j2hGH(GUJ$3 zdtaIHavS$|?`_$}{@VHU?>u0CF>s&QQ;LZx5Y2egdsnGs2aW!=Oo4^1knyKrlPD{I zVe4A`u8TRN>jIdzVs33+)Rti)u4z$1RiX7(B{2WS)tVCcZ?Kf=&;CDe(Vxbnq~f&) zM0y55dbZt|c&XbrdZlHNB(NEP@1?iK$R&=9gtOJwcd@8?>LidD#cis?zPZvTr;lHM z4;zJjGk|sc^XaR{a0S{k`e5r7zJz~k5(+jW9Adnt{Mie8tC?oO{)es z{O*Tdv|Da*OTFP$aMJp1xoNz3diAk~OIbs;+_KJrEs^!2!NpDG$Y@jzwcvcsX{V`y zO)c2(ooWWfteIaojc$vMs~*2fR(4mTS_d~&^goDbS$k+e-)w$SF}bMne(~(bCugsE z1ZIBz;&Slx{MobfH#BeP_;q~oZ-e6}Pd3-dI6$(wdy%)>r9S%Go=O%-H&#AN{)#ws zDf!CjBYQ7!nf&|VcUuo1MN~^dVx;arpAIfgpFJfOc$!>!bobuw;XgMuX>PBtH*o_2 z!Gi9s|OFZn?HN@is`tDjN*b3 z0AxzHSxdTb2gLVV3Odmy;cyK=j4N;b08CEwdRihF{R=oCX$j;EId6PvoDMVKy7S#J zR7A{B0#akzwFbL?xXnE6*XQYQ_waD5dD=y95h3imGuCYoW7<3)f$37dtW3Cz73T-A zdhHjU^QJw{sx%N-QGX_PJq!V}ejqMW z=d%uu9=!C|n+Yb%q+Z2L!Op(2`+inSitH_rz+=dZSUOZWQ0HoZRFG)8lWC~0 zEdScR%fPH(fA`itTyq=M=h_v6G#oE!iuekRR%22}V+8RZL(sFu1d24RWuwHUn<`1^ zW}DVSY8C$;RY-XF8Fp26g&zGDt{r+CC;9tIM4+eT{~rHupzYDivyyy`W|H2LB+laCK(8CXBQ~R z;^fWJ*^mGxQCqCnm?zkt*;mQ9sHVvZy@W^C#VSkQcN>mH&-qSEZlfB13iGyIVXJ2Q}dmOS^oJ zy+KmF8VHFj7oT1r+!bE6#vzjTl?Y;XDO2RuBRp6$!mWBtJ_f~Tv@D@_qg46g?0=ld zk1`^Vz|uH2|L%KUT+@D zBHd^}*D($znM1Fadj^ORxC+K8f%x4U@(o(Ht zxkYupiTC@h-G-l5>#OK2S3U<*Xne2zhJmKuI;{1ZBwEhYK( z#Li!Eze_coh^wIyk=WPHWpFsk;H#*eRRiDy+tCnX4WU>YKdFbTj%F`9T|gx$ZBui{ zQ%*vgD;G3XGc;4)aNy*;Oh07BvT~jL7@}O!n6H#qt|=kDPh8GY>|nER&0$*j)wGBK3?5uh{){%T2Hlx`~<1q=VD-De?N8t6rVpeuuCA5Vjzt^BxRQ1 zU^30%_gE;X90HD{)paVA$}UnB>WRyWkEgijnD@bb+mD=p4|>{_m$iMmgZKeXgZL8p z%+MnmnswQ#p$t&kGmG58rFHAU0{Csk&&=A|_x4xLEA&Mt`H)N#<;+P~^1eeS_exnd zRhNzlZeLO3A*VWmp)`RRV%n#{FcPEi&MAxKZ?t^!=k1 ztiAD-WW4AE9l|DPg+Mo6j5x#>T218PL43HLJfPd^Buy5WLxX9+OV5qP_^|LKT@=_i z+AFwA6n)s8Sw5K*pY(Qr_n+T=6<=kOh>jLIKT448#jSl|*!YHXh%VvT6!pT5WBv@1 zh+|g}-C2laLK6;40x#_T@}JfxUmN&gA7pOt$1n5(X48J$`t$|50<}aLh8#QW9)gE? zeW8diA7IStOO#=-uE!sG^?ZtQY|4%V^F|bH@dLHkP5HHb!Fka!!)yDprqGvMBP&ua zj^l2qFHSg}Ur0uzRd(S@xR&NZUvxx*M6zynRSbkGH;16)U!`VuHa3<(u5wfK%N zssb4i7jk^0Oo=T=^l9^mf?ycEq&MvLO<%%=1T8~ROhP$YM={qH`t)TkwJ>9$9Z|@8 zr|Pfsak3qp7*E#m0HXSohhZP{86{$ZLNp0dDicq}Np#RxNCKJ`3^t2woQG12rg=82 zrlUns-soI8E?}Ah!CVv=_k-P-7bq{VL$`vNLZ;9b1SmUDEntaR6Yz8MS(n&bg74}N6B=R+;B|P6{Zg&K2kb}m~_U@!_(~=g<@LL zr`boeqTrxW3eo<~IOitA5loU;ibzXLxH5^(Wac4QC3^}@t@dE3AEl$~BH9@yelo7e+#!p9c9kQhchup8VB@Z?4RryIQIPlNj9Dh0(3fdGrepxiE3WRO07e_- zO>5d(!P${9wh>2Vb)BN(A3emy}(nxwE4 zu&Z$)QDQUV2QMy9cmjXUBYmu?rcRZxjMAl(x=JPAEl4`#q@G5WERBt+Q_raGl)_79 zRBdrXH>zs_AR-NkstM1U8k|*>Gl0PZ)(Yg5y38UmR=-iPU=X3k>4$7Y9(4Gk0-sV8 zEr=3lFleO|?gbRnFkB7QNP2;Pq_-z3F- zy-A9D3YBWLkGSRvG9OM>U4-ijy{2t4$!=n!{88E^^9?VRsh?4bNHXFBNe1|d8) zy%zF97;K&?xF?F5Q{N(wi(igkuf+HS zC5&r>1vrT~X&li>1o<@CLY%J=W@54|i44l7BIj}>qI@%m_tU}xh|O_tCR4l$;^>EB zm99`#m*sMc(dDQ+2Xf}3MI?PJvj!7RPZq+`P+5)}r{%QewrNB~>9hI`Lj1 zs2sl6n-omDR_vJal1a#O3w44(7Kc?nayx^ll88j-gsffmdQW?s-gfj1t~NP|RHNTN#I zbQq74P0UpaGhXq5s+L3=CQD?lY3H>5$St9|{3`LQfIY%{Ul=f(Pqn@7@O^f?ODspE zIC>6S7-P43SQDJA7EeNGEmq^c>R$5#tj20qxC>^MIqi#Lg*&x{(KAImoVOS;^!lcb zbt}>hY-(5(_3LbmOdW0Ma7 zJ5tutX65d^_}+T90C5t?D=|ke;tI}aHd1GBU9|_Fr!yAAM$A4rmSw%Mp|zKe0=`b{ zn_j(6jBl9$RY0o0O5Vr(M(d_YzCaIzk1g>)sDwnyM^3Ff6yiN5lUagct8jS^5r@3> zyx_IvG}>a$oJ?RZ^X_(i02GwP+KG8l@P(n%@2a;=l#xrM(TGNI@8S$%aoDML%h~7< z>qusrH7T(69Ju|Da(CHfpJXu7l5aEw2t}}Q@koHfxP-|^>W>g@Oi_87?Bg63R3iFAgw%Ld?#`Bx$lsi2o1_?4V@=)kXi%zsvY(#FkaXjDO%08P+g$&K^z#BBf zES;@VBbUl;RcG)PtAd0#nRDf!N&QwHnk>-4G|f=jOWqSW3td4!(ZS$5}1wLE0`i11ww*0 zQn>sR=J@lgDPOIj3kN5JC)25uRSDICGIpyu!CMT*#iE|M23SecSBb*W9u=&A#>N{3Mn4KF$uls<(@}sm8S}RhdGDp_eb1)i+yuNC~Z{8MUDJ_^dQTs(5dx z{fqPp7lgNUN(Tf52h6L^Y68*KROv{I`@8i`PTQnmn^*cvkpvvR!lKHkbSZJyvd|z- z1f(0|iF20CpZK1&kj~V`p_(xIMsb#Kwopk|OlVaMBcH}svDuSy{L5x0SgDtcdXuI^ z*@*?wxg~K0_^}OZ*H%&4no&aysIu(xefyQ6%ka2Tm=_;31Tqfsk~bLnQ>bgHVhTUD z1PJ8%b$WTOs3#1`xJ*`*r+k%=mS$XnILihwXQ&7Zcd83sUD$Kk8hlK2mntcq%J{EmT(s@Meq^FL{r0-LOHf z6Cp%tZseUsVWqrBU8~YD!IdgKvftA{)v=m>B+R64P2@xu(chL?N>NCB=cx&ocgeq! zHCIeiDZMlWjo=e!?CD21=;XJgcBmZYVVb8C7F!p1IeZ|UHD33yN2&k(D&fRA6KRR?L!hSn0`_E}Yo}w(oq2#7qauR_b-FMWt@yVxA9` zThc;veITcynjiU%(RGo5SdDttj1&>?9jO=jI&5hL{f1#5=QluldT^1aQ{V?v>RFW{ zN)nK33L{844+iQ!QIN{nD#6r(xG+}za;Zrj1PwVo6pP(MW)IflRBa$Y=A zf6()%$!M96i|8={TP8rJ>GLDt8jBo_rc?@w19Sce{oWSSbWs#^Wzl$_mjCg;#{coZ zD2NKmBk=+OH5EEL( z^;LW&UwJ)i2!L(~e{TJd=4BRLF0YGe()!opI**>FOCa-$6xQRf@cGdsU6$ELs{3Ez zElS(*81Ie+Jm=5xkUl@EkgHCJ+*AMG=gabA1!Mk*O+CxrE7UH5skIQ0(!%N0j%bN-{2i6>8iL{8GB`{TlnwA#v6hC-nEOfOd zY&9x(4ZO;mbcg|8@-gO*kH*kJV4GCWJiNXJw7!-b(-BSSm(?b4Jk#O|_Iz~>`*W0) zqh&_BEKZiKKtx#t3#b0Js&O&bLWpaMC*FK!7M_dGW!j@G>p`nnthw z!3?_0;xXEv3$fmP4;|eD- zH>hjCHvE2Z|El>P?(Kc)`~PfBoRWp;~|VeDDYO z@4^1V-}%4aB>%@h?JcWv4=#kgH2)xWbT{we{KgpMiPxGW?0KcKfZT~riup}R7RS!0 z8&T-t;r_wb(M380J~0F04Yxh;Wm)uXl@1?Sc<|p*Zt0i6x@1))-c6{!zzB*m!8oKI z%Y+z(bDaRE*<+3PO$72a@(slhq3B6B4Ohe*yQ7kVRmEhnN=Q9Fw)MCw$Vm*+1WwXH z$#myJ-q`8(I8PVmuvU6}^)S-r=M*(=BUTh-Ii`Upv!XmUL4GVo;{;)XGj{svQEz)} zm&e~J4Q{?1f|z~XL-=(GwLy$R_>usmNsYG$_hOY!v1?FA#xY8E55@hbz&V56@K5UsWgn0#3#|9)`K#zSdU^cn^6cdGv*TCM%h#`7qAT|CiziU=i?bI` zUqL&k&re@m#?UT&7g0=abn(OSvuD^C)OP$D#_$Sz8J(QJ{PES<_di@lKb${%atdEQ zK84;LKYn(~jls}Po*kb(??q3JpC5mJN^hM*Jvc%v$bF37{BVk2Vw=bC|C7tJ^A|Xn zlk*psui#@3#`@|~zx?Lx;Sl%q5R<;|jUu&xhZuX2@V0ACZ)jHo6 z$HwE#*0XXEHnIM+D3|8bWRx$a>O)$VAgQSekrtaulCn&0)CUQ+)Xx^_$86CeGyMoI z0Jd|f{KT-q{c5ieCLa~%GX~I^UX7Beegc<* z7zkP>w=#8tq7;i_syhh6?Q)d1BzYiDG0lI!Wx@&WFoZX%>G61JSo8a$F-MmluBg)` zG@AuNnUUH$wt65uyF2)f+U4{T$p|}qNE96W;)bB)`;HS2xdozMbQ-{oM~2VMfe~ZUShRIe9mp=| z_+ZrZGvy)#ZM1>iF-X%+T&7R!9RJvAms{ao|HI{e(SaQx^pEGtl%jQ2?{8xTWpfn5_% z8@q)^ltgI6hBQ49oI$YDa3wuW*eT5lE9SgL${ocxRcJ1o`;M^+p&EfFLpx^=^PDOV zd^w`J4A6M~s_)z?Y6*^_!3jNnmJLVloLVVCr6MIraT<(%?V1vcrs8l+H!s9Ki%w@5 zrkqJXE?A?B^No9--3jY$>Eqwned+Xtl>1Vh*NaL4@_6a^+uL4=SNt+N!=SaE?H zLtE+tkYslA3cW6EV*~NPG}tx0fDIk)1+{R*ui9_g>o1+a-nrwJcH5`;Qr~o24u889 zEO-RWOXqIYZ3Dm2DXJLy#y|tuP`Af*F!lg`^A`VkXN|?lrs569nF%lLO05DDVZj7q z0$WT4yD(svlfwEnXXg~|uF?*9l$KMFTE+3CME^6iD<_m|#YDt#O)B%1`Udkj;oVj# zSNM+U22|NyKRh#xzyRu_*5SZr(5vyLDbz1XaZN{?&W}qAtc2lB)E;w5qlL*hQt)14 zCs|#LyM@nrtLkgr470u~)cpFqgEwTnu}wvT#*^#Jkm-u`>bfoJB$XzH4U?zVpzplR zt1m@icg{JzP(n^VZhe6%g5NRsF>ri$5B!l$#(}Zi>a^4bxmr8O^;`Hi9s4RY_DyG| zHe9G|-FR!q={FrB{4(da@%p~?36_pQP3lm(PxN`h#O|Zp99)qk?CGX<2y+ILDUaIr zg@qF3e3~rK4UO>}DrC1iZB(ERt8v@AzH8Mf_Rh3uKX!&zTY0;rCGW(@y<#h6Qr>i& z7Hg~Dk>p`LwGX`hI>H)ScxQ9GY$%;!wop(ud1^$n;1WsiPM$n_*3q5mAs5KY1Di#?q){M=g_{@)3N$SofM9qSKQ7 zcF)v>MuowAvkI#Aqia|~h4&8MZP%mQk4F3HZZR16^lq!VHgw~bjDe7=&`H_At5X@9 zM_tQ8IPK7JBB5d7k|vi;@%K}(v0L5o+w}sd0P)A1%Yd?0Zcy2B`S3>dSOKU~+7AySiivLiJUnH`uWRCi+fcGaQ)+d=@oGWeQt| z40aTzOy9#Sbh_Qp`)ejlOB&rR_RW@(#@1}pFWtuhspL-dOcb}+6I9|-gt3v?$$}xN zrBf-OS3=t=;Cq-i(E;OJKcj-39c_Tw;R!=myJDISIL5oQUb?_&quCGqeJ5BNQTd*d zuRLtcj$t;W2J>j=?cGjNjZk0Qt)jafS}^>B(ERw*JNPI_gWYOJHUgsf!K$fVc4TgT&|aKi+!Py{NwcM++h;D|LS@20DWatou5N%H`_b$M z`Uf*nD-@HEs%v`9t*x&sTnE&C|8SG(3ahE{-9vQ$K8kkCgiR_1sJ`Qs#R{M-{&E#n z(1&$us_#>K5Gpsh#a}++ZG#pt!c_5u_eUiXBFu-fu)bpuC+2kftZ=e}lab1y=Ybv= zMl&Ii7^VbY5LSs16I{&G95qZ7`}Nl%%R6SSde&TYfwwDx4@@{i%b*zkHOSXNzdNcx zjBR7GKcu-v&t^?xFUsN1hN-;3sm$$!>)ZFFavJ8w@l)F9lno^rM} zpjsR2H}vWbMb^RQ+f0z|m}tih6z2}Qchg>EIH)}8v}-S5FjgJ4!}(TX+mwWjrc028 zI^DOb>UYqdbvty__eP^wLzz`u4UujOR!3LfgaV02+^pz5nuwr`3{Zoeil+>$3*nYA zsTa2qg#tca|?PMXEJrzHuaZV`Mo?osc5(fv(&?iGQ7EiFE0AgQ#^GTlxEi3oMNk-I?B3&=jTt(p8jZ6U{YH;6{vEx3XU%?U!6VfN6*tm z0$U`B6n>4@0V!5{pHdwR-n=@yJnctskU)^3a-#Y}fRKJ|&PxqdzBrfXuk)2#?V2j} zOSMYRU!K0OrU?mEtqv{|nBza6J{h2Ek2B925bWP;b8X5uRCak@s|xnfkHLztY<-Si zogU*jFP1|y)x_D?CPqrus(5nx?9{J9(9gUoRLO=a2>%N`kfxKUVhiUk{>K2CdG++!`5Q?@ z!HKSc0)coq;XrA(F8hsIb!XqdIDd8e#M&Ml4C+^+%hh)}LoxOx%RR}hI>1~<0VXyi znZh%#|G*va9t7re8XaO%@~^x8kx`A+_l7}%>5yh=w za-?O+LV>}6GjZ7AADof}y3o>nVlW!q?P(99HpJ6^KY0c_VUR~2C6fh%>fNmAnt`cW=GZGmXlw*Ky?X0rD=@$J9{5Y>jG}%71-z3okSOO@ z6!me|5{x`XF2usb;0?ECOZf|a-|vuVvVJ-)XtVJ-Q|KZ(KNZ0_x~JqKygAQboL>BJ z4#)W%&j^LrGNd6QVAZy`lyz${S!Vjt@e~88XrTp{h1@YP?wL71n-$@T?blVs=?QZS3QmR zh`0?{p2}V?XGy+0DKnT0jATwfE?}9-<)soY=xI^BU(Qv9YSIN}qsMRuWh&kQPoEus zKRAJt!=A4f!14XmK7TpD?=RZ@=qJ59+M2&cfdi&6Q zTQNs&+r-+0-o8(7`<#*Uvc<`?-Iu?imysh1F+_n%gm-CMl^pE%*ymv7R1>HIS(mms zVy242eRm3&@$sfl@hffj-TV9g^ckKfP{ZYU+i(4wU=7IXxD!A;N_`0q_WAt)m*c$8 zTtxl-!@&E(hXmdyUcp(#`|ABh=HnKYk~@vfYeuE*cK@4=&6`Qp75?08y=fD{X*hzy z;k3Ougel}sG%-p}C5E+ew$go7NdtrA)FGUW-;r-r9X?~U)yOYio}Qc?KWiAfuoO;{ z2fTx+$@QGnt@eNmV5>Kv;WP9GJhVCx=AWD@tZQ6Xao?&U{YY^oDyL&%F&7L;UAcQw zE@A6PONY3+wcNn1X2j(j+^uk7tqOPIzTJtmkq_{emu_%-dB55Bm)Bzgw%Jub8|{+H zX4LV_V|D+&$}j|knQDh(k;STdA8Xb{Ai1Gy^t5F&uY#RIKDQQZx5A;KL81Mhn9~isPJ3!ld1?T`8o%<6cv2fkNdXI6kdKAc znV#Lzu40K#xZy|~-$jYXX}uU22Wi@DSOPc7G&wj^F{?s5PThKR=za1b8!wY7tzBgg zC}A#~+2}T&3$5AYR8)#HMz9`PV5frN4J&;`5Wp2-(mG88;+CcV=ubtM<1h45r`Fb6N-apwQ+?C z@!%}KZJXNL*c0=yVVAx&C9u5xzVTDDlFli@z$^s^`z$fo6V?1Fsb=M>Ght~2^F_9D zb~v!{_sDc6X~AIu1^s=9x`iQqEzn(SjdHlR2My%^c!3ZV#?(_q9g4-va?Y^8?=sxe zbW$g?^t`VW;HDa%=t?>7nEAaST+I1YH(cL$_Tl8weFZOUVNr2H=oG#G9`zyFNa((} zNb9AN^Z?sl1W3W#ZD%ZOtkE`=iUQ5VkTakW>LK=`_vy`%+Uc;(-|oNj#+~s5JAQ+M zcEI5uS1`+$Y3S|f=!nxCw(IXHtIKuz5sg;XB$Ii}l`GOIGMmOPoL*+9aN4V=Z=&y- z+u3QicifeY{sg^??pF9ecEVpS=F7!^Vf$hblc!@&=f?)S4LqIu_8bFZmb9v!_EGzm zZ*$w!bsnoSuIjbtZb!kMJ8gfvp>6k7I_ErS**W~p0RF;n2ZQysRkvu%>E?IsZr%09 zK4}=b(PX;BQ^b!9p?sCBh%k1TWY)s9R(uc%^;xKpBm?DJ(116pgdRQtkY`Jhlr}%(Z>&>Rjj{r?G|QeeDc)(cCYna$`{wc3?1-{^KgV$)mXu7gX|{Wr%YB2j1Ddi5YUv9 zqZyFkQZa1Ne7dZd*D7iAPKqk*j~Biuuxa-Sv{Y2KV6GLKx{wnNTn7^fTo)!KxC$;- zwKhjishf!I;2OI@k1U^d726{!&t; zoGmKx3xI1FHL;lkvTzXVOt)xIdU5>xG_vq6_~+0RdV2i)?AeclaJhT2$#ldS`TR7x zHJuqvjq(>+;9S^1{N7^QWj!H6tFa#6aF9|iSs2hqr?UNXoNY_%S0@n^XP{s@B!v^p zLYRM)v%s?6gSf(qv#k|*y&}QthiWkS26`xvHc=M2LQm$DzeV6eVLL+(R(6m0!jnW4 z+<`Fz(nwYTlj+0zu{N-QM5$@@@Lu&D(rZM)T?kb3%htmkx(z{J=7{B;3*#PEBcgyc zPsamdb&sT|n1(vmCu2%7M7!bMGzx8W8t*twbbT9@%DNi>)o93iz4KazMg7|Ucz6Gs zk2}%b$ZN+EW)F7hfD)_bpG7kmIBPHvR}!y!)~pzPGFMG!p`o5J#U5g@j>0<+s#*n*arh zPH50fnw#e0rm{Cq$GLiHlDGJmA*`B_Rjq@Zj7C7&ZU1^ElxmLwiLd7A2q7scpo~&d zl9vm?i6UJzAV^~GDA5V+ijv4_Vg$E>ksvLOhZ}I!|25rW5mIy%}OfPn^skiUn0r~Dm-xXJVCGOfB1y{^shOG%NGX$b3R95!cd)!33^Js5;us1&3Tig>p z;`ks^``xk;c#4!mKEwBbgy}^?Z>tPsEz-m|ky-A`7a1BS@D!SNp+e$ZR%F5;5vF9b zDqf=5m@;gm)E=mhy!wZcCt&H<;h0OpTwq%!q9~_o~v(N_)yX;A|mA;oP`* zP)Z(Xxf-Q;Qf7tUt3tUC5Xjh=)72!7Q;kxr)vWoo5u&joyUOr=ZRQkr3^BCB)xFms ze(05d+qRtx$VHsjuH{_Tokn4jn3I6)=28A;Nlxt#pWVC@y-B01LfQ0*Bgff=O|ruS z%<>%;z>-{>8H57(9k^q}dRR4BogreBqXLf2xcES&0+$tFMie-;MVb06&=(UGH;IXB zC0Ofd-AY>g5^|#}L5lK236S*h7FBE~Ki=}v(ObOvhM{23Ew7l1%j3(}7kw}36fiJ^ zo=PYIlk6jF01R|t#$(nXJ@6+x$=wW;#(I85dUUBNl9 zrK<83ipS86E6(Mei4t5n#*k^$cX2X3!I$YUp&uPaqhBXu3!@FUFmEsnR_7QWa&Mqi zpfdp-GS>t9@oTvsqur6nFC+bpAo5qm9^NOcVFbl|}DoMKbD3R~p zq>{7%mQROriyo3s))LP?MMF7$jAWA!vI>^QO=u3rF8m5Sg2mZ|*aY`>XOpJ9(e-X; zmx|(uRK*@)H+_z!fG+IQ&M~V-s}$`H3x?D8`7Hz!{ZM2aPdsJFf*P$OgI~2wWnZQe zaf+;(731Nu26}4#WOyX#NwCad3e_G(@NZy#Y|8x+)Atl}II0pJhUlSk0dRLzt9ZK6 zQ;yp8>N<09N^PoQAWGGhZ>(Wv>(wvylGquVT+>N(KwAimiSjhm+lW0v@Uo9Wqx6uD@FmC5Po20(WquU?!p_8FmLrn4M9FX(m(B zY+-@NEy`uJSYOl43|2ixHzBx0gp89wPQLG!E@+PvLy}i#V|~MPvBC>U(R`-9m0eGd znN>1|!sZ4^A#Sd3zCI& zT8leBBC|!48#Zp%$~d%4#A8x3;L7ErZy#*zAw74IwzerTDm#bL|6@*c>`!u$nZ{vec zqkG%VDpc*0h6SszYcP7aM$H-rhYMzcCE5nbFNd%P9mCgpp~)kSO~ZIJJx$Kozl@l; z4_Trk^5Gn!=!pRlqTV21XH2^#Z{l54!v4zToH8`ZO|i6%EUJ)ptwI(BGcD?dv^Ok} zl}thflW3-LD2NxJh{56#NTgJ}s?nsiTokzVN2K;qWM1RJ6MO;mTd{_cbdZo}aYp8> zu_J*Nb_r1aDz-fEUJ9>o&y)>iL0y6gC3efY0r@;T3Temp4+DKhcpErja8ZZqG4IB*i-? zK=C{vb)scu>2nh-dG3Q;itfT*^wn4Ilk%!+xD-!GdCA`76uvugo!vDSbRD{4F242W zvW<19fgJUoe_E8)y0u1sDD%@{MLb<|UEd8+HLTk6J4WN}>*OQ)7*fdae2O_W7Hp4B zBFwNvnM7)V37+d1x^a@PhI4S=r#CAMkr?YNA4=ofP?PL&O=bMCN#Bv?HoH{nT`+g& zlcTYuYPIeboRyVwX4Ag#f<$BmiNe|Y!$ik)WA7bvq0i=v8-~0j5FH6j0VY}vK=CtBn?Aobo%inwQWoYa zd|6i%TK+kq%3#e66Qz!Y6)uVd1}(L{X)h#0qH{-IxkV z9wwfcJuB>qmBnt50av71A4N)0FFGP83+jtxj4tY#2)v3YyLc@-X=w7Tx%ML^M9Ja2 z5e%~!cw0ZP`et+FDeAKZ(&UrkQg5u2`K@hYpLx=Pl_{pTMJ*nOO-b4vSS~zt zBhiEa?SkE73;x3!n}K!jP_j;_i-+k-u`=4);4t)&M+HS98R=CQ(0d4PQ%*-!%R;Nq ziBc2TS#k{WMm>oQdf8FDp9M;&d@1QRMmIE|scEq^Y(3Ra7`!(;Hcl!Ws0xElU>dRt z<1&Wm_)PAU3kFsmXO!UQT4lM;mxw}#42h^?d~lb?jCbhL?Mg9hJr|Un4-m(VfSlY! zd!1X{XXrT5XHz($+ng7m<^&@)5>3=E%f{=pEmTz-b-vx*q!(GY^`FR4YC);`hB0SD z*&GP@2hny7^0XiED@n?E>ivsU?E|$*kb_cmExuiFXLKSG;DoZpJMv3~R>cOcE3h^a zM(KrmIR(3GgGtv$zVP*@CFU$&98aelQP^1{GCx_)W;Y@xsw5_Jd|UNdC8%~W)_^^E zT9h`!Bh*0T+{y}PMU^eC)|6ICRgbrbB)3xsjgwxFi)A2O$>k)W$U427EG(4jM{kG? zlm^tDR?9%dvrR9YbEGH>n=HMuXCQ-w@KXv!n#~itLu(YTAd)b zayCI%l6B_~5kpC4)Xr3zF=7NE5|MKctBTF5m4eN-Xsb>>xZwn0Ty>K(B2TufX|lyJ z3*yoecH=cuEa`nIcj+y+wPtJ=R$V&iRVMoaHuYLEcLHnx#v~yf!U*tb0{gNAP6j5N z5OpgUr~-sC(U~EIh(DLV-7JzJdw?E@Fqhi%@8VJ2i4oL*`M0on$v&<7n9}MzEXE| zUy?$@>J+)9n74b;em4xu!o}a3qNo(-3jY+D4C#}MGQ@@f(DWpr`4YB*lR}ptkBa$C zI4}C`=Fo#u!;wYR(@$0&Lu>GrRM1sX#pi=JuP{x=C;FR$ zfB4TA=bSf#KVcRO^ZJ^plN3NojNd88>@xLR6+OhdO|O>tccX#{X`8j5aaK?{dIquGK;$5T#u0Y~ zXn^t0oy-w zwT=o98CgdMHgy~?w2yBO-oc+VIo(^-R*~Ya1C$r%=_5hUv~N;$7-{>21!NJX0JCJ? z!4dbiCzC;%A^fxg+t>nzP2+L|(bx1-L0q$9wXIpx7al6%uktz=T)kgzWkz82N2C{nVi5EvOa`#uv*8)>5wxGo*O3^f3b;zY z!%FB(AVlTohSKTh$IH<{};^;AIzb5O_X3DFgA|oV% zTdk>LIL}I0n-mv3{gH|)Zc8O?PmopxFAls$=!Kz>?y72u1kisN^}Bg z0`$bVz}Zj*f-o$VliX=dUCeLe{F$Yeg`k=kF*_lvQnW(nr!|!k;Ghr%{WjD)CA(L` z>Q#aI7XX`N(OL(_x5yaIJE633mReEGXXZ`in%a&R5el>mo*^?_wAbS3X4%#Cf=`CO zb*#-1+B&?s`xsnd)td~94=Js!(zf;#0tdOLrHeYyv2_4p(`EXAU8K#6c)eK6tNz~J zxBwP@4zmlyy(q8t?(I#pVVRUSdvNEZqeXnZm`(4<-*}YhAAh83EcQ5Eh<%5e*GY5( zCYSx$s?0lUz9MnF#uIfi?${W6FOfKidYi39R~aItI)N)(7mgcBa@}e+YKk=}6>~*t z*-?G!sn=eD2yynp?gYazWOK%CLv#6v99fMM316{GasR#~a-5D|gptRnrlY8{d(ew^ z5A2345C1aAbh~Uz&bI; zm)(WNd)jNvze#C4?pL;SQ!BnE8$qCI&wyrk_!&kt8BWOSrgc%9lkb%=%#}r`{4#Ps z9;pLp_28rcv8WoE9E|h8mL91>zH`eRA~cfl!g$g5_##{Cpi5l=SC12j#*EPBs#4aZ zE^O7H1LCn+hkVQq#EFGbE>0#^yg{QO>NsC(|JEoao_!>&q2>dl^hTi&7u+T5{6=b9 z%0lduX_c9inNiGMTjjYcQog_(7k1Qxk;7BmUZ&JuIGMXJxL^UR( z^w8T)!rX0Umz_6oa7m?*>2PQuWsgA4=1puXCZIk0{WNF)6OgcqC2);7e?88sQG#A| zvEE5!!>0FL5l4cMG zX?cc{tP*c6l~f?EIUkt`hGEkTGn zZXsHq;9CqPmag;${E%c*m4c`wd{+`&U~9~2LOn|FaD1&sCKJw*_@`2Qw{|bB>MnY9 z=B$S4r{Npb6%=b(%`mES)!LTq>N=jX4vzl9Oj%uW3xR(-@k-aoK;pJ?*Exv#cE??S zwsfrR^ga|??1`@pKy&&Ickc979f87>Q9LFwpz?y3E`kxSgyT%t!d{`Zrz~UioPE3h z4s{3cF)!lNvlo}I!ktzCV3_a&nuJ@2Jj$chP@$8F@nmh?TR5Kx1m=L|<2Q zACA)x*(gQ$i4531&4$4BicG)UMs%@;8Is|0f`Mg8Std6z{o840?Ox5Zjk5x%xh1r~>dvyFil3gHzj$?e{G>eTF<}r`QCm*I*@i}0s{5jaToq$(@C9X5|W4EpfO8P0}X03o(a6CYHRl*GkVj( zCdHZ8>uzW|nyv14csJmAq7+BC9ng5>*20!099SpRwOVo)qx!~#MAcOY?%&nkL!nSs zH^T3*)Ojpjj*a_EIKj(&o{ZkZX*oMPYkz*`p7JdYt&jiwiAV9S+cX3xBN3fmHWb$S zbazZPFx~5|y2m|<{v?yxcRs-%y0%&e8NNq_Dy~wWvqN7*pdiA68A~Kb(EQ84v|JS& zC##^b!Cpe&YY*@w2oG6bv}F)R*=6TKS2p1UDAI}SRfjq2Zp~lY?~w8SXa<$!kY!yQ zFC&dXqPWEHjpuQa$#~qcYU*x!M+=iOuJNxruK20YW#*0Wn#hmeP>fwhC;J%FVFlcgDGlY9b6d8;PK`5m_ODK7- z(dKlw%P{?mvC|P+Pbv{s*rbvUM$TJ)yFqrf@kIgISclxBEDHo%ab2v4)aB^Z4D3D9 zGE8w%swYei8)}mn6wYJj#m91V%|ueCSymO>ciOFvqEJ#KxYhfN_Tnt+5Na26gcJ}3 zZhq`zH*U;GpaL~^Ugw4*lDmg>@Jie5KyAPV*#<}bbM{Z^o&;MVY*ohi%$}yX3ghS7 zaE#h$+K67G+cFI;tF&xvW*rgrRd%Yk`4KRdAb;e`>9i+t74}!*`v|yzvb2gd(0|n) z`+-?WPvJo=%7QFjb_Z3-Ko<}dhF`<2yZ0d}_sV6y_bu`$AjWzIc$6vLnXbi zp;h|(m3OcscGuTJn-l$?>vcRq{8$s;%04&j?^ngh7OUzuN;d48<2Zd4<$F0Bof?wN@i>`l zY@`zeiu~6&x)@!TMGA}Erc;{D{pj3BXv0N|YJYDJdEFR8BR{0*UzNtN754b8YVYvj zHxKU}?D_N0Q;!|%)UQr@4uVy;-<*Ksuob`s1F}$M<4&DSuWNCOR^_?^9;h+d!sr_D z?%R9qZc~XR-rAl0V*l{4dkc-xnh{II1K1^t&i*!xv99c(wXPyJcwjup^10>fDMW!h zLSH>Ww^6rgB$LbeVTWI=#S6gK63u1mBkB_NL#?}#dTr3V2emf%oLOza7oXh*y?f|n z{(F)xFkRaM7S;C7HV4fvOp#{}Z4Z$VFTc|-bI+A&&^w=|K{eUvrt7)t4(KLP{<g0@uAXDiS)Dw;FPs+**;tA#o z6fR9)i7JE9S6{8e&GB6Rv}&8?fpYsqvkci*m7;`JC8J7rBFKk*R;b-fvTT5(B9eG^_GgTFgu*YxIYYSNTc3w+z!Ds0JNnNcuylDd7_O?@{rn_X9DHIye! z9tp9M)6l@zv!%7J_?c|Fj+Ibg93gC<>rkI*aE|9hcb(3h@6Yf>?v%96utBT%zoj?B zA}Pmb%5)u)3@#8@Nzz^jM^j-Yk_DfZC3^jKPKta{7Sr96fI`mV8+7mpVH8zXwQ)c| za+lyvbb`#ER$P0&gjk`i^`Xye6TNa}tOux!{=75dCyPo*iT18_Rp|&+UKTeDCQoFB zUFr#H9WnmcheVPT$8p?*-4+abPx!E>oEe(9qOQkNtYGDYSg3Sdf<9GM3i@@O`<`Y$ zC*w^yew3fZGOu~_Hn8a7$Cg_%0VJ;VkYxx(i6!zqeLE&1pC9$Kx}%nCn+K=zAxo|n z0Rwd_A+}3qEn_^vMCC!?-Y%pu$|a4)6t_avhn|iC5WTm&n2vQlDs}?a zb8+J#^=gPx%8OL#1VdxhHA;4&qsRmq?>SIi^l~x}#I}h%rwA;RK&Y(TD3VNrx#`Yp z6nuO+O{xr0wDlbo|L2d}>e+yV22>)gnghC8zK2pEuzQqSVs-CY);Sx46N?g;3cfWswl2gxf>(Mt>rN6H}Dze>vyh|1z% zCp0&~U)zRbiL+=eCTnXxnpQ>eG{8sWagOb8M-ngnuSLf306dcQphs#kV-Az4si3XS zW5oZkjjvcDL~U1*Ah_{PXiEBuwqXbe4a@Z8ws8y5Lbx%~cYEh-w_{)j^W{3rw;?=1DUf-bB zqg@GNagCnciJSwFRTp>!;1Yt$C^b_~QlW%FpgeyDtc<)&%>=8B%Ai zr_)zRk<>eFmR4UK$DmBwlZ`|u>BL@Ag*f`I%@m*vira4uVP!RfmZ^`cn|!1N>H(H5 zY0?BgVO}!SdJGWuk5n1uX9?C#l{t8`hy=l&SyUggkXn@r@@=7V1rkFi6+-nOd##G{ z{XZBbq(>>^@^lIf%(s$1CB%qEVzdARAWhWJN)fCjOxya=wi;~9rr@N+1}R=o6EO21 zC~ZPAcR4~=JD}}MGfs%D;4(@59n5>;4(=bvEH&*gzb7V6z?wdbim>nEK*tm`VZ>5; z0X4gebUH=QUX}febw?_T7&f+&SkTl?E8>M8x)~YVM%ZGbu%R^qTzRx%~Q|G-2##%SKs2w{SwvL*=cHD8KHTe(O7Ai=({e$?<*TEL6p1NS1SiMm%m7bKR3QEJVjPdN8Li_6~=8YC< zN3%_qX_X|wI*ZLzp~Eec2O69@ey8ll4W{bmikfZMIDgaBvQ_^gdvNVAX2pO`$&t*h z`_!6*fi`9x={ElpkY3pVn3-H=-^KQD#-I*$j4^W?ATx~ww#fy*L&DFW-W%E1+1Z;=VTqNc#+?y_PX~Bhx&2pOR<mHW*aXVo?n+&Z~nNN_&!7v?r-Z@w03W{2Q(J16bXV zziT`FIGQfU>41S)us9|K?KUO`#IR4!zs$j(W*4evdXZb_Rz}YN9>!{{%#g--RkK8f zO&Hk0e)JM^VT-i5afaP`xdMc<_G-pK_oL_Vum1`^33W!BDPI~(r$=IH?bjlulsBDB zaz8qDd@|rG<=BBBI;fntlB?Iet8^_6?1qA1a+Z&gT0r*JG_a==%Sg#>P&cjKNV)`? zf7i35wi*pTc!SzjXH7AEv?y;R-mEv%@)i1jtUL@a%B_Axd&A3@h(V1wZgR%6dHHQ2b%SLp)#!Bp?F(FS}Zo^H1r z{iUHc;`*SB2~XE_UgBu+X-q2G3bl@%_MqK;XSp@fToW`s{Xa0v&K!NaX^y@mv^{q3 zHmx~5{jR+Z-1?M{#8W=x&Q<9=BA=w3Pf&%v1$>4!xkF_>0OICrPQ$=hr~Iyr!) za2oHt-PubQqrGwzf3?$#_``KkEpX~~9EgW>a#(wNXK!}1hsStN2tW)y+Ogk~yjbQb zMffJbvfDk=eel0iU1>S0q`u{0H80@mk}zdO_=R=J7&9%+N6EZG50bhYskwMFO3Mv1 ztI0jlb9M);DgmpdY=L7qgxS>R>6M4T6`XlJDG{xKmz$w54qvs~2Buq@Q1G=wBo84*-8jv@Ycg?gwZ?u40Gq>t6PdQ2p4uU;|R3)f}tGah;biSn@D$ zdG_jQnO>zI>zP6LYqT3zi;#OKzo!u~C{lBdgQD+I1d3n3I6FCia_YoHHBCih_6#1T z)j2Znc!?s~z#wxV2?zneO7v?fizqc-mg^EEpx~E5L*t$3vdm`a$|L?nm{T$%ojUBg zMOw`h;2T>Fdk$qzs5wSN{CU{roQ|)QMt~3g)459Sw7cu^!|G1c)k3X@R9K_YxK+$z z%m{lz1*p-o5o6M?<9Fd&b8#wRi(`!`fxWr{xdtq;0CP!_Y(cS1m6b1#8lP8NNCb*Q zdd0n_n4pY^$Tdi?n4vPwxfO9sZ9=FRYPt==6I@|<&IADey{uY@N!yb|7u zI6$!uG@B#u3Omlx+#1=+GsHOVqQN8+jDd$imIO`3Uy5`?Rls>HmDsr9@}5;!AhgcL z>vskMZ{0Er1-;^Bp-__RY%CE~n5PxZp5=|cjY%9`jdg|r^5%fauS8T02TR~JM2a(9 zWHziSac@QGdY0r=KUs<}SigFxQ|pt$LlG2vW@C%FQOi@G)^2d^-L-znr^;F|9ss^T z=+q3ykdGZ{H9A`u^dDYWGmn|cYf9m`DZ2UyClOJ8l$cGSR4OCqamz109aDDU9z|M` zS^siW z5h-kTGkIoD&|!aYkK5D`yr=!M z6iK$#727H(PKx|PnrDO)$cS1x_pG3eXene(5bNOZ|zzl-6V5U{=+D z#s$UZHX4Zmla!x+Mka@MCGJKH-|VZ;TDd2=;pBnOIm*@=tV(1$u;DX0bvJMLiZ2Fe1VNfu;n(Rhm~MN!PsdP-=i$t4y>Xl^yYw+So`H(t`w@Wf}V&oot

y>w-0k;31nP{WrocQke%$Sx(n zUD*>qPN&+S-;7EYEjnEe>1$dtMR*5Jy`g(5k4DZd`YnAXap-b zd){Hn63Pq# z47`Y5SbNFA^FTuX%2*}Pm#GQ%v<9k7BUpq6Rx^xfrpg-gFE(`^kH^a7i%ZXSGJ<&G z0X^m{B{@z|y1ki8a%Y`A56sLv-C70MBw1a{UdK~)#FGnD@JN}hk4&{k?y4T?8LT~w@;-=*gJil&ZmI!Y*@$JXoX0cN0wZJroENphSt)?0W@hbVhXAUyTOl}b zOuBFv@s6lOO*OZ24*S)#7&<$CT+nychDN*%{xUfE_#n=x9(OARRK)zoTP*j^$}hCV z(bN7{BY{|Ityw1lVUk{}kAGWc>4MVogkJVmbd9I&v9h>#s;(mF`o{l+o&SVSIDXaK zo}X|)p9sXPqb;0Xw4L8M{KDEDrl@qN>OY>Z8%akeP=eje7(PqXBlsGYrS) z=Lo`eC6QfGOHA>wT#!yMA>x^;{5(K8cBM6QHkLzeW~%ceuJ%_PaA?)=EA(v_d(beo zTyh#lSyqs+WF}Pe&qo$8DX*&@VK->n@NIbvsdnS!<D1{-8o=#F@ziq0AMuWtt zO)xW*)T1t>ZmUn#A{wbU72V$;I#+OkXr{g1U&?V*Wr$F$rk`s-%{m<_H2o;cPtVv5 zTfxCHr-9Z~Oyrk+P?`JW62wXsP!pMH^v&K1z;@VIJ?ke$VGh@iXxw;4((y*|6sPMa zSOB`aer^(02dqn0Va(A_e2F4+AII^}4%D@`CxSZPnl3w};r<;qOCYCR5N`Y9EW1aeY#9sWZ zkPBVuwY^HF?+5ToC*k(t&jwEinM`-}J7t4q+!1eSVoNo%(y|d~N=ugcMZScaWYjG^ z5tRV@i)v|-b_3HI650dTAo;WFS?eo}I}^OSRrKb`(cN)R^m2dMw)1NCSGO4NW)n$w zZC#WvEo(|F6jn_D742RE&wSJ>&b+uSYt2^?+aFnG1L~8uNApLzwBtd zRh9s9Wgre_>)Q?bM$RZ$rR2%A**fcLa9!)MuYrr9xf$jik{V2}%w^&&&wnR+ZG0zUdg-Tx#RAyCJbuzR#l+16||d}1tdhz>|E)U<6v!a{So z#yh|^D(nZKKQbwJ3Fh=4w<# zwN)`$3uFs*l~>N1t;WVv+jgO?*CBJeg@3kIXTVP8v4!;+Q_C>wDhKSOj07?}@?070 zH*M~Bs{K7&g^8sxS;;mKjfo?*Gmik3xXmTD)mZa-Ng#D31*tKw%+&>@KJjX^ug<q_@G$=CwVJj zb8>Ui2PL1o%O)Z^e|dWuT`6Utf4$EZx;G2kTMF5YyW?|q2m_a(D+eAG~ELfw!b)8uuXjcLIeYulJ%(~d~?E7{bIVAOJ}X-7GsBc6`}^RCK4ZkPQP+#z;1 zN&gPzRGtIy_oh%F_!q6g4PYSh+?Z#Bk~Z#9E*duq*Z8T!J=!9~j|v6@@W_uGJx8Q( z4N~%au*UO3ijeNqb#EMR_VIcpKJ|v>uzcm*E16Xw7Id5?XOP2J#hwk<*^o^eqXNSY zMG?Sh)|^DoZ=QJDfFWcJA?><`3{u{(TVJ-&(Euoct@8`GMa38DPi~sm)q#+C|x3SowDlIRe1O$L?Ra!hEtu4l9`G0Txgu?NfcW~ z{-wCsfoe_Fq~SCO!Lyp~$Y_W*D}UHiyV8zmTk2C9L2*#gl6yf#)=yaZ0w;b;-O6V7 zp3)u=3+t?`_dLwD(J4^K?Q%$~v$YE0cX4%{P`>+QC}pL60x_6vy8 zjS|~tX-!FLswFL@Nx!!j(NJ>vJh|*YkEDLb4QC;XU$M0P=WFth$8*)k;6|b(b|}hi z#2=>1v>Z|AZq-$%s!dy>7f`q~NJDNc%3x8VUk0$*A+QU_rJ9t!O?c9bj}YnFBF?`o z(dUA5B1%+_;iiBZw5pO1?7|`(&v0#eruPrHCf- zi#_viD0lv1jFqv%`4*gl2&ZyY7OY05gbdJ>b+#9)LQ!OrmoPFKeK>X# zK0P^oc6xbwJ0lD2?7nLt=`iaPuzPed8{P^-G}hQlaNdab;XrP?YCD89v3wbEXRWJG zK?3wB*^;}8hy<&c2IaK6t}X#f3L1_Avtf6+ab6t8Nh89c6BxT6YU#Un?XN9llNmTp zzut%fWoRQb)$ke5Og;qz8dr=$9w!2}c#~Or7 z;DMQtBR-{ZZj(kkZqUr9EX{U)Zf@8WZMM$IH7wZK1h32v()S}Z?CwS|zfETKEu({4 zw{dw3L3X%G6$ekj4-j>NFP)MW#%{2ZUWCRH9|O0B-hhR;D*uU!VeQm9*{e6XVN!OU z<89@}Cer(O9?-ZgCh{GF7xOAvofFs}hmE?BF%n8zgUW@+jNyF48qxZVN zy+8?^7ybO8{`{&;)7+Fete3b+QRQih+zWLBR#+FfUoRjVO&9sPUSxEWSp9eqXarYL z6$p%EiI9A{!2M7!v)oKppbJ?JBGW>bcsOhSY@s(f%kP785ezwP?gAYNvtfprng_FD zoOZme^14brc?v}#ij~hrL1a~dF;pqLk~!bEB)%%O(YavDm&FIa2I9d?lJ))gVSK=h z`E;zj$l<*!VC1VnOWcaY*;aN!Dv)0?bXuxl>Z$>PnDdOb>HeV;j%Sk71)e}HRSeA5 zS8Za>oUf0cJsX_<`Siu*MOz_$`5jh)zUb#mSAv0Y(Gy9U2J!Bq;r!D-F(z#_Pe&Q~ zC9x7tu*AmD(P=TCOEyi@NoL6_rzAh@>xBH;aPp}Mdky~D`Q9zEz3A!8+ewE zd%y8>2O1Klsa?y%7igSc?U{<-@;73`<*%cY^XD&*FV7yIJv+Pnaq#^72{Xj?ua)vd zZ)M;v#sfT8{w98GA~>~ilA<8l64>5b%;s&$hdzx2e{5pqdzSqRX9x*qta3`qH%L6I zv_!Qu%pj_xdl6@h_`gTE(Fv$1Te!=^dZzgTL{jt%pUsPMAwF|2@hb@{qTC)Z(jEGv zAKk^x620Yk=;|E@$sfq?L;0P*lWZbXIV$_?iaS)*5&he>iV>&I6?b-aKOhM4?k)^w zSX3!|V2Gxp9YX6EEMVq$nm9jL?ZwfKdG$J-&X0Ef%%ACm?ld=Ca>LqO^f^siFE^Cq zE*)DzwVq!h#pbQ)X2RW6;=`zhXq%96?5eHzh)n4{6j9;!~}!jzSC2mPOG9-JNdp)Kxrn zK!g|7=PahYxOO$4WYv6{0OefHW;Yl{y+8-@(Yh7=v!)psfbf|#Ltd3$m|IRgYHofa zlp|T~GS~cBq$61k)Z3fphkT&Z{2C>_vEb8-^cb({=ZO0$Hg+H%!j!g&j zi$^+rQfD;_q>a76_c~oWQQ6nsz=*c4@FlM2@pO7BNtC*30^VS#GAS|e};8lQ@FQDV> z;M3e8%VNwK!!5bwF3@(2o~tS)ja(;~vsGQIU9|CAD0hz0&qmln1-d57id8m9clwzD zKU11>^@O7OWd(=H|F7I3$mM+KQGC_E*djdBy}x+-WE!rNB2bk5;+fa&5)yvS;i z=xS~yPl%N~$0~O;x)895g*dlv%MvP09e%2Blrkv=_jVDFnjZ|!aku5UvY}kxvm3YU zAe@|dnGb|JwK}XOVDtHB$cw!%y2ox!{|_GSKREoI{{JKBzpTG4Himjg`mAD!@dhX$NlGS0ha{&-c|6T#nY2LwGdMpu zSr~+bu6fwo9-0#=v&}4}!OfRLxbzT5FsrodRE1xTFr`Wl$FheRHB$l}fjU&=oYli# zBx>_HPV7Raip@jU#mpT|hJBhW%N&|z&xUaUQ%9{5QjR!}b2J_$_r`D@iGy?k~4=d=H>y=!ky97p1RuRn#B+zw<8%o~#IW~)5lOnk$ufXt(E zQ!D{zLTz}I4N0ioe1Q8P_r>m$Tz5;djWJ-dTz2cqs;LQQxrgI`AD! zvQI|$vU$~IK+)D3-7lsmD*L+XuN0d zAZDZ4Wi_L2bb)QRNl6N3hTcJTb-i7=ginp1^5fWlVRReF_oCTm8f$6ou2H$FX>HcJ zYPXsl9Ym-Ca8or`bmA7f-NYoJMJC=4@^g8O@&z^u5Em+VGJa6g562W#7yi%6KfI21@LZyp^VE9tkav$W^tnBQ4olg0Q3TAS#W(F3?X8<6hg#u z*{3LmrPU9ir!0@VLkgfe*)0Z8od(EZ=T9@c%j={NLK!E;5Xv|y1qjiB zol-EM3L!?J5F~~@>s!7`<9LTDtzw9_l`V#7TiIfumVjWF4Ebznw+#7gdAAJt>`9P{ z?m~1inJ`^UHcS_j3lt$lS#arh>ftnm9&(ugA?{|g-4!-hIt|l@WWw|z*#IHNqnYg! zqHeT0y^c}u!HAyEK}`w_Y>3d@rl_R?WE=zW97+b@L6k26Bv}!?1`DF6@1TLfdS@+g zRXYhc0{c6IF6MY+1Xo-BNX5;O9PRnzlqik~v({S%Bki{Q@nLqm7D|PUS}5+Vg+l%X zS|}9u*Fr&PE`%CNg`H|>Y_AQa{q<0Oi5^Z~poh~?wV~mu4*q6%Yd!p_^DdMDhNzJ} z2oOUSL1w#rKCP5ak9Ujfi&M@V@0Q*lCwm;!uMJ8kP-FgtX$j1|j|7edi(O6Z#SGV( zc;VX`+>Pe2{#{t+_c5f!3Io%1`lBdhQ8-|&Ws_FPZ`sO|xUM(9vfzGmgRS3aTYWO-?x^U^{q$N&fNvxtHC%)7Lz8G;#yAQs#c>&4imAm2w3*~D0oX! zOpr@~#{ary^*ZMK0+uQ|K^}nD)1YTG33`Ac`%-1>nWs#Ct!3{WKKJ*Lu1v*7v80d* zqCRg?mAk1OR$ufaG3tuZcF z=z}jQ0VkVDhWy-1q<_(3aUeWiP%0}0K)@w0Q@^Wy2Nq&KrOQlkodc7Fjldcx*l#g!BaS6k`j{lE>)p%VN3E^3-*-@F!qRy4sr%S-a)pyq89nSg z71YO*KkbQD`aF)0nN+fem#cRlD*;o|U*VK_R-U0SVHoCT&Ww+^P)@Bcy}I^~X1n)M zZ+CDn=_i&~l!xhvo09sqm%tESoKO@7P}J!?tb8KoC-n%suKR!kyO?-cYmE;tTH3&2 z%*{d`vmi^HKe~hkPfUUeUa%KM{+vWs;~v3t03*hNGQer5^W!v|WR#@C>3~yaq>qiy z{E*icZ3tS9a%lTFSmyn-KlFLdtv*^sn?7%}=&^i$w6PCfSOeUyrM1M;>uo7ur^#`h z_9}$Yl6VuWSXoLm-@}BVW{Eo0yVO6U0FnLA>iJ(=*Q#}09S;wD{=3EgBU>u??EkXm zOfesw|GkDga&KcQ%3D_BXH9Moc*g2R7Xn(=lum@M+BF$)eK()a&eG{`qxs$91}%Tn zwt15pr=2?WHL;WA*u(qN=+{wb-swl64v_%SVJce7U{&w&pPLFrQ9V4?bFi5N%}S1r zSk;_CKVwb?Bh%)Z#AVNa{`GH`Q%h<#kNM<-1otbjgnMo%Lh|a9;AY_ciTiu}eMIm2 zcwZ-kAiTaKZlLgEhSn0YC*Ihe^9Mt@@%=Be(d-v#{@tBA;ovP%HboX6A0JilO$iFq z!b94$IZ59`7i-Vc!XM;S(eRU4{*Bp-;gpy;jNST#!Z$4^#Jv&4@BQ5`#VbYAg*7mx zjFr17-nCpJl9|v8J-9xzOnReGJY~yA+@$!MQ_J#aUd<&{RgnN`v-|!4Rs8w$XZNi& zd{28*4D9flK6zC4zQ>J|`jA^fP8^1O zbKtfQ0@5;jz(KGOu57xP3{^!$>;YfIiF2^!krBV^g!}d4m3!ecb$)@c@+VI67jwiR zc0^PtNjCuhms|)X!AXE-?ES&7s3>m~4z3`#cZMFuZVm=h%Uory$V`Cdym`X=R7bGF zGa(_qPXlEu_N4GiI|36(kd)EncFKNYiCmHu#C0)&TVPt2gf`%hD)GY%gjt6=WV^7# zVk;=8790o!WMMvy=83FFlzZBDa}+ONS>*Q$`WunPk7@a{kqDQvH}){TNo@D*jjOl+ z&>x8>MknkC`=1lUv5=etXV0&HJwltpsm0PZ0d#@&_X8IR5e)f5?nOUh*Y3}B$?d%g}`RVVA_m84u|*+ zYbv=lEU6nHm<21oT7U9wfkpaEsi5d zz|&raqF#lf!e3B3_8~6rLtNU2xE!81zXz$B*@rm04{>fE;=N?vqeu+A>*?JGSxAo{BC zk@g-3OGtMzxC`t#e?Di4Ji_^$31qIpuaIv<0ww%fL{b&g;`p=TV&!b6%K;s~*~=d& z==lao2}9`&TXaL0j|zN%4fdr0P!V|I&XDA>pWv{Uf_3kHbTjA)GoJV$F2QKR^uvKH z+{4DRwA1|`5v z(8vB#b>(vtu2gHagfE>Vj_s#5j$ogp<_s6kofU&a4lE^>sJ(aigx`K8-jo8hMSFB>+2DL zOr~zJ=|OycLxt@5{B;X&_Fczl29V-pv9GS2Qy8H6ka``f5T;fP4Rd0|IVRDj=-A5U zcwJ9(F?A6Ygv9FE7MAdJeB-U!_*cPfC|*_wW~Ac7M3Z46U<#`0@>e46=O2H;tGgc2 z5erSBNnnECsfUVr9(PuZg1hLc&f~Oz6rS=CFACsCp|?&SDPaJ{{{FDPfX>M}kFT9T ztc3T>F-)<}^xwWIi5aFu)+7d zv1ZHJNabBF9j4;%;q$(YR>7 z=J{VKTlC%kFX5kP{r?KCZqEC|{=A?1D1a_mR?R96dh5tN$#(*&k|>?Ucnm|Mn8a^w z*~;Z~Y^G*}0KKy*DH=8Kv-KlL5t5>&f5v=GUlHwmWuc-}MT<;bwQEk|+gqekb^EO| zcGdXMXntxaEzxg?{3;cv!#HDmEY?=y1+UmvNOmmSR%pYm-BzM(S-`E(?Ow&LkZ+%% z??Ecob9=`q?j57FcZ~9$G4lJ;tC_uHWcQAd+dIbo`PjGoHm(5wFdtk7t}yu{YrzeS zf)<3kNZnOobfOWt$VD!4k&9g9A{V*HMJ{rYi(KR)7rDqqE^?8JT;w7bxyVH>a*>N% bM>38 From 517538a9a73ad07ee9b9cf4a6bc887c6976ebc57 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 15 Jul 2026 16:28:35 -0400 Subject: [PATCH 177/219] remove comments --- salt/libvirt/dev/saltDev.sls | 67 ++++++++++++++++++++++++++++++++++ salt/reactor/push_strelka.sls | 2 +- salt/reactor/push_suricata.sls | 2 +- 3 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 salt/libvirt/dev/saltDev.sls diff --git a/salt/libvirt/dev/saltDev.sls b/salt/libvirt/dev/saltDev.sls new file mode 100644 index 000000000..016407635 --- /dev/null +++ b/salt/libvirt/dev/saltDev.sls @@ -0,0 +1,67 @@ +# This state is designed to run on a development manager running in a libvirt VM. It will map the default pillar and salt directories +# from /opt/so/saltstack/default to your local development machine as the source path. +# The VM requires a filesystem to be added. Only the source path should be changed to your development codebase +# Driver: virtio-9p +# Source path: ~/project/securityonion +# Target path: saltDev + +# If you want a directory to be RW, then kvm must have group privileges. +# ll /home/user/projects/securityonion/salt/hypervisor +# total 48 +# drwxrwxr-x 3 user kvm 4096 Feb 13 11:18 ./ +# drwxrwxr-x 64 user user 4096 Feb 13 10:32 ../ +# -rw-rw-r-- 1 user kvm 2238 Feb 12 15:06 defaults.yaml +# -rw-rw-r-- 1 user kvm 1467 Feb 12 15:06 init.sls +# -rw-rw-r-- 1 user kvm 70 Feb 13 09:37 soc_hypervisor.yaml +# drwxrwxr-x 3 user kvm 4096 Feb 12 15:06 tools/ + +# Ensure required kernel modules are configured for loading +/etc/modules-load.d/virtio-9p.conf: + file.managed: + - contents: | + 9pnet_virtio + 9pnet + 9p + - mode: 644 + - user: root + - group: root + +# Load the kernel modules immediately (in the correct order) +load_9p_modules: + cmd.run: + - names: + - modprobe 9pnet_virtio + - modprobe 9pnet + - modprobe 9p + - unless: lsmod | grep -E '9pnet_virtio|9pnet|9p' + +# Ensure mount point exists +/opt/so/saltstack/default: + file.directory: + - user: root + - group: root + - mode: 755 + - makedirs: True + +# Configure fstab entry using mount.fstab_present +# Configure fstab entry using mount.fstab_present +saltdev_fstab: + mount.fstab_present: + - name: saltDev + - fs_file: /opt/so/saltstack/default + - fs_vfstype: 9p + - fs_mntops: _netdev,trans=virtio,version=9p2000.L + - fs_freq: 0 + - fs_passno: 0 + +# Mount the filesystem if not already mounted +mount_saltdev: + mount.mounted: + - name: /opt/so/saltstack/default + - device: saltDev + - fstype: 9p + - opts: _netdev,trans=virtio,version=9p2000.L + - require: + - file: /opt/so/saltstack/default + - mount: saltdev_fstab + - cmd: load_9p_modules diff --git a/salt/reactor/push_strelka.sls b/salt/reactor/push_strelka.sls index 52e3fd3ef..31f66e0a6 100644 --- a/salt/reactor/push_strelka.sls +++ b/salt/reactor/push_strelka.sls @@ -6,7 +6,7 @@ # Writes (or updates) a push intent at /opt/so/state/push_pending/rules_strelka.json # and returns {}. The so-push-drainer schedule picks up ready intents, dedupes # across pending files, and dispatches orch.push_batch. Reactors never dispatch -# directly -- see plan /home/mreeves/.claude/plans/goofy-marinating-hummingbird.md. +# directly import fcntl import json diff --git a/salt/reactor/push_suricata.sls b/salt/reactor/push_suricata.sls index cce95fdb7..11fbd1286 100644 --- a/salt/reactor/push_suricata.sls +++ b/salt/reactor/push_suricata.sls @@ -6,7 +6,7 @@ # Writes (or updates) a push intent at /opt/so/state/push_pending/rules_suricata.json # and returns {}. The so-push-drainer schedule picks up ready intents, dedupes # across pending files, and dispatches orch.push_batch. Reactors never dispatch -# directly -- see plan /home/mreeves/.claude/plans/goofy-marinating-hummingbird.md. +# directly import fcntl import json From f9b154ccef3ade676439f16824fc348aac7b1ace Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 15 Jul 2026 16:32:29 -0400 Subject: [PATCH 178/219] remove comments --- salt/manager/tools/sbin/so-push-drainer | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/so-push-drainer b/salt/manager/tools/sbin/so-push-drainer index fbbd952a4..8cc97a8d5 100644 --- a/salt/manager/tools/sbin/so-push-drainer +++ b/salt/manager/tools/sbin/so-push-drainer @@ -21,8 +21,7 @@ is older than debounce_seconds, this script: * deletes the contributed intent files on successful dispatch Reactor sls files (push_suricata, push_strelka, push_pillar) write intents -but never dispatch directly -- see plan -/home/mreeves/.claude/plans/goofy-marinating-hummingbird.md for the full design. +but never dispatch directly """ import fcntl From 5867b50720dc395f1fb354b781680451e83729eb Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 16 Jul 2026 12:28:23 -0500 Subject: [PATCH 179/219] include so-yaml.py in get_soup_script_hashes() so we ensure its at latest version before using it later on in soup --- salt/manager/tools/sbin/soup | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index ec2dcaed5..a96acdf43 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -437,6 +437,8 @@ get_soup_script_hashes() { GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}') CURRENTSOFIREWALL=$(md5sum /usr/sbin/so-firewall | awk '{print $1}') GITSOFIREWALL=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-firewall | awk '{print $1}') + CURRENTSOYAML=$(md5sum /usr/sbin/so-yaml.py | awk '{print $1}') + GITSOYAML=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py | awk '{print $1}') } highstate() { @@ -1224,7 +1226,7 @@ upgrade_salt() { verify_latest_update_script() { get_soup_script_hashes - if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then + if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" && "$CURRENTSOYAML" == "$GITSOYAML" ]]; then echo "This version of the soup script is up to date. Proceeding." else echo "You are not running the latest soup version. Updating soup and its components. This might take multiple runs to complete." @@ -1233,7 +1235,7 @@ verify_latest_update_script() { # Verify that soup scripts updated as expected get_soup_script_hashes - if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then + if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" && "$CURRENTSOYAML" == "$GITSOYAML" ]]; then echo "Succesfully updated soup scripts." else echo "There was a problem updating soup scripts. Trying to rerun script update." From cc2bfc26e20751470dca8e78f740f80383d94382 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 16 Jul 2026 15:13:51 -0400 Subject: [PATCH 180/219] Fix typos in CPU affinity descriptions --- salt/suricata/soc_suricata.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index ce6b7d008..c0b07a0cb 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -153,12 +153,12 @@ suricata: cpu-affinity: management-cpu-set: cpu: - description: Bind management threads to a core or range of cores. This can be a sigle core, list of cores, or list of range of cores. set-cpu-affinity must be set to true for this to be used. + description: Bind management threads to a core or range of cores. This can be a single core, list of cores, or list of range of cores. set-cpu-affinity must be set to true for this to be used. forcedType: "[]string" helpLink: suricata worker-cpu-set: cpu: - description: Bind worker threads to a core or range of cores. This can be a sigle core, list of cores, or list of range of cores. set-cpu-affinity must be set to true for this to be used. + description: Bind worker threads to a core or range of cores. This can be a single core, list of cores, or list of range of cores. set-cpu-affinity must be set to true for this to be used. forcedType: "[]string" helpLink: suricata vars: From 9e7e6edae070d622a533fca34eba5cdd48849844 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 16 Jul 2026 15:28:22 -0400 Subject: [PATCH 181/219] Add unit tests for _beacons and wire into CI Add 100%-coverage unit tests for the three custom salt beacons (postgres_pillar_beacon, rules_beacon, zeek) and add salt/_beacons to the python-test workflow's paths trigger and matrix. To pass the workflow's flake8 lint over the whole directory: - zeek.py: reindent to 4 spaces, drop trailing blank line, noqa the Salt-injected __salt__ references (F821); no logic change. - postgres_pillar_beacon.py: noqa C901 on beacon() (complexity 13 > 12). --- .github/workflows/pythontest.yml | 3 +- salt/_beacons/postgres_pillar_beacon.py | 2 +- salt/_beacons/postgres_pillar_beacon_test.py | 165 ++++++++++++++++++ salt/_beacons/rules_beacon_test.py | 172 +++++++++++++++++++ salt/_beacons/zeek.py | 37 ++-- salt/_beacons/zeek_test.py | 59 +++++++ 6 files changed, 417 insertions(+), 21 deletions(-) create mode 100644 salt/_beacons/postgres_pillar_beacon_test.py create mode 100644 salt/_beacons/rules_beacon_test.py create mode 100644 salt/_beacons/zeek_test.py diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index a4cc92d8d..dc95b01c3 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -5,6 +5,7 @@ on: paths: - "salt/sensoroni/files/analyzers/**" - "salt/manager/tools/sbin/**" + - "salt/_beacons/**" jobs: build: @@ -14,7 +15,7 @@ jobs: fail-fast: false matrix: python-version: ["3.14"] - python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin"] + python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin", "salt/_beacons"] steps: - uses: actions/checkout@v3 diff --git a/salt/_beacons/postgres_pillar_beacon.py b/salt/_beacons/postgres_pillar_beacon.py index d22eef0ea..074e9e8af 100644 --- a/salt/_beacons/postgres_pillar_beacon.py +++ b/salt/_beacons/postgres_pillar_beacon.py @@ -83,7 +83,7 @@ def _query(sql): return result.stdout -def beacon(config): +def beacon(config): # noqa: C901 retval = [] watermark = _read_watermark() diff --git a/salt/_beacons/postgres_pillar_beacon_test.py b/salt/_beacons/postgres_pillar_beacon_test.py new file mode 100644 index 000000000..377f0ff04 --- /dev/null +++ b/salt/_beacons/postgres_pillar_beacon_test.py @@ -0,0 +1,165 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +import os +import shutil +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +import postgres_pillar_beacon + + +class TestPostgresPillarBeacon(unittest.TestCase): + + def setUp(self): + # Point WATERMARK_FILE at a throwaway dir so the real read/write helpers + # (and their os.makedirs/os.rename) run against actual files, then clean + # it all up in tearDown. + self.tmpdir = tempfile.mkdtemp() + self.watermark = os.path.join(self.tmpdir, 'state', 'watch.id') + patcher = patch.object(postgres_pillar_beacon, 'WATERMARK_FILE', self.watermark) + patcher.start() + self.addCleanup(patcher.stop) + + def tearDown(self): + shutil.rmtree(self.tmpdir, ignore_errors=True) + + # -- trivial contract ------------------------------------------------- + + def test_virtual_returns_true(self): + self.assertTrue(postgres_pillar_beacon.__virtual__()) + + def test_validate_returns_valid(self): + self.assertEqual(postgres_pillar_beacon.validate({}), (True, 'valid')) + + # -- _read_watermark -------------------------------------------------- + + def test_read_watermark_valid(self): + postgres_pillar_beacon._write_watermark(42) + self.assertEqual(postgres_pillar_beacon._read_watermark(), 42) + + def test_read_watermark_missing_file_returns_none(self): + # tmp watermark file was never created + self.assertIsNone(postgres_pillar_beacon._read_watermark()) + + def test_read_watermark_garbage_returns_none(self): + os.makedirs(os.path.dirname(self.watermark), exist_ok=True) + with open(self.watermark, 'w') as f: + f.write('nope') + self.assertIsNone(postgres_pillar_beacon._read_watermark()) + + # -- _write_watermark ------------------------------------------------- + + def test_write_watermark_round_trip(self): + postgres_pillar_beacon._write_watermark(7) + with open(self.watermark) as f: + self.assertEqual(f.read(), '7') + + def test_write_watermark_swallows_oserror(self): + with patch.object(postgres_pillar_beacon.os, 'makedirs', side_effect=OSError): + # Must not raise; failure is logged and the beacon retries next pass. + postgres_pillar_beacon._write_watermark(5) + self.assertFalse(os.path.exists(self.watermark)) + + # -- _query ----------------------------------------------------------- + + def test_query_success_returns_stdout_and_builds_argv(self): + completed = subprocess.CompletedProcess(args=[], returncode=0, stdout='rows', stderr='') + with patch.object(postgres_pillar_beacon.subprocess, 'run', return_value=completed) as mock_run: + result = postgres_pillar_beacon._query('SELECT 1;') + self.assertEqual(result, 'rows') + argv = mock_run.call_args[0][0] + self.assertEqual(argv[:5], ['docker', 'exec', 'so-postgres', 'psql', '-U']) + self.assertIn('SELECT 1;', argv) + self.assertFalse(mock_run.call_args[1].get('shell', False)) + + def test_query_timeout_returns_none(self): + with patch.object(postgres_pillar_beacon.subprocess, 'run', + side_effect=subprocess.TimeoutExpired(cmd='psql', timeout=30)): + self.assertIsNone(postgres_pillar_beacon._query('SELECT 1;')) + + def test_query_generic_exception_returns_none(self): + with patch.object(postgres_pillar_beacon.subprocess, 'run', side_effect=Exception('boom')): + self.assertIsNone(postgres_pillar_beacon._query('SELECT 1;')) + + def test_query_nonzero_returncode_returns_none(self): + completed = subprocess.CompletedProcess(args=[], returncode=1, stdout='', stderr='bad') + with patch.object(postgres_pillar_beacon.subprocess, 'run', return_value=completed): + self.assertIsNone(postgres_pillar_beacon._query('SELECT 1;')) + + # -- beacon: first run / seeding -------------------------------------- + + def test_beacon_seeds_when_postgres_not_ready(self): + with patch.object(postgres_pillar_beacon, '_read_watermark', return_value=None), \ + patch.object(postgres_pillar_beacon, '_query', return_value=None), \ + patch.object(postgres_pillar_beacon, '_write_watermark') as mock_write: + self.assertEqual(postgres_pillar_beacon.beacon({}), []) + mock_write.assert_not_called() + + def test_beacon_seeds_to_max_id_and_emits_nothing(self): + with patch.object(postgres_pillar_beacon, '_read_watermark', return_value=None), \ + patch.object(postgres_pillar_beacon, '_query', return_value='7\n'), \ + patch.object(postgres_pillar_beacon, '_write_watermark') as mock_write: + self.assertEqual(postgres_pillar_beacon.beacon({}), []) + mock_write.assert_called_once_with(7) + + def test_beacon_seed_unparseable_is_swallowed(self): + with patch.object(postgres_pillar_beacon, '_read_watermark', return_value=None), \ + patch.object(postgres_pillar_beacon, '_query', return_value='abc'), \ + patch.object(postgres_pillar_beacon, '_write_watermark') as mock_write: + self.assertEqual(postgres_pillar_beacon.beacon({}), []) + mock_write.assert_not_called() + + # -- beacon: steady state --------------------------------------------- + + def test_beacon_query_failure_returns_empty(self): + with patch.object(postgres_pillar_beacon, '_read_watermark', return_value=10), \ + patch.object(postgres_pillar_beacon, '_query', return_value=None), \ + patch.object(postgres_pillar_beacon, '_write_watermark') as mock_write: + self.assertEqual(postgres_pillar_beacon.beacon({}), []) + mock_write.assert_not_called() + + def test_beacon_emits_events_and_advances_watermark(self): + sep = postgres_pillar_beacon.FIELD_SEP + rows = '11%s5%snode1\n12%s6%s\n' % (sep, sep, sep, sep) + with patch.object(postgres_pillar_beacon, '_read_watermark', return_value=10), \ + patch.object(postgres_pillar_beacon, '_query', return_value=rows), \ + patch.object(postgres_pillar_beacon, '_write_watermark') as mock_write: + result = postgres_pillar_beacon.beacon({}) + self.assertEqual(result, [ + {'tag': 'audit_settings', 'id': 11, 'setting_id': '5', 'node_id': 'node1'}, + {'tag': 'audit_settings', 'id': 12, 'setting_id': '6', 'node_id': ''}, + ]) + mock_write.assert_called_once_with(12) + + def test_beacon_skips_malformed_blank_and_noninteger_rows(self): + sep = postgres_pillar_beacon.FIELD_SEP + rows = ( + '\n' # blank line -> skipped + '13%s7\n' # too few fields -> skipped + 'abc%s8%snodeX\n' # non-integer id -> skipped + '14%s9%snodeY\n' # the one good row + ) % (sep, sep, sep, sep, sep) + with patch.object(postgres_pillar_beacon, '_read_watermark', return_value=10), \ + patch.object(postgres_pillar_beacon, '_query', return_value=rows), \ + patch.object(postgres_pillar_beacon, '_write_watermark') as mock_write: + result = postgres_pillar_beacon.beacon({}) + self.assertEqual(result, [ + {'tag': 'audit_settings', 'id': 14, 'setting_id': '9', 'node_id': 'nodeY'}, + ]) + mock_write.assert_called_once_with(14) + + def test_beacon_no_new_rows_does_not_advance_watermark(self): + with patch.object(postgres_pillar_beacon, '_read_watermark', return_value=10), \ + patch.object(postgres_pillar_beacon, '_query', return_value=''), \ + patch.object(postgres_pillar_beacon, '_write_watermark') as mock_write: + self.assertEqual(postgres_pillar_beacon.beacon({}), []) + mock_write.assert_not_called() + + +if __name__ == '__main__': + unittest.main() diff --git a/salt/_beacons/rules_beacon_test.py b/salt/_beacons/rules_beacon_test.py new file mode 100644 index 000000000..adcd19181 --- /dev/null +++ b/salt/_beacons/rules_beacon_test.py @@ -0,0 +1,172 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +import hashlib +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +import rules_beacon + + +class TestRulesBeacon(unittest.TestCase): + + def setUp(self): + # Isolate all on-disk state (watermarks and the dirs we fingerprint) in a + # throwaway tree, and point WATERMARK_DIR at it so the real read/write + # helpers run against actual files. + self.tmpdir = tempfile.mkdtemp() + self.state = os.path.join(self.tmpdir, 'state') + patcher = patch.object(rules_beacon, 'WATERMARK_DIR', self.state) + patcher.start() + self.addCleanup(patcher.stop) + + def tearDown(self): + shutil.rmtree(self.tmpdir, ignore_errors=True) + + def _make_dir(self, name, files=None): + path = os.path.join(self.tmpdir, name) + os.makedirs(path, exist_ok=True) + for fname, content in (files or {}).items(): + with open(os.path.join(path, fname), 'w') as f: + f.write(content) + return path + + # -- trivial contract ------------------------------------------------- + + def test_virtual_returns_true(self): + self.assertTrue(rules_beacon.__virtual__()) + + def test_validate_returns_valid(self): + self.assertEqual(rules_beacon.validate({}), (True, 'valid')) + + # -- _paths_from_config ----------------------------------------------- + + def test_paths_from_config_list_of_dicts(self): + config = [{'interval': 10}, {'paths': {'/a': 'suricata', '/b': 'strelka'}}] + self.assertEqual( + rules_beacon._paths_from_config(config), + {'/a': 'suricata', '/b': 'strelka'}, + ) + + def test_paths_from_config_plain_dict(self): + self.assertEqual( + rules_beacon._paths_from_config({'paths': {'/a': 'suricata'}}), + {'/a': 'suricata'}, + ) + + def test_paths_from_config_skips_non_dict_items(self): + self.assertEqual(rules_beacon._paths_from_config(['bogus', 42]), {}) + + def test_paths_from_config_paths_not_a_dict(self): + self.assertEqual(rules_beacon._paths_from_config({'paths': 'nope'}), {}) + + def test_paths_from_config_unexpected_type(self): + self.assertEqual(rules_beacon._paths_from_config('nonsense'), {}) + + # -- _excluded -------------------------------------------------------- + + def test_excluded_matches_temp_and_editor_files(self): + for pathname in ('/rules/foo.swp', '/rules/foo~', '/rules/4913', '/rules/.#foo'): + self.assertTrue(rules_beacon._excluded(pathname), pathname) + + def test_excluded_allows_real_rule_files(self): + self.assertFalse(rules_beacon._excluded('/rules/suricata.rules')) + + # -- _fingerprint ----------------------------------------------------- + + def test_fingerprint_missing_dir_is_empty_tree_digest(self): + missing = os.path.join(self.tmpdir, 'does-not-exist') + self.assertEqual(rules_beacon._fingerprint(missing), hashlib.sha1().hexdigest()) + + def test_fingerprint_changes_when_content_changes(self): + d = self._make_dir('rules', {'a.rules': 'alert'}) + before = rules_beacon._fingerprint(d) + with open(os.path.join(d, 'a.rules'), 'w') as f: + f.write('alert tcp any any -> any any') # different size + self.assertNotEqual(rules_beacon._fingerprint(d), before) + + def test_fingerprint_ignores_excluded_files(self): + d = self._make_dir('rules', {'a.rules': 'alert'}) + before = rules_beacon._fingerprint(d) + with open(os.path.join(d, 'a.rules.swp'), 'w') as f: + f.write('editor swap') + self.assertEqual(rules_beacon._fingerprint(d), before) + + def test_fingerprint_skips_unstatable_entries(self): + # A dangling symlink appears in os.walk's file list but os.stat raises + # OSError, exercising the except-continue path. + d = self._make_dir('rules', {'a.rules': 'alert'}) + good = rules_beacon._fingerprint(d) + os.symlink(os.path.join(d, 'missing-target'), os.path.join(d, 'broken.link')) + self.assertEqual(rules_beacon._fingerprint(d), good) + + # -- _read_watermark / _write_watermark ------------------------------- + + def test_watermark_round_trip(self): + rules_beacon._write_watermark('suricata', 'deadbeef') + self.assertEqual(rules_beacon._read_watermark('suricata'), 'deadbeef') + + def test_read_watermark_missing_returns_none(self): + self.assertIsNone(rules_beacon._read_watermark('suricata')) + + def test_read_watermark_empty_file_returns_none(self): + os.makedirs(self.state, exist_ok=True) + with open(rules_beacon._watermark_file('suricata'), 'w') as f: + f.write('') + self.assertIsNone(rules_beacon._read_watermark('suricata')) + + def test_write_watermark_swallows_oserror(self): + with patch.object(rules_beacon.os, 'makedirs', side_effect=OSError): + rules_beacon._write_watermark('suricata', 'deadbeef') + self.assertIsNone(rules_beacon._read_watermark('suricata')) + + # -- beacon ----------------------------------------------------------- + + def _config(self, mapping): + return [{'paths': mapping}] + + def test_beacon_seeds_first_run_and_emits_nothing(self): + with patch.object(rules_beacon, '_fingerprint', return_value='hash1'), \ + patch.object(rules_beacon, '_read_watermark', return_value=None), \ + patch.object(rules_beacon, '_write_watermark') as mock_write: + result = rules_beacon.beacon(self._config({'/rules/suricata': 'suricata'})) + self.assertEqual(result, []) + mock_write.assert_called_once_with('suricata', 'hash1') + + def test_beacon_emits_on_change(self): + with patch.object(rules_beacon, '_fingerprint', return_value='newhash'), \ + patch.object(rules_beacon, '_read_watermark', return_value='oldhash'), \ + patch.object(rules_beacon, '_write_watermark') as mock_write: + result = rules_beacon.beacon(self._config({'/rules/suricata': 'suricata'})) + self.assertEqual(result, [{'tag': 'suricata', 'path': '/rules/suricata'}]) + mock_write.assert_called_once_with('suricata', 'newhash') + + def test_beacon_no_change_emits_nothing(self): + with patch.object(rules_beacon, '_fingerprint', return_value='samehash'), \ + patch.object(rules_beacon, '_read_watermark', return_value='samehash'), \ + patch.object(rules_beacon, '_write_watermark') as mock_write: + result = rules_beacon.beacon(self._config({'/rules/suricata': 'suricata'})) + self.assertEqual(result, []) + mock_write.assert_not_called() + + def test_beacon_end_to_end_with_real_files(self): + # Exercise the full stack (real fingerprint + real watermark files) across + # two poll passes: first seeds silently, second fires after a write. + d = self._make_dir('rules', {'a.rules': 'alert'}) + config = self._config({d: 'suricata'}) + + self.assertEqual(rules_beacon.beacon(config), []) # seed pass + self.assertEqual(rules_beacon.beacon(config), []) # unchanged pass + + with open(os.path.join(d, 'b.rules'), 'w') as f: + f.write('alert tcp any any -> any any') + self.assertEqual(rules_beacon.beacon(config), [{'tag': 'suricata', 'path': d}]) + + +if __name__ == '__main__': + unittest.main() diff --git a/salt/_beacons/zeek.py b/salt/_beacons/zeek.py index 117c2b401..6c4d01b11 100644 --- a/salt/_beacons/zeek.py +++ b/salt/_beacons/zeek.py @@ -3,31 +3,30 @@ import logging def status(): - cmd = "runuser -l zeek -c '/opt/zeek/bin/zeekctl status'" - retval = __salt__['docker.run']('so-zeek', cmd) - logging.info('zeekctl_module: zeekctl.status retval: %s' % retval) + cmd = "runuser -l zeek -c '/opt/zeek/bin/zeekctl status'" + retval = __salt__['docker.run']('so-zeek', cmd) # noqa: F821 + logging.info('zeekctl_module: zeekctl.status retval: %s' % retval) - return retval + return retval def beacon(config): - retval = [] + retval = [] - is_enabled = __salt__['healthcheck.is_enabled']() - logging.info('zeek_beacon: healthcheck_is_enabled: %s' % is_enabled) + is_enabled = __salt__['healthcheck.is_enabled']() # noqa: F821 + logging.info('zeek_beacon: healthcheck_is_enabled: %s' % is_enabled) - if is_enabled: - zeekstatus = status().lower().split(' ') - logging.info('zeek_beacon: zeekctl.status: %s' % str(zeekstatus)) - if 'stopped' in zeekstatus or 'crashed' in zeekstatus or 'error' in zeekstatus or 'error:' in zeekstatus: - zeek_restart = True - else: - zeek_restart = False + if is_enabled: + zeekstatus = status().lower().split(' ') + logging.info('zeek_beacon: zeekctl.status: %s' % str(zeekstatus)) + if 'stopped' in zeekstatus or 'crashed' in zeekstatus or 'error' in zeekstatus or 'error:' in zeekstatus: + zeek_restart = True + else: + zeek_restart = False - __salt__['telegraf.send']('healthcheck zeek_restart=%s' % str(zeek_restart)) - retval.append({'zeek_restart': zeek_restart}) - logging.info('zeek_beacon: retval: %s' % str(retval)) - - return retval + __salt__['telegraf.send']('healthcheck zeek_restart=%s' % str(zeek_restart)) # noqa: F821 + retval.append({'zeek_restart': zeek_restart}) + logging.info('zeek_beacon: retval: %s' % str(retval)) + return retval diff --git a/salt/_beacons/zeek_test.py b/salt/_beacons/zeek_test.py new file mode 100644 index 000000000..db3b4236f --- /dev/null +++ b/salt/_beacons/zeek_test.py @@ -0,0 +1,59 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +import unittest +from unittest.mock import MagicMock + +import zeek + +ZEEKCTL_CMD = "runuser -l zeek -c '/opt/zeek/bin/zeekctl status'" + + +class TestZeekBeacon(unittest.TestCase): + + def setUp(self): + # zeek.py relies on the __salt__ dunder that Salt injects at load time. + # Nothing defines it under test, so we attach a dict of mock loader + # functions to the module and remove it again afterwards. + self.salt = { + 'docker.run': MagicMock(return_value='Zeek is running'), + 'healthcheck.is_enabled': MagicMock(return_value=True), + 'telegraf.send': MagicMock(), + } + zeek.__salt__ = self.salt + self.addCleanup(lambda: delattr(zeek, '__salt__')) + + # -- status ----------------------------------------------------------- + + def test_status_runs_zeekctl_and_returns_output(self): + self.salt['docker.run'].return_value = 'Zeek is running' + result = zeek.status() + self.assertEqual(result, 'Zeek is running') + self.salt['docker.run'].assert_called_once_with('so-zeek', ZEEKCTL_CMD) + + # -- beacon ----------------------------------------------------------- + + def test_beacon_disabled_returns_empty_and_skips_telegraf(self): + self.salt['healthcheck.is_enabled'].return_value = False + self.assertEqual(zeek.beacon({}), []) + self.salt['telegraf.send'].assert_not_called() + + def test_beacon_running_reports_no_restart(self): + self.salt['docker.run'].return_value = 'Zeek is running' + self.assertEqual(zeek.beacon({}), [{'zeek_restart': False}]) + self.salt['telegraf.send'].assert_called_once_with('healthcheck zeek_restart=False') + + def test_beacon_unhealthy_status_triggers_restart(self): + # Each of these status tokens should flag a restart (the or-chain in beacon). + for status_text in ('Zeek is stopped', 'Zeek crashed', 'Zeek error state', 'Zeek error:'): + with self.subTest(status=status_text): + self.salt['docker.run'].return_value = status_text + self.salt['telegraf.send'].reset_mock() + self.assertEqual(zeek.beacon({}), [{'zeek_restart': True}]) + self.salt['telegraf.send'].assert_called_once_with('healthcheck zeek_restart=True') + + +if __name__ == '__main__': + unittest.main() From 141116f5500b21d900a7db3ffcb30249e9adccd1 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 16 Jul 2026 16:52:06 -0400 Subject: [PATCH 182/219] Make so-salt-minion-wait work without requiring a restart The wait required both a socket gate and a log gate to pass. The log gate greps the minion log for salt's one-time startup line "Minion is ready to receive requests!", which scrolls out of the log tail on a minion that has not restarted recently. On such a minion the log gate could never pass, so the script ran to its full 120s timeout and exited 1 even though the minion was healthy and connected. This also false-timed-out when salt_minion_service reported a non-restart change (e.g. an enable toggle). The log gate's only remaining purpose is closing the ~2.8s post-connect window where the master sockets are up but _post_master_init() is still loading. Gate it on the current pid's uptime: enforce the ready line only within READY_LINE_WINDOW (90s) of (re)start, and let the already restart-independent socket gate be the steady-state authority past that. The fresh-restart path is unchanged, and if uptime can't be read the strict behavior is kept. --- salt/salt/minion/init.sls | 17 ++++--- salt/salt/tools/sbin/so-salt-minion-wait | 63 ++++++++++++++++++------ 2 files changed, 58 insertions(+), 22 deletions(-) diff --git a/salt/salt/minion/init.sls b/salt/salt/minion/init.sls index 1f1ec8305..fa94ec7be 100644 --- a/salt/salt/minion/init.sls +++ b/salt/salt/minion/init.sls @@ -131,13 +131,16 @@ salt_minion_service: {% endif %} - order: last -# block until the just-restarted salt-minion daemon logs "Minion is ready to receive requests!" -# for the current instance, so follow-on jobs and the next highstate iteration do not race the -# restart. onchanges + require on salt_minion_service catches every restart trigger uniformly -# because watch mod_watch results replace the service state's running entry. wait logic lives in -# /usr/sbin/so-salt-minion-wait (deployed by salt_sbin from salt/tools/sbin/); it keys the ready -# line to the current daemon pid (resolved via systemd, not the pidfile) and corroborates with the -# master req/publish sockets. set_log_levels above enforces the log_level_logfile: info that the +# block until the salt-minion daemon is ready for the current instance, so follow-on jobs and the +# next highstate iteration do not race the restart. onchanges + require on salt_minion_service +# catches every restart trigger uniformly because watch mod_watch results replace the service +# state's running entry. wait logic lives in /usr/sbin/so-salt-minion-wait (deployed by salt_sbin +# from salt/tools/sbin/); its steady-state authority is the master req/publish sockets for the +# current daemon pid (resolved via systemd, not the pidfile), and it corroborates a just-restarted +# instance with the pid-tagged "Minion is ready to receive requests!" log line only within a short +# window of startup. Because that socket signal does not require a recent restart, the wait also +# succeeds cleanly when salt_minion_service reports a non-restart change (e.g. an enable toggle) +# rather than false-timing-out. set_log_levels above enforces the log_level_logfile: info that the # ready line depends on. salt restarts this unit with --no-block, so mod_watch returns while the old # daemon is still up; the script waits for systemd's restart job to drain before it reads MainPID. wait_for_salt_minion_ready: diff --git a/salt/salt/tools/sbin/so-salt-minion-wait b/salt/salt/tools/sbin/so-salt-minion-wait index 984f144f3..cf16ab0d1 100644 --- a/salt/salt/tools/sbin/so-salt-minion-wait +++ b/salt/salt/tools/sbin/so-salt-minion-wait @@ -5,21 +5,29 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -# Block until the just-restarted salt-minion daemon reaches the point where salt itself logs -# "Minion is ready to receive requests!". Invoked from the wait_for_salt_minion_ready state in -# salt/minion/init.sls after salt_minion_service fires its watch-driven restart, so follow-on jobs -# and the next highstate iteration do not race it. +# Block until the current salt-minion daemon is ready to receive requests. Invoked from the +# wait_for_salt_minion_ready state in salt/minion/init.sls after salt_minion_service fires its +# watch-driven restart, so follow-on jobs and the next highstate iteration do not race it. It is +# also correct on an already-running minion (no recent restart): the steady-state readiness signal +# is the live master sockets, so it does not depend on a restart having just happened. # -# Salt logs that line from Minion.tune_in() only after sync_connect_master() returns, which means -# the pub channel authenticated, the long-running req channel connected, and _post_master_init() -# finished loading modules and compiling pillar. Two signals reproduce that: +# Salt logs "Minion is ready to receive requests!" from Minion.tune_in() only after +# sync_connect_master() returns, which means the pub channel authenticated, the long-running req +# channel connected, and _post_master_init() finished loading modules and compiling pillar. Two +# signals reproduce that: # -# 1. Primary the pid-tagged ready line in the minion log. Salt's log_fmt_logfile embeds +# 1. Steady state the pid holds an ESTABLISHED req connection to a master on 4506 plus a second +# (publish) connection to that same master IP on another port. The publish port is +# learned from the master's auth reply and is absent from minion config, so it is +# derived from the connection rather than read from config. This is the always-on +# authority: it reflects whatever daemon is running now, restart or not. +# 2. Startup only the pid-tagged ready line in the minion log. Salt's log_fmt_logfile embeds # [%(process)d] just before the message, so this is keyed to one daemon instance. -# 2. Corroborating that same pid holds an ESTABLISHED req connection to a master on 4506 plus a -# second (publish) connection to that same master IP on another port. The publish -# port is learned from the master's auth reply and is absent from minion config, -# so it is derived from the connection rather than read from config. +# Salt logs it exactly once per start, so it exists only to close a ~2.8s window +# after the sockets come up where they are established but _post_master_init() is +# still finishing. It is therefore required only within READY_LINE_WINDOW seconds +# of (re)start (by pid uptime); past that the line has scrolled out of the log and +# the socket gate alone decides. See instance_ready(). # # The daemon pid is resolved from systemd, never from /var/run/salt-minion.pid. salt_minion() runs # the real minion in a multiprocessing child; that child writes the pidfile, owns the sockets and @@ -43,6 +51,12 @@ INITIAL_SLEEP=3 TIMEOUT=120 MASTER_PORT=4506 LOG_TAIL_LINES=10000 +# Seconds after a (re)start during which the pid-tagged ready line is still required. Past this the +# daemon is clearly beyond the ~2.8s post-connect race and the socket gate is authoritative -- the +# one-time ready line has scrolled out of the log tail on a long-running minion. Kept under TIMEOUT +# so a fresh minion that connects but never logs the line still falls back to socket-only near the +# end instead of false-timing-out. +READY_LINE_WINDOW=90 DEFAULT_LOG_FILE="/opt/so/log/salt/minion" LOG_FILE="$DEFAULT_LOG_FILE" @@ -103,6 +117,15 @@ resolve_daemon_pids() { printf '%s\n' "${children:-$mainpid}" } +# Elapsed seconds since this pid started (Linux procps etimes). Empty/non-numeric -> failure, so the +# caller can fall back to the strict (log-gate-enforced) behavior when uptime cannot be read. +pid_uptime() { + local pid=$1 secs + secs=$(ps -o etimes= -p "$pid" 2>/dev/null | tr -d ' ') + case "$secs" in ''|*[!0-9]*) return 1 ;; esac + printf '%s\n' "$secs" +} + # True iff the ready line tagged with this pid is in the current or most recently rotated log. ready_logged() { local pid=$1 f @@ -135,9 +158,19 @@ socket_ready() { } instance_ready() { - local pid=$1 - if [ "$USE_LOG_GATE" -eq 1 ] && ! ready_logged "$pid"; then - return 1 + local pid=$1 uptime + # The log gate only closes the ~2.8s window right after the master sockets come up where they are + # established but _post_master_init() is still loading modules/compiling pillar. Salt logs the + # pid-tagged ready line exactly once at startup, so on a daemon that started long ago the line has + # scrolled out of the log tail and the gate could never pass -- making the wait require a recent + # restart. Enforce it only while the daemon is young enough that the race could still be open; past + # READY_LINE_WINDOW the socket gate is authoritative. If uptime can't be read, keep the strict + # behavior (uptime=0 -> gate enforced) so the fresh-restart path never regresses. + if [ "$USE_LOG_GATE" -eq 1 ]; then + uptime=$(pid_uptime "$pid") || uptime=0 + if [ "$uptime" -lt "$READY_LINE_WINDOW" ] && ! ready_logged "$pid"; then + return 1 + fi fi if [ "$USE_SOCKET_GATE" -eq 1 ] && ! socket_ready "$pid"; then return 1 From 8095b828411d230bf964b336ca352c7d892ec8a9 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 16 Jul 2026 16:58:21 -0400 Subject: [PATCH 183/219] soup use so-salt-minion-wait to ensure salt-minion is ready --- salt/common/tools/sbin/so-common | 36 -------------------------------- salt/manager/tools/sbin/soup | 13 ++++-------- 2 files changed, 4 insertions(+), 45 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 4e6580ae1..b5d1ab286 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -602,42 +602,6 @@ run_check_net_err() { fi } -wait_for_salt_minion() { - local minion="$1" - local max_wait="${2:-30}" - local interval="${3:-2}" - local logfile="${4:-'/dev/stdout'}" - local elapsed=0 - - echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Waiting for salt-minion '$minion' to be ready..." - - while [ $elapsed -lt $max_wait ]; do - # Check if service is running - echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Check if salt-minion service is running" - if ! systemctl is-active --quiet salt-minion; then - echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion service not running (elapsed: ${elapsed}s)" - sleep $interval - elapsed=$((elapsed + interval)) - continue - fi - echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion service is running" - - # Check if minion responds to ping - echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Check if $minion responds to ping" - if salt "$minion" test.ping --timeout=3 --out=json 2>> "$logfile" | grep -q "true"; then - echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion '$minion' is connected and ready!" - return 0 - fi - - echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Waiting... (${elapsed}s / ${max_wait}s)" - sleep $interval - elapsed=$((elapsed + interval)) - done - - echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - ERROR: salt-minion '$minion' not ready after $max_wait seconds" - return 1 -} - salt_minion_count() { local MINIONDIR="/opt/so/saltstack/local/pillar/minions" MINIONCOUNT=$(ls -la $MINIONDIR/*.sls | grep -v adv_ | wc -l) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 52f310e81..751621647 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1563,18 +1563,13 @@ verify_es_version_compatibility() { } wait_for_salt_minion_with_restart() { - local minion="$1" - local max_wait="${2:-60}" - local interval="${3:-3}" - local logfile="$4" - - wait_for_salt_minion "$minion" "$max_wait" "$interval" "$logfile" + /usr/sbin/so-salt-minion-wait local result=$? if [[ $result -ne 0 ]]; then echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion not ready, attempting restart..." systemctl_func "restart" "salt-minion" - wait_for_salt_minion "$minion" "$max_wait" "$interval" "$logfile" + /usr/sbin/so-salt-minion-wait result=$? fi @@ -2030,7 +2025,7 @@ main() { echo "" echo "Running a highstate. This could take several minutes." set +e - wait_for_salt_minion_with_restart "$MINIONID" "60" "3" "$SOUP_LOG" || fail "Salt minion was not running or ready." + wait_for_salt_minion_with_restart || fail "Salt minion was not running or ready." highstate set -e @@ -2043,7 +2038,7 @@ main() { check_saltmaster_status echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes." - wait_for_salt_minion_with_restart "$MINIONID" "60" "3" "$SOUP_LOG" || fail "Salt minion was not running or ready." + wait_for_salt_minion_with_restart || fail "Salt minion was not running or ready." # Stop long-running scripts to allow potentially updated scripts to load on the next execution. if pgrep salt-relay.sh > /dev/null 2>&1; then From aaea6dbd58fbe251e0de189ceb2203f838584533 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 17 Jul 2026 09:18:03 -0400 Subject: [PATCH 184/219] so-salt-minion-wait: report ready immediately when already ready Running the wait on a healthy, steady-state minion always reported readiness after a 3s floor: salt-minion (pid 4114640) ready after 3s That floor came entirely from the unconditional sleep "$INITIAL_SLEEP" (3s) before the poll loop. The sleep is vestigial: it predates restart_pending and never even covered the restart race (see 89e6a746c -- "INITIAL_SLEEP=3 expired inside that window"). Salt restarts the unit with --no-block and the restart job is enqueued before service.restart returns, so an in-flight restart is visible to restart_pending on the first loop iteration; the sleep protects nothing now. Drop the INITIAL_SLEEP constant and the pre-loop sleep and start elapsed at 0. The loop already sleeps at the bottom of each iteration, so the first readiness check now runs immediately: an already-ready minion returns "ready after 0s", while the restart path (guarded by restart_pending) and the mid-startup log gate are unchanged. --- salt/salt/tools/sbin/so-salt-minion-wait | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/salt/salt/tools/sbin/so-salt-minion-wait b/salt/salt/tools/sbin/so-salt-minion-wait index cf16ab0d1..d54e39677 100644 --- a/salt/salt/tools/sbin/so-salt-minion-wait +++ b/salt/salt/tools/sbin/so-salt-minion-wait @@ -47,7 +47,6 @@ set -u -INITIAL_SLEEP=3 TIMEOUT=120 MASTER_PORT=4506 LOG_TAIL_LINES=10000 @@ -178,9 +177,7 @@ instance_ready() { return 0 } -sleep "$INITIAL_SLEEP" - -elapsed="$INITIAL_SLEEP" +elapsed=0 pids="" announced_pending=0 while [ "$elapsed" -lt "$TIMEOUT" ]; do From 811b799b0b24bc59aa21b4aa6505bf74f6bc227a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 17 Jul 2026 10:57:30 -0400 Subject: [PATCH 185/219] ignore file already closed --- salt/common/tools/sbin/so-log-check | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 65b1041fe..42ceea47b 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -132,6 +132,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|HTTP 404: Not Found" # Salt loops until Kratos returns 200, during startup Kratos may not be ready EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Cancelling deferred write event maybeFenceReplicas because the event queue is now closed" # Kafka controller log during shutdown/restart EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Redis may have been restarted" # Redis likely restarted by salt + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|file already closed" # Go logging race condition during container restart fi if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then From 48a7d66964e910b6793a36fc27621147d07d80a7 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 17 Jul 2026 15:20:20 -0400 Subject: [PATCH 186/219] Update baseline agents --- salt/soc/defaults.yaml | 3 ++- salt/soc/soc_soc.yaml | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 8df30736f..9705c1863 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1531,7 +1531,8 @@ soc: agentic: false agentMapping: Orchestrator: sonnet - Hunter: sonnet + Investigator: sonnet + Detection Engineer: sonnet onionconfig: saltstackDir: /opt/so/saltstack bypassEnabled: false diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 47c7ea3ee..f09317f39 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -783,8 +783,11 @@ soc: Orchestrator: description: The initial agent in most agentic conversations. This agent will delegate requests to specialized agents. global: True - Hunter: - description: This agent is specialized in querying events. + Investigator: + description: This agent investigates alerts, explains events and records, and hunts through event data. It can also acknowledge alerts and escalate to cases. + global: True + Detection Engineer: + description: This agent manages detections and their overrides, including tuning noisy rules and authoring rule content. global: True client: assistant: From 4886034fefb603f584c90a2701144eea96cdf179 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 17 Jul 2026 15:25:55 -0400 Subject: [PATCH 187/219] add date and time to output --- salt/common/tools/sbin/so-common | 2 +- salt/manager/tools/sbin/soup | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index b5d1ab286..402d84363 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -666,7 +666,7 @@ systemctl_func() { echo "" echo "${echo_action^}ing $service_name service at $(date +"%T.%6N")" - systemctl $action $service_name && echo "Successfully ${echo_action}ed $service_name." || echo "Failed to $action $service_name." + systemctl $action $service_name && echo "Successfully ${echo_action}ed $service_name at $(date +"%T.%6N")." || echo "Failed to $action $service_name at $(date +"%T.%6N")." echo "" } diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 751621647..f5b40310e 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -2023,7 +2023,7 @@ main() { enable_highstate echo "" - echo "Running a highstate. This could take several minutes." + echo "Running a highstate at $(date +"%T.%6N"). This could take several minutes." set +e wait_for_salt_minion_with_restart || fail "Salt minion was not running or ready." highstate @@ -2037,7 +2037,7 @@ main() { check_saltmaster_status - echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes." + echo "Running a highstate at $(date +"%T.%6N") to complete the Security Onion upgrade on this manager. This could take several minutes." wait_for_salt_minion_with_restart || fail "Salt minion was not running or ready." # Stop long-running scripts to allow potentially updated scripts to load on the next execution. From ad78e84ccd7c114fa1719d15de267e31295570ac Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 20 Jul 2026 12:41:14 -0400 Subject: [PATCH 188/219] Prevent PostgreSQL from leaking plaintext passwords to postgres.log PostgreSQL's log_min_error_statement defaults to 'error', so whenever a CREATE/ALTER ROLE ... PASSWORD statement errored, the full statement text -- including the plaintext password -- was written to /opt/so/log/postgres/postgres.log. The role-provisioning paths (init-db.sh for so_postgres, so-telegraf-postgres for per-minion telegraf roles) both dispatch such DDL, the latter on every state.apply. - init-db.sh / so-telegraf-postgres: SET log_min_error_statement = panic before the password-bearing DDL so an error no longer emits the STATEMENT line. The ERROR message itself (no password) still logs, preserving debuggability. - logrotate: add a postgres stanza (daily, keep 14, copytruncate, compress) so postgres.log is rotated like every other service and leaked content can't persist indefinitely. copytruncate is required because the container holds the log open via redirected stderr. - soup: scrub any already-logged PASSWORD lines from postgres.log during post_to_3.2.0, rewriting in place to preserve the inode postgres is writing to. --- salt/logrotate/defaults.yaml | 10 +++++++ salt/logrotate/soc_logrotate.yaml | 7 +++++ salt/manager/tools/sbin/soup | 27 +++++++++++++++++++ salt/postgres/files/init-db.sh | 4 +++ salt/postgres/tools/sbin/so-telegraf-postgres | 4 +++ 5 files changed, 52 insertions(+) diff --git a/salt/logrotate/defaults.yaml b/salt/logrotate/defaults.yaml index 2261bb4f7..e37193d0d 100644 --- a/salt/logrotate/defaults.yaml +++ b/salt/logrotate/defaults.yaml @@ -150,6 +150,16 @@ logrotate: - extension .log - dateext - dateyesterday + /opt/so/log/postgres/*_x_log: + - daily + - rotate 14 + - missingok + - copytruncate + - compress + - create + - extension .log + - dateext + - dateyesterday /opt/so/log/telegraf/*_x_log: - daily - rotate 14 diff --git a/salt/logrotate/soc_logrotate.yaml b/salt/logrotate/soc_logrotate.yaml index f407ab48d..f329ed623 100644 --- a/salt/logrotate/soc_logrotate.yaml +++ b/salt/logrotate/soc_logrotate.yaml @@ -91,6 +91,13 @@ logrotate: multiline: True global: True forcedType: "[]string" + "/opt/so/log/postgres/*_x_log": + description: List of logrotate options for this file. + title: /opt/so/log/postgres/*.log + advanced: True + multiline: True + global: True + forcedType: "[]string" "/opt/so/log/telegraf/*_x_log": description: List of logrotate options for this file. title: /opt/so/log/telegraf/*.log diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index f5b40310e..8def509e1 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -905,6 +905,30 @@ bootstrap_so_soc_database() { docker restart so-soc } +scrub_postgres_log_passwords() { + # Before 3.2 PostgreSQL could log the full CREATE/ALTER ROLE ... PASSWORD + # statement -- with the plaintext password -- to postgres.log whenever such a + # DDL errored, because log_min_error_statement defaulted to 'error'. The leak + # is fixed going forward (the provisioning scripts now SET log_min_error_statement + # = panic), but any already-logged secret persists in the on-disk log. Purge it + # here. Only the manager runs so-postgres, so this is a local scrub. + local log=/opt/so/log/postgres/postgres.log + [[ -f "$log" ]] || return 0 + if ! grep -qai "PASSWORD '" "$log" 2>/dev/null; then + echo "No leaked passwords found in $log." + return 0 + fi + echo "Removing leaked password statements from $log." + local tmp + tmp=$(mktemp) + # Drop every line carrying a PASSWORD literal, then rewrite in place with + # `cat >` so the file keeps its inode -- postgres holds it open via redirected + # stderr and must keep logging to the same file. + grep -avi "PASSWORD '" "$log" > "$tmp" 2>/dev/null || true + cat "$tmp" > "$log" + rm -f "$tmp" +} + # Existing grids should keep ILM unless an admin explicitly opts in to DLM. pin_elasticsearch_data_retention_method() { local elasticsearch_file=/opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls @@ -991,6 +1015,9 @@ post_to_3.2.0() { bootstrap_so_soc_database + # Purge any plaintext passwords a pre-3.2 postgres logged on DDL errors. + scrub_postgres_log_passwords + # Generate 9.3.7 elastic agent installers echo "Regenerating Elastic Agent Installers" /sbin/so-elastic-agent-gen-installers diff --git a/salt/postgres/files/init-db.sh b/salt/postgres/files/init-db.sh index d12bc4c9b..d1f1e375f 100644 --- a/salt/postgres/files/init-db.sh +++ b/salt/postgres/files/init-db.sh @@ -8,6 +8,10 @@ if [ -z "${SO_POSTGRES_PASS:-}" ] && [ -n "${SO_POSTGRES_PASS_FILE:-}" ] && [ -r SO_POSTGRES_PASS="$(< "$SO_POSTGRES_PASS_FILE")" fi psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + -- Shield the plaintext password below from postgres.log if this DDL errors: + -- log_min_error_statement defaults to 'error', which would append a STATEMENT line + -- containing the full CREATE/ALTER ROLE ... PASSWORD text. panic suppresses that. + SET log_min_error_statement = panic; DO \$\$ BEGIN IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${SO_POSTGRES_USER}') THEN diff --git a/salt/postgres/tools/sbin/so-telegraf-postgres b/salt/postgres/tools/sbin/so-telegraf-postgres index ef7c3f9e6..e6cc69fa6 100644 --- a/salt/postgres/tools/sbin/so-telegraf-postgres +++ b/salt/postgres/tools/sbin/so-telegraf-postgres @@ -71,6 +71,10 @@ EOSQL -v role_user="$ROLE_USER" \ -v role_pass="$ROLE_PASS" \ -U postgres -d so_telegraf <<'EOSQL' +-- Shield the plaintext password below from postgres.log if this DDL errors: +-- log_min_error_statement defaults to 'error', which would append a STATEMENT line +-- containing the full CREATE/ALTER ROLE ... PASSWORD text. panic suppresses that. +SET log_min_error_statement = panic; SELECT format( CASE WHEN EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'role_user') THEN 'ALTER ROLE %I WITH LOGIN PASSWORD %L' From 011749ad09d383879e212e7624424130601e01b1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 20 Jul 2026 15:04:58 -0400 Subject: [PATCH 189/219] Self-heal PostgreSQL SOC database bootstrap init-db.sh only runs on a fresh PGDATA (docker-entrypoint-initdb.d), so a cluster left partially initialized -- e.g. the container restarted mid first-init by a watch trigger -- never recovered: the so_postgres role existed but its schema grants and the so_telegraf database were missing, breaking SOC with "permission denied for schema public". - init-db.sh: make the role upsert race-safe. Try CREATE ROLE and fall back to ALTER on duplicate_object/unique_violation instead of IF NOT EXISTS, so a concurrent creator no longer aborts the script (set -e + ON_ERROR_STOP=1) before the grants and so_telegraf creation run. The whole script is now idempotent and safe to re-run. - postgres/enabled.sls: move postgres_wait_ready here (was telegraf-gated in telegraf_users.sls) and add postgres_bootstrap_soc_db, which re-runs init-db.sh every highstate so a partially-initialized cluster self-heals. - telegraf_users.sls: drop its now-duplicate postgres_wait_ready definition; it resolves from postgres.enabled via the existing include. - soup: remove bootstrap_so_soc_database and its post_to_3.2.0 call. The highstates soup runs before postupgrade now reconcile the SOC DB, making the one-shot bootstrap redundant. --- salt/manager/tools/sbin/soup | 29 +++-------------------------- salt/postgres/enabled.sls | 20 ++++++++++++++++++++ salt/postgres/files/init-db.sh | 14 ++++++++++---- salt/postgres/telegraf_users.sls | 15 ++++----------- 4 files changed, 37 insertions(+), 41 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 8def509e1..eb63a1d60 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -880,31 +880,6 @@ recollate_postgres() { echo "" } -bootstrap_so_soc_database() { - # init-db.sh is mounted into so-postgres at /docker-entrypoint-initdb.d/init-db.sh - # and runs automatically only on a fresh data directory. Hosts upgrading from - # 3.1.0 already have /nsm/postgres populated, so the so_soc bootstrap block - # added in 3.2 never fires. Re-run the script explicitly; it's idempotent. - echo "Bootstrapping database via init-db.sh." - # The postgres image has no USER directive, so `docker exec` defaults to - # root, and the container env intentionally omits POSTGRES_USER (the upstream - # entrypoint defaults it transiently during first-init only). Recreate both - # so psql inside init-db.sh resolves the connect user correctly. - local exec_cmd="docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh" - if ! /usr/sbin/so-postgres-wait; then - FINAL_MESSAGE_QUEUE+=("WARNING: so-postgres was not ready during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd") - return 0 - fi - if ! $exec_cmd; then - FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the database may not have been bootstrapped. Re-run manually: $exec_cmd") - return 0 - fi - echo "Database bootstrap complete." - - echo "Restarting so-soc container to pick up database changes" - docker restart so-soc -} - scrub_postgres_log_passwords() { # Before 3.2 PostgreSQL could log the full CREATE/ALTER ROLE ... PASSWORD # statement -- with the plaintext password -- to postgres.log whenever such a @@ -1013,7 +988,9 @@ post_to_3.2.0() { # Recollate due to image OS rebase recollate_postgres - bootstrap_so_soc_database + # The SOC database (so_postgres role, schema/db grants, so_telegraf) is + # bootstrapped idempotently by the postgres.enabled highstate that runs + # before this postupgrade step, so no explicit bootstrap call is needed here. # Purge any plaintext passwords a pre-3.2 postgres logged on DDL errors. scrub_postgres_log_passwords diff --git a/salt/postgres/enabled.sls b/salt/postgres/enabled.sls index b0eadd205..ad6ff7819 100644 --- a/salt/postgres/enabled.sls +++ b/salt/postgres/enabled.sls @@ -85,6 +85,26 @@ so-postgres: - x509: postgres_crt - x509: postgres_key +postgres_wait_ready: + cmd.run: + - name: /usr/sbin/so-postgres-wait + - require: + - docker_container: so-postgres + - file: postgres_sbin + +# Re-run the SOC database bootstrap on every highstate so a cluster left +# partially initialized -- e.g. the container was restarted mid first-init by a +# watch trigger, so docker-entrypoint-initdb.d never completed -- self-heals: +# the so_postgres role, its schema/database grants, and the so_telegraf database +# are all reconciled idempotently. init-db.sh is idempotent and race-safe. +# POSTGRES_USER is injected because the container env intentionally omits it. +postgres_bootstrap_soc_db: + cmd.run: + - name: docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh + - require: + - cmd: postgres_wait_ready + - file: postgresinitdb + delete_so-postgres_so-status.disabled: file.uncomment: - name: /opt/so/conf/so-status/so-status.conf diff --git a/salt/postgres/files/init-db.sh b/salt/postgres/files/init-db.sh index d1f1e375f..540980469 100644 --- a/salt/postgres/files/init-db.sh +++ b/salt/postgres/files/init-db.sh @@ -12,13 +12,19 @@ psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-E -- log_min_error_statement defaults to 'error', which would append a STATEMENT line -- containing the full CREATE/ALTER ROLE ... PASSWORD text. panic suppresses that. SET log_min_error_statement = panic; + -- Race-safe upsert: try CREATE, and if the role was created concurrently + -- (another session re-entering init) fall back to ALTER. Catching the + -- exception -- rather than an IF NOT EXISTS check -- avoids a TOCTOU window + -- where CREATE ROLE would abort the whole script with a duplicate-key error + -- and skip the grants below. Both SQLSTATEs are covered: duplicate_object + -- (role already exists) and unique_violation (racy pg_authid index insert). DO \$\$ BEGIN - IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${SO_POSTGRES_USER}') THEN + BEGIN EXECUTE format('CREATE ROLE %I WITH LOGIN PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}'); - ELSE - EXECUTE format('ALTER ROLE %I WITH PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}'); - END IF; + EXCEPTION WHEN duplicate_object OR unique_violation THEN + EXECUTE format('ALTER ROLE %I WITH LOGIN PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}'); + END; END \$\$; GRANT ALL ON SCHEMA public TO "$SO_POSTGRES_USER"; diff --git a/salt/postgres/telegraf_users.sls b/salt/postgres/telegraf_users.sls index 5e3566a95..62c33b892 100644 --- a/salt/postgres/telegraf_users.sls +++ b/salt/postgres/telegraf_users.sls @@ -8,23 +8,16 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'telegraf/map.jinja' import TELEGRAFMERGED %} -{# postgres_wait_ready below requires `docker_container: so-postgres`, which is - declared in postgres.enabled. Include it here so state.apply postgres.telegraf_users - on its own (e.g. from orch.deploy_newnode) still has that ID in scope. Salt - de-duplicates the circular include. #} +{# postgres_wait_ready and the so-postgres container are declared in + postgres.enabled; include it so the require references below resolve and so + state.apply postgres.telegraf_users on its own (e.g. from orch.deploy_newnode) + still has those IDs in scope. Salt de-duplicates the circular include. #} include: - postgres.enabled {% set TG_OUT = TELEGRAFMERGED.output | upper %} {% if TG_OUT in ['POSTGRES', 'BOTH'] %} -postgres_wait_ready: - cmd.run: - - name: /usr/sbin/so-postgres-wait - - require: - - docker_container: so-postgres - - file: postgres_sbin - # Ensure the shared Telegraf database exists. init-db.sh only runs on a # fresh data dir, so hosts upgraded onto an existing /nsm/postgres volume # would otherwise never get so_telegraf. From 387781c6292c96aec01b3f0a66b103a9a29d9057 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 20 Jul 2026 15:19:02 -0400 Subject: [PATCH 190/219] Trim verbose comments in postgres provisioning changes --- salt/manager/tools/sbin/soup | 17 +++-------------- salt/postgres/enabled.sls | 9 +++------ salt/postgres/files/init-db.sh | 12 +++--------- salt/postgres/telegraf_users.sls | 6 ++---- salt/postgres/tools/sbin/so-telegraf-postgres | 4 +--- 5 files changed, 12 insertions(+), 36 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index eb63a1d60..9f02eb32a 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -881,12 +881,7 @@ recollate_postgres() { } scrub_postgres_log_passwords() { - # Before 3.2 PostgreSQL could log the full CREATE/ALTER ROLE ... PASSWORD - # statement -- with the plaintext password -- to postgres.log whenever such a - # DDL errored, because log_min_error_statement defaulted to 'error'. The leak - # is fixed going forward (the provisioning scripts now SET log_min_error_statement - # = panic), but any already-logged secret persists in the on-disk log. Purge it - # here. Only the manager runs so-postgres, so this is a local scrub. + # Purge plaintext passwords a pre-3.2 postgres could log on DDL errors. local log=/opt/so/log/postgres/postgres.log [[ -f "$log" ]] || return 0 if ! grep -qai "PASSWORD '" "$log" 2>/dev/null; then @@ -896,9 +891,7 @@ scrub_postgres_log_passwords() { echo "Removing leaked password statements from $log." local tmp tmp=$(mktemp) - # Drop every line carrying a PASSWORD literal, then rewrite in place with - # `cat >` so the file keeps its inode -- postgres holds it open via redirected - # stderr and must keep logging to the same file. + # Rewrite in place (cat >) to keep the inode postgres is writing to. grep -avi "PASSWORD '" "$log" > "$tmp" 2>/dev/null || true cat "$tmp" > "$log" rm -f "$tmp" @@ -988,11 +981,7 @@ post_to_3.2.0() { # Recollate due to image OS rebase recollate_postgres - # The SOC database (so_postgres role, schema/db grants, so_telegraf) is - # bootstrapped idempotently by the postgres.enabled highstate that runs - # before this postupgrade step, so no explicit bootstrap call is needed here. - - # Purge any plaintext passwords a pre-3.2 postgres logged on DDL errors. + # SOC database bootstrap is handled by the postgres.enabled highstate. scrub_postgres_log_passwords # Generate 9.3.7 elastic agent installers diff --git a/salt/postgres/enabled.sls b/salt/postgres/enabled.sls index ad6ff7819..fa91d146a 100644 --- a/salt/postgres/enabled.sls +++ b/salt/postgres/enabled.sls @@ -92,12 +92,9 @@ postgres_wait_ready: - docker_container: so-postgres - file: postgres_sbin -# Re-run the SOC database bootstrap on every highstate so a cluster left -# partially initialized -- e.g. the container was restarted mid first-init by a -# watch trigger, so docker-entrypoint-initdb.d never completed -- self-heals: -# the so_postgres role, its schema/database grants, and the so_telegraf database -# are all reconciled idempotently. init-db.sh is idempotent and race-safe. -# POSTGRES_USER is injected because the container env intentionally omits it. +# Reconcile the SOC database (role, grants, so_telegraf) every highstate so a +# partially-initialized cluster self-heals. POSTGRES_USER is injected because +# the container env omits it. postgres_bootstrap_soc_db: cmd.run: - name: docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh diff --git a/salt/postgres/files/init-db.sh b/salt/postgres/files/init-db.sh index 540980469..4d65b0c97 100644 --- a/salt/postgres/files/init-db.sh +++ b/salt/postgres/files/init-db.sh @@ -8,16 +8,10 @@ if [ -z "${SO_POSTGRES_PASS:-}" ] && [ -n "${SO_POSTGRES_PASS_FILE:-}" ] && [ -r SO_POSTGRES_PASS="$(< "$SO_POSTGRES_PASS_FILE")" fi psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL - -- Shield the plaintext password below from postgres.log if this DDL errors: - -- log_min_error_statement defaults to 'error', which would append a STATEMENT line - -- containing the full CREATE/ALTER ROLE ... PASSWORD text. panic suppresses that. + -- Keep the password out of postgres.log if this DDL errors. SET log_min_error_statement = panic; - -- Race-safe upsert: try CREATE, and if the role was created concurrently - -- (another session re-entering init) fall back to ALTER. Catching the - -- exception -- rather than an IF NOT EXISTS check -- avoids a TOCTOU window - -- where CREATE ROLE would abort the whole script with a duplicate-key error - -- and skip the grants below. Both SQLSTATEs are covered: duplicate_object - -- (role already exists) and unique_violation (racy pg_authid index insert). + -- Idempotent, race-safe upsert: CREATE, falling back to ALTER if the role + -- already exists or is created concurrently. DO \$\$ BEGIN BEGIN diff --git a/salt/postgres/telegraf_users.sls b/salt/postgres/telegraf_users.sls index 62c33b892..4c63d40b0 100644 --- a/salt/postgres/telegraf_users.sls +++ b/salt/postgres/telegraf_users.sls @@ -8,10 +8,8 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'telegraf/map.jinja' import TELEGRAFMERGED %} -{# postgres_wait_ready and the so-postgres container are declared in - postgres.enabled; include it so the require references below resolve and so - state.apply postgres.telegraf_users on its own (e.g. from orch.deploy_newnode) - still has those IDs in scope. Salt de-duplicates the circular include. #} +{# postgres.enabled declares the so-postgres container and postgres_wait_ready + that the requires below reference. Salt de-duplicates the circular include. #} include: - postgres.enabled diff --git a/salt/postgres/tools/sbin/so-telegraf-postgres b/salt/postgres/tools/sbin/so-telegraf-postgres index e6cc69fa6..17e4da744 100644 --- a/salt/postgres/tools/sbin/so-telegraf-postgres +++ b/salt/postgres/tools/sbin/so-telegraf-postgres @@ -71,9 +71,7 @@ EOSQL -v role_user="$ROLE_USER" \ -v role_pass="$ROLE_PASS" \ -U postgres -d so_telegraf <<'EOSQL' --- Shield the plaintext password below from postgres.log if this DDL errors: --- log_min_error_statement defaults to 'error', which would append a STATEMENT line --- containing the full CREATE/ALTER ROLE ... PASSWORD text. panic suppresses that. +-- Keep the password out of postgres.log if this DDL errors. SET log_min_error_statement = panic; SELECT format( CASE WHEN EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'role_user') From b104f0955a54c3877b737675ce8df013316dc93e Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:33:25 -0500 Subject: [PATCH 191/219] es annotations --- salt/elasticsearch/soc_elasticsearch.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 95880bb9d..995dcd38a 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -645,6 +645,9 @@ elasticsearch: global: True advanced: True helpLink: elasticsearch + so-assistant-chat: *dataStreamSettings + so-assistant-session: *dataStreamSettings + so-elastic-agent-monitor: *dataStreamSettings so-logs-soc: *dataStreamSettings so-logs-system_x_auth: *dataStreamSettings so-logs-system_x_syslog: *dataStreamSettings @@ -667,7 +670,10 @@ elasticsearch: so-logs-elastic_agent_x_auditbeat: *dataStreamSettings so-logs-elastic_agent_x_cloudbeat: *dataStreamSettings so-logs-elastic_agent_x_endpoint_security: *dataStreamSettings + so-logs-endpoint_x_actions: *dataStreamSettings + so-logs-endpoint_x_action_x_responses: *dataStreamSettings so-logs-endpoint_x_alerts: *dataStreamSettings + so-logs-endpoint_x_diagnostic_x_collection: *dataStreamSettings so-logs-endpoint_x_events_x_api: *dataStreamSettings so-logs-endpoint_x_events_x_file: *dataStreamSettings so-logs-endpoint_x_events_x_library: *dataStreamSettings @@ -675,6 +681,7 @@ elasticsearch: so-logs-endpoint_x_events_x_process: *dataStreamSettings so-logs-endpoint_x_events_x_registry: *dataStreamSettings so-logs-endpoint_x_events_x_security: *dataStreamSettings + so-logs-endpoint_x_heartbeat: *dataStreamSettings so-logs-elastic_agent_x_filebeat: *dataStreamSettings so-logs-elastic_agent_x_fleet_server: *dataStreamSettings so-logs-elastic_agent_x_heartbeat: *dataStreamSettings @@ -987,8 +994,6 @@ elasticsearch: helpLink: elasticsearch sos-backup: *indexSettings so-detection: *indexSettings - so-assistant-chat: *indexSettings - so-assistant-session: *indexSettings so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings index_sorting: description: Sorts the index by event time, at the cost of additional processing resource consumption. From f542d1e7ce7177c06e31bc200b884f80b360ccf6 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:34:03 -0500 Subject: [PATCH 192/219] include current version in so-elastic-fleet-package-upgrade attempt --- .../tools/sbin_jinja/so-elastic-fleet-package-upgrade | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade index f3d77b852..d645403d5 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade @@ -17,7 +17,7 @@ if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && L if [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then echo "{{ PACKAGE }} integration version $INSTALLED_VERSION is already at the reported latest version $LATEST_VERSION, skipping upgrade." else - echo "Upgrading {{ PACKAGE }} package to version $LATEST_VERSION..." + echo "Upgrading {{ PACKAGE }} package from $INSTALLED_VERSION to version $LATEST_VERSION..." if ! elastic_fleet_package_install "{{ PACKAGE }}" "$LATEST_VERSION"; then PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1)) PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}") From 8d168e96614156b9b3b48c448af5c13d3481bc81 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:57:08 -0500 Subject: [PATCH 193/219] remove rollover config from ilm warm phase --- salt/elasticsearch/soc_elasticsearch.yaml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 995dcd38a..bf5b665c3 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -887,17 +887,6 @@ elasticsearch: global: True advanced: True helpLink: elasticsearch - rollover: - max_age: - description: Maximum age of index. Once an index reaches this limit, it will be rolled over into a new index. - global: True - advanced: True - helpLink: elasticsearch - max_primary_shard_size: - description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index. - global: True - advanced: True - helpLink: elasticsearch shrink: method: description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size. From c4c1464b2a8b17a7652d1a70ae83a7abe72c6293 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 21 Jul 2026 10:07:33 -0400 Subject: [PATCH 194/219] Better support SigmaHQ rules --- .../files/soc/playbook_placeholder_map.yaml | 15 +- .../files/soc/sigma_playbook_pipeline.yaml | 123 ++++++++++- salt/soc/files/soc/sigma_so_pipeline.yaml | 209 +++++++++++++++++- 3 files changed, 339 insertions(+), 8 deletions(-) diff --git a/salt/soc/files/soc/playbook_placeholder_map.yaml b/salt/soc/files/soc/playbook_placeholder_map.yaml index eeddb4474..aac5a12ac 100644 --- a/salt/soc/files/soc/playbook_placeholder_map.yaml +++ b/salt/soc/files/soc/playbook_placeholder_map.yaml @@ -14,18 +14,22 @@ CommandLine: process.command_line CurrentDirectory: process.working_directory +Details: registry.data.strings Image: process.executable -ImageLoaded: dll.name +ImageLoaded: dll.path ParentImage: process.parent.executable ParentName: process.parent.name ParentProcessGuid: process.parent.entity_id ProcessGuid: process.entity_id -TargetFilename: file.name +TargetFilename: file.path TargetObject: registry.path TargetUserName: user.target.name User: user.name community_id: network.community_id dns_resolved_ip: dns.resolved_ip +QueryName: dns.question.name +dll_basename: dll.name +file_basename: file.name document_id: soc_id dst_ip: destination.ip dst_port: destination.port @@ -40,10 +44,15 @@ public_ip: network.public_ip related_hosts: related.hosts related_ip: related.ip src_ip: source.ip -dns_query_name: dns.query_name +dns_query_name: dns.question.name flow_id: log.id.uid payload: network.data.decoded rule_category: rule.category rule_name: rule.name rule_uuid: rule.uuid src_port: source.port +# PowerShell channel (ps_script EID 4104 / classic EID 400) +ScriptBlockText: powershell.file.script_block_text +script_block_id: powershell.file.script_block_id +script_block_hash: powershell.file.script_block_hash +EngineVersion: powershell.engine.version diff --git a/salt/soc/files/soc/sigma_playbook_pipeline.yaml b/salt/soc/files/soc/sigma_playbook_pipeline.yaml index 6fcdedbb4..04bd2ffaf 100644 --- a/salt/soc/files/soc/sigma_playbook_pipeline.yaml +++ b/salt/soc/files/soc/sigma_playbook_pipeline.yaml @@ -1,8 +1,9 @@ name: Security Onion - Playbook Pipeline priority: 97 transformations: - # Route string fields to their lowercase-normalized .caseless subfield so wildcard - # matches are case-insensitive. + # Route to lowercase-normalized .caseless subfields for case-insensitive matching. + # file.path.caseless exists on Defend only (Sysmon file events lack it); + # registry.path / dll.path / file.name have no .caseless on any source. - id: case_insensitive_string_fields type: field_name_mapping mapping: @@ -10,3 +11,121 @@ transformations: process.parent.executable: process.parent.executable.caseless process.command_line: process.command_line.caseless process.parent.command_line: process.parent.command_line.caseless + file.path: file.path.caseless + # file_activity: playbook-only pseudo-category spanning all file operations. + - id: playbook_file_activity_add-fields + type: add_condition + conditions: + event.category: 'file' + rule_conditions: + - type: logsource + category: file_activity + # EventType -> event.action is safe as a projection only — the values differ + # (Sysmon DeleteFile/SetValue vs ECS deletion/modification). + - id: playbook_file_activity_event_type + type: field_name_mapping + mapping: + winlog.event_data.EventType: event.action + rule_conditions: + - type: logsource + category: file_activity + - id: playbook_file_event_event_type + type: field_name_mapping + mapping: + winlog.event_data.EventType: event.action + rule_conditions: + - type: logsource + category: file_event + - id: playbook_file_delete_event_type + type: field_name_mapping + mapping: + winlog.event_data.EventType: event.action + rule_conditions: + - type: logsource + category: file_delete + - id: playbook_file_rename_event_type + type: field_name_mapping + mapping: + winlog.event_data.EventType: event.action + rule_conditions: + - type: logsource + category: file_rename + - id: playbook_registry_set_event_type + type: field_name_mapping + mapping: + winlog.event_data.EventType: event.action + rule_conditions: + - type: logsource + category: registry_set + # WMI-Activity events carry no event.category; event.dataset is source-specific + - id: playbook_wmi_activity_add-fields + type: add_condition + conditions: + event.dataset: 'wmi.operational' + rule_conditions: + - type: logsource + category: wmi + - id: playbook_wmi_event_activity_add-fields + type: add_condition + conditions: + event.dataset: 'wmi.operational' + rule_conditions: + - type: logsource + category: wmi_event + - id: playbook_ps_script_field-mapping + type: field_name_mapping + mapping: + ScriptBlockText: powershell.file.script_block_text + Path: file.path + rule_conditions: + - type: logsource + category: ps_script + - id: playbook_ps_script_add-fields + type: add_condition + conditions: + event.dataset: 'windows.powershell_operational' + event.code: '4104' + rule_conditions: + - type: logsource + category: ps_script + # Data (raw EID 400/600 message blob) -> process.command_line: the winlog + # integration parses HostApplication into it; `message` is analyzed text and + # wildcard matches on it silently return zero. No .caseless on this dataset. + - id: playbook_ps_classic_start_field-mapping + type: field_name_mapping + mapping: + Data: process.command_line + rule_conditions: + - type: logsource + category: ps_classic_start + - id: playbook_ps_classic_start_add-fields + type: add_condition + conditions: + event.dataset: 'windows.powershell' + event.code: '400' + rule_conditions: + - type: logsource + category: ps_classic_start + - id: playbook_ps_classic_provider_start_field-mapping + type: field_name_mapping + mapping: + Data: process.command_line + rule_conditions: + - type: logsource + category: ps_classic_provider_start + - id: playbook_ps_classic_provider_start_add-fields + type: add_condition + conditions: + event.dataset: 'windows.powershell' + event.code: '600' + rule_conditions: + - type: logsource + category: ps_classic_provider_start + # Alert docs nest the host under event_data.host.name; no top-level host.name. + - id: playbook_alert_host_field + type: field_name_mapping + mapping: + host.name: event_data.host.name + rule_conditions: + - type: logsource + category: alert diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index dee93c84e..281acb8a9 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -131,13 +131,22 @@ transformations: type: field_name_mapping mapping: winlog.event_data.Details: registry.data.strings - # field rename only; EventType values (SetValue/CreateKey) still differ from - # event.action values (modification/creation) - winlog.event_data.EventType: event.action rule_conditions: - type: logsource product: windows category: registry_set + # ecs_windows renames Initiated -> network.direction without translating the value, + # so rules compile to network.direction:"true" and never match (field holds + # ingress/egress). network.initiated carries the boolean on both Defend and Sysmon. + # Keyed on network.direction because ecs_windows has already renamed by this layer. + - id: ecs_fix_network_connection_initiated + type: field_name_mapping + mapping: + network.direction: network.initiated + rule_conditions: + - type: logsource + product: windows + category: network_connection - id: ecs_fix_image_load type: field_name_mapping mapping: @@ -150,6 +159,30 @@ transformations: - type: logsource product: windows category: image_load + # Defend reports the driver image in dll.path (file.path is null on driver events) — + # same renames as image_load. + - id: ecs_fix_driver_load + type: field_name_mapping + mapping: + file.path: dll.path + file.code_signature.signed: dll.code_signature.exists + winlog.event_data.Signature: dll.code_signature.subject_name + file.code_signature.status: dll.code_signature.status + winlog.event_data.Hashes: dll.hash.sha256 + rule_conditions: + - type: logsource + product: windows + category: driver_load + - id: ecs_fix_file_rename + type: field_name_mapping + mapping: + # Defend records the pre-rename path in file.Ext.original.path; Sysmon's + # SourceFilename has no ECS equivalent. + winlog.event_data.SourceFilename: file.Ext.original.path + rule_conditions: + - type: logsource + product: windows + category: file_rename - id: linux_security_add-fields type: add_condition conditions: @@ -323,6 +356,33 @@ transformations: rule_conditions: - type: logsource category: file_event + # Without event.type scoping, file_delete rules also match creations. + - id: endpoint_file_delete_add-fields + type: add_condition + conditions: + event.category: 'file' + event.type: 'deletion' + rule_conditions: + - type: logsource + category: file_delete + # Defend reports renames as event.action:rename with event.type:change. + - id: endpoint_file_rename_add-fields + type: add_condition + conditions: + event.category: 'file' + event.action: 'rename' + rule_conditions: + - type: logsource + category: file_rename + # event.type:change selects writes and excludes Defend's read-only registry events. + - id: endpoint_registry_set_add-fields + type: add_condition + conditions: + event.category: 'registry' + event.type: 'change' + rule_conditions: + - type: logsource + category: registry_set # Scope image_load rules to Elastic Endpoint library events (event.category:library, dll.* # populated). - id: endpoint_image_load_add-fields @@ -351,6 +411,149 @@ transformations: rule_conditions: - type: logsource category: network_connection + # Scope on lookup actions, not network.protocol:dns (also matches Zeek port-53). + - id: endpoint_dns_query_add-fields + type: add_condition + conditions: + event.category: 'network' + event.action: + - 'lookup_requested' + - 'lookup_result' + rule_conditions: + - type: logsource + category: dns_query + # OS gates: without them a `product: windows` rule in these categories also matches + # Linux/macOS. Separate conditions (not `product:` on the mappings above) so the + # mappings keep scoping product-less rules. + - id: endpoint_file_create_windows_os-gate + type: add_condition + conditions: + host.os.type: 'windows' + rule_conditions: + - type: logsource + category: file_event + product: windows + - id: endpoint_file_create_linux_os-gate + type: add_condition + conditions: + host.os.type: 'linux' + rule_conditions: + - type: logsource + category: file_event + product: linux + - id: endpoint_file_create_macos_os-gate + type: add_condition + conditions: + host.os.type: 'macos' + rule_conditions: + - type: logsource + category: file_event + product: macos + - id: endpoint_file_delete_windows_os-gate + type: add_condition + conditions: + host.os.type: 'windows' + rule_conditions: + - type: logsource + category: file_delete + product: windows + - id: endpoint_file_delete_linux_os-gate + type: add_condition + conditions: + host.os.type: 'linux' + rule_conditions: + - type: logsource + category: file_delete + product: linux + - id: endpoint_file_delete_macos_os-gate + type: add_condition + conditions: + host.os.type: 'macos' + rule_conditions: + - type: logsource + category: file_delete + product: macos + - id: endpoint_file_rename_windows_os-gate + type: add_condition + conditions: + host.os.type: 'windows' + rule_conditions: + - type: logsource + category: file_rename + product: windows + - id: endpoint_file_rename_linux_os-gate + type: add_condition + conditions: + host.os.type: 'linux' + rule_conditions: + - type: logsource + category: file_rename + product: linux + - id: endpoint_file_rename_macos_os-gate + type: add_condition + conditions: + host.os.type: 'macos' + rule_conditions: + - type: logsource + category: file_rename + product: macos + - id: endpoint_network_connection_windows_os-gate + type: add_condition + conditions: + host.os.type: 'windows' + rule_conditions: + - type: logsource + category: network_connection + product: windows + - id: endpoint_network_connection_linux_os-gate + type: add_condition + conditions: + host.os.type: 'linux' + rule_conditions: + - type: logsource + category: network_connection + product: linux + - id: endpoint_network_connection_macos_os-gate + type: add_condition + conditions: + host.os.type: 'macos' + rule_conditions: + - type: logsource + category: network_connection + product: macos + - id: endpoint_dns_query_windows_os-gate + type: add_condition + conditions: + host.os.type: 'windows' + rule_conditions: + - type: logsource + category: dns_query + product: windows + - id: endpoint_dns_query_linux_os-gate + type: add_condition + conditions: + host.os.type: 'linux' + rule_conditions: + - type: logsource + category: dns_query + product: linux + - id: endpoint_dns_query_macos_os-gate + type: add_condition + conditions: + host.os.type: 'macos' + rule_conditions: + - type: logsource + category: dns_query + product: macos + # registry_set / image_load / driver_load are not gated — windows-only telemetry. + # wmi / wmi_event are not scoped here: their only handle is event.dataset + - id: endpoint_driver_load_add-fields + type: add_condition + conditions: + event.category: 'driver' + rule_conditions: + - type: logsource + category: driver_load # Maps "alert" category to SO Alert events - id: alert_so_add-fields type: add_condition From 4e1935f8a0bad057dee6a15c7360ed7d7e3ef958 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 21 Jul 2026 11:58:11 -0400 Subject: [PATCH 195/219] postgress updates --- salt/common/tools/sbin/so-log-check | 1 + salt/soc/defaults.yaml | 2 +- salt/soc/merged.map.jinja | 2 +- salt/soc/soc_soc.yaml | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 42ceea47b..56c0278da 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -133,6 +133,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Cancelling deferred write event maybeFenceReplicas because the event queue is now closed" # Kafka controller log during shutdown/restart EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Redis may have been restarted" # Redis likely restarted by salt EXCLUDED_ERRORS="$EXCLUDED_ERRORS|file already closed" # Go logging race condition during container restart + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|relation \"audit_settings\" does not exist" # salt checking for changes before SOC starts fi if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 9705c1863..e04b966c0 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1539,7 +1539,7 @@ soc: postgres: host: "" port: 5432 - sslMode: "allow" + sslMode: "require" database: securityonion user: "" password: "" diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index a421711e2..452fba0b9 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -88,7 +88,7 @@ 'host': GLOBALS.manager_ip, 'password': PG_PASS, 'port': 5432, - 'sslMode': 'allow', + 'sslMode': 'require', 'user': PG_USER, } }) %} diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index f09317f39..eb4cd3154 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -486,7 +486,7 @@ soc: global: True advanced: True sslMode: - description: "Use encrypted connections to the PostgreSQL server. Must be one of the following values: disable, allow, prefer, require, verify-ca, verify-full. Defaults to allow." + description: "Use encrypted connections to the PostgreSQL server. Must be one of the following values: disable, allow, prefer, require, verify-ca, verify-full." global: True advanced: True database: From 894d323323fe3cad8204721f054d1fd26d4d74bd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 21 Jul 2026 15:08:24 -0400 Subject: [PATCH 196/219] Map default agents to model displayName, not id agentMapping values are model displayNames (the canonical selector); the stock config used the model id, which only resolved via the legacy id@adapter fallback. Use the Claude Sonnet displayName so agent-to-model resolution matches the documented contract. --- salt/soc/defaults.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 9705c1863..c0e01f80d 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1530,9 +1530,9 @@ soc: healthTimeoutSeconds: 5 agentic: false agentMapping: - Orchestrator: sonnet - Investigator: sonnet - Detection Engineer: sonnet + Orchestrator: Claude Sonnet + Investigator: Claude Sonnet + Detection Engineer: Claude Sonnet onionconfig: saltstackDir: /opt/so/saltstack bypassEnabled: false From ac46636196387378d30d15b2bd7844500c2944ca Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 21 Jul 2026 16:05:18 -0400 Subject: [PATCH 197/219] ensure salt-master service restarted last --- salt/salt/master.sls | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/salt/master.sls b/salt/salt/master.sls index 2c47a95c6..12ad0e344 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -94,7 +94,9 @@ salt_master_service: - file: checkmine_engine - file: pillarWatch_engine - file: engines_config - - order: 9002 + - require: + - cmd: wait_for_salt_minion_ready + - order: last {% else %} From f77fad80871b504649ca7d168743982a5424b8e1 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 21 Jul 2026 16:29:24 -0400 Subject: [PATCH 198/219] apply salt.master state instead of just salt.minion --- salt/manager/tools/sbin/soup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 9f02eb32a..174d6fd7b 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1998,11 +1998,11 @@ main() { # Testing that salt-master is up by checking that is it connected to itself check_saltmaster_status - # update the salt-minion configs here and start the minion + # update the salt-master and salt-minion configs here and start the minion # since highstate are disabled above, minion start should not trigger a highstate echo "" - echo "Ensuring salt-minion configs are up-to-date." - salt-call state.apply salt.minion -l info queue=True + echo "Ensuring salt-master and salt-minion configs are up-to-date." + salt-call state.apply salt.master -l info queue=True echo "" # ensure the mine is updated and populated before highstates run, following the salt-master restart From 334978ad925177aea161147211e970af907dfe11 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 22 Jul 2026 11:55:12 -0400 Subject: [PATCH 199/219] Rename Detection Engineer agent key to DetectionEngineer The agent name is also the agentMapping config key and a SOC setting id segment, so the space made it awkward to target. Matches the rename in securityonion-soc. --- salt/soc/defaults.yaml | 2 +- salt/soc/soc_soc.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 51182c976..c89b39dcf 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1532,7 +1532,7 @@ soc: agentMapping: Orchestrator: Claude Sonnet Investigator: Claude Sonnet - Detection Engineer: Claude Sonnet + DetectionEngineer: Claude Sonnet onionconfig: saltstackDir: /opt/so/saltstack bypassEnabled: false diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index eb4cd3154..fa0d39ae4 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -786,7 +786,7 @@ soc: Investigator: description: This agent investigates alerts, explains events and records, and hunts through event data. It can also acknowledge alerts and escalate to cases. global: True - Detection Engineer: + DetectionEngineer: description: This agent manages detections and their overrides, including tuning noisy rules and authoring rule content. global: True client: From f4aa9932ffef0ee7a91a104f513e1305ff36097a Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 22 Jul 2026 12:13:45 -0500 Subject: [PATCH 200/219] add default templates for system integration metrics indices --- salt/elasticsearch/defaults.yaml | 714 ++++++++++++++++++++++ salt/elasticsearch/soc_elasticsearch.yaml | 13 + 2 files changed, 727 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 14d753c21..5179fac2b 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -3453,6 +3453,720 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-metrics-system_x_core: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.core@package + - metrics@custom + - system@custom + - metrics-system.core@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.core@custom + index_patterns: + - metrics-system.core-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.core-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_cpu: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.cpu@package + - metrics@custom + - system@custom + - metrics-system.cpu@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.cpu@custom + index_patterns: + - metrics-system.cpu-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.cpu-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_diskio: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.diskio@package + - metrics@custom + - system@custom + - metrics-system.diskio@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.diskio@custom + index_patterns: + - metrics-system.diskio-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.diskio-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_filesystem: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.filesystem@package + - metrics@custom + - system@custom + - metrics-system.filesystem@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.filesystem@custom + index_patterns: + - metrics-system.filesystem-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.filesystem-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_fsstat: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.fsstat@package + - metrics@custom + - system@custom + - metrics-system.fsstat@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.fsstat@custom + index_patterns: + - metrics-system.fsstat-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.fsstat-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_load: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.load@package + - metrics@custom + - system@custom + - metrics-system.load@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.load@custom + index_patterns: + - metrics-system.load-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.load-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_memory: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.memory@package + - metrics@custom + - system@custom + - metrics-system.memory@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.memory@custom + index_patterns: + - metrics-system.memory-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.memory-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_network: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.network@package + - metrics@custom + - system@custom + - metrics-system.network@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.network@custom + index_patterns: + - metrics-system.network-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.network-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_ntp: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@settings + - metrics-system.ntp@package + - metrics@custom + - system@custom + - metrics-system.ntp@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.ntp@custom + index_patterns: + - metrics-system.ntp-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.ntp-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_process: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.process@package + - metrics@custom + - system@custom + - metrics-system.process@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.process@custom + index_patterns: + - metrics-system.process-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.process-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_process_x_summary: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.process.summary@package + - metrics@custom + - system@custom + - metrics-system.process.summary@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.process.summary@custom + index_patterns: + - metrics-system.process.summary-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.process.summary-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_socket_summary: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.socket_summary@package + - metrics@custom + - system@custom + - metrics-system.socket_summary@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.socket_summary@custom + index_patterns: + - metrics-system.socket_summary-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.socket_summary-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_uptime: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.uptime@package + - metrics@custom + - system@custom + - metrics-system.uptime@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.uptime@custom + index_patterns: + - metrics-system.uptime-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.uptime-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-windows_x_forwarded: index_sorting: false data_stream_lifecycle: diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index bf5b665c3..ef9c066d3 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -697,6 +697,19 @@ elasticsearch: so-metrics-vsphere_x_datastore: *dataStreamSettings so-metrics-vsphere_x_host: *dataStreamSettings so-metrics-vsphere_x_virtualmachine: *dataStreamSettings + so-metrics-system_x_core: *dataStreamSettings + so-metrics-system_x_cpu: *dataStreamSettings + so-metrics-system_x_diskio: *dataStreamSettings + so-metrics-system_x_filesystem: *dataStreamSettings + so-metrics-system_x_fsstat: *dataStreamSettings + so-metrics-system_x_load: *dataStreamSettings + so-metrics-system_x_memory: *dataStreamSettings + so-metrics-system_x_network: *dataStreamSettings + so-metrics-system_x_ntp: *dataStreamSettings + so-metrics-system_x_process: *dataStreamSettings + so-metrics-system_x_process_x_summary: *dataStreamSettings + so-metrics-system_x_socket_summary: *dataStreamSettings + so-metrics-system_x_uptime: *dataStreamSettings so-common: *dataStreamSettings so-endgame: *dataStreamSettings so-idh: *dataStreamSettings From 382dee1d0644d09620994a21114fbf273125ea85 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 22 Jul 2026 12:19:36 -0500 Subject: [PATCH 201/219] include global_override data_retention default for addon integrations that don't yet have a default value defined --- salt/elasticsearch/template.map.jinja | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 72be1ec58..c460f4ab1 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -109,9 +109,15 @@ {% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %} {% do settings.index_template.template.settings.index.pop('sort') %} {% endif %} -{% if DATA_RETENTION_METHOD == 'DLM' and settings.index_template.data_stream is defined and settings.data_stream_lifecycle is defined %} -{% if settings.data_stream_lifecycle.data_retention is defined and settings.data_stream_lifecycle.data_retention %} -{% do settings.index_template.template.update({'lifecycle': {'data_retention': settings.data_stream_lifecycle.data_retention}}) %} +{% if DATA_RETENTION_METHOD == 'DLM' and settings.index_template.data_stream is defined %} +{# Addon defaults are generated without data_stream_lifecycle, so fall back to global defaults. #} +{% if settings.data_stream_lifecycle is defined %} +{% set DATA_STREAM_LIFECYCLE = settings.data_stream_lifecycle %} +{% else %} +{% set DATA_STREAM_LIFECYCLE = DEFAULT_GLOBAL_OVERRIDES.data_stream_lifecycle %} +{% endif %} +{% if DATA_STREAM_LIFECYCLE.data_retention is defined and DATA_STREAM_LIFECYCLE.data_retention %} +{% do settings.index_template.template.update({'lifecycle': {'data_retention': DATA_STREAM_LIFECYCLE.data_retention}}) %} {% else %} {% do settings.index_template.template.update({'lifecycle': {}}) %} {% endif %} From 445ae58919fb9813371e3976324dd43cf32da05b Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:34:12 -0500 Subject: [PATCH 202/219] add optional dlm configs --- salt/elasticsearch/soc_elasticsearch.yaml | 37 +++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index ef9c066d3..a885f15bc 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -64,6 +64,43 @@ elasticsearch: flood_stage: description: The max percentage of used disk space that will cause the node to take protective actions, such as blocking incoming events. helpLink: elasticsearch + lifecycle: + default: + rollover: + description: This property accepts a key value pair formatted string and configures the conditions that would trigger a data stream to rollover when it has lifecycle configured. + forcedType: string + regex: ^max_age=(auto|[1-9][0-9]*[hd]),max_primary_shard_size=[1-9][0-9]*gb,min_docs=(0|[1-9][0-9]*),max_primary_shard_docs=[1-9][0-9]*$ + regexFailureMessage: Must be in the format of "max_age=auto|,max_primary_shard_size=gb,min_docs=,max_primary_shard_docs=". + advanced: True + global: True + data_streams: + lifecycle: + poll_interval: + description: How often Elasticsearch checks what the next action is for all data streams with a built-in lifecycle. + forcedType: string + regex: "^[1-9][0-9]*[mhd]$" + regexFailureMessage: Must be a number followed by m, h, or d. + advanced: True + global: true + helpLink: elasticsearch + target: + merge: + policy: + merge_factor: + description: Data stream lifecycle implements tail merging by updating the Lucene merge policy factor for the target backing index. The merge factor is both the number of segments that should be merged together, and the maximum number of segments that we expect to find. + forcedType: int + regex: "^[1-9][0-9]*$" + advanced: True + global: true + helpLink: elasticsearch + floor_segment: + description: Data stream lifecycle implements tail merging by updating the Lucene merge policy floor segment for the target backing index. This floor segment size is a way to prevent indices from having a long tail of very small segments. + forcedType: string + regex: "^[1-9][0-9]*[MG]B$" + regexFailureMessage: Must be a number followed by MB or GB, such as 100MB. + advanced: True + global: true + helpLink: elasticsearch action: destructive_requires_name: description: Requires explicit index names when deleting indices. Prevents accidental deletion of indices via wildcard patterns. From e4c14a92949073c4e8b706b88092280394ee1d8c Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 22 Jul 2026 16:12:24 -0500 Subject: [PATCH 203/219] add gc.log to log4j2 delete policy --- salt/elasticsearch/files/log4j2.properties | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/log4j2.properties b/salt/elasticsearch/files/log4j2.properties index 050071581..a938b6247 100644 --- a/salt/elasticsearch/files/log4j2.properties +++ b/salt/elasticsearch/files/log4j2.properties @@ -20,7 +20,8 @@ appender.rolling.strategy.type = DefaultRolloverStrategy appender.rolling.strategy.action.type = Delete appender.rolling.strategy.action.basepath = /var/log/elasticsearch appender.rolling.strategy.action.condition.type = IfFileName -appender.rolling.strategy.action.condition.glob = *.log.gz +# age delete regular securityonion.log.gz and gc.log.NN files +appender.rolling.strategy.action.condition.regex = (?:.*[.]log[.]gz|gc[.]log[.][0-9]+) appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified appender.rolling.strategy.action.condition.nested_condition.age = 7D From 01a873b2d917d9257d338821564dc68a82f11d66 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 23 Jul 2026 13:39:19 -0400 Subject: [PATCH 204/219] Map av to both defender and defend --- salt/soc/files/soc/sigma_so_pipeline.yaml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index 281acb8a9..724740673 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -46,20 +46,24 @@ transformations: - type: logsource product: opencanary # Maps "antivirus" category to Windows Defender logs shipped by Elastic Agent Winlog Integration + # and to Elastic Defend malware alerts # winlog.event_data.threat_name has to be renamed prior to ingestion, it is originally winlog.event_data.Threat Name - - id: antivirus_field-mappings_windows-defender + - id: antivirus_field-mappings type: field_name_mapping mapping: - Signature: winlog.event_data.threat_name + Signature: + - winlog.event_data.threat_name + - rule.name rule_conditions: - type: logsource category: antivirus - - id: antivirus_add-fields_windows-defender + - id: antivirus_add-fields type: add_condition conditions: - winlog.channel: 'Microsoft-Windows-Windows Defender/Operational' - winlog.provider_name: 'Microsoft-Windows-Windows Defender' - event.code: "1116" + event.code: + - "1116" + - "malicious_file" + - "memory_signature" rule_conditions: - type: logsource category: antivirus From e2513daddce985be87db4d4d675d80e1b34bbbd5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 23 Jul 2026 14:39:19 -0400 Subject: [PATCH 205/219] Remove icsnpp-modbus from defaults.yaml Removed 'icsnpp-modbus' from the list of modules. --- salt/zeek/defaults.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/zeek/defaults.yaml b/salt/zeek/defaults.yaml index 4058b01b8..c87dfeec5 100644 --- a/salt/zeek/defaults.yaml +++ b/salt/zeek/defaults.yaml @@ -62,7 +62,6 @@ zeek: - securityonion/file-extraction - securityonion/community-id-extended - oui-logging - - icsnpp-modbus - icsnpp-dnp3 - icsnpp-bacnet - icsnpp-ethercat From 6bd3c414bb10fbe04fd409eccdc58e12de2ea3e4 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 23 Jul 2026 15:16:26 -0400 Subject: [PATCH 206/219] Add default esheap value so nodes highstate without the pillar value Previously salt/vars/elasticsearch.map.jinja read the elasticsearch heap size via a raw pillar access (INIT.PILLAR.elasticsearch.esheap). If the esheap key was missing from a minion pillar, building GLOBALS raised 'dict object has no attribute esheap', cascading into every state that imports GLOBALS (elasticsearch.enabled, logstash.config, telegraf.config, etc.) and blocking highstate entirely. Add an esheap default ('600m', matching the es_heapsize() floor) to elasticsearch/defaults.yaml and fall back to it via dict .get() so a missing pillar value degrades gracefully. The per-node pillar value still wins when present. --- salt/elasticsearch/defaults.yaml | 1 + salt/vars/elasticsearch.map.jinja | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 14d753c21..52521d7e8 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,5 +1,6 @@ elasticsearch: enabled: false + esheap: '600m' version: 9.3.7 index_clean: true data_retention_method: DLM diff --git a/salt/vars/elasticsearch.map.jinja b/salt/vars/elasticsearch.map.jinja index 433b10a3e..ce0f5d951 100644 --- a/salt/vars/elasticsearch.map.jinja +++ b/salt/vars/elasticsearch.map.jinja @@ -1,10 +1,11 @@ {% import 'vars/init.map.jinja' as INIT %} +{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} {% set ELASTICSEARCH_GLOBALS = { 'elasticsearch': { 'es_cluster_name': INIT.PILLAR.elasticsearch.config.cluster.name, - 'es_heap': INIT.PILLAR.elasticsearch.esheap + 'es_heap': INIT.PILLAR.elasticsearch.get('esheap', ELASTICSEARCHDEFAULTS.elasticsearch.esheap) } } %} From f2d81cea3f561d52c4c71a6c788552e2a84340d0 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:06:36 -0500 Subject: [PATCH 207/219] call highstate in soup with a retry if first fails vs bailing --- salt/manager/tools/sbin/soup | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index f5b40310e..dd12659aa 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -442,7 +442,13 @@ get_soup_script_hashes() { } highstate() { - # Run a highstate. + # Run a highstate with a retry attempt. + if salt-call state.highstate -l info queue=True; then + return 0 + fi + + echo "Initial highstate attempt had a problem; retrying in 30 seconds." + sleep 30 salt-call state.highstate -l info queue=True } From b109ca4e9b1343dd98b52df27ec75eceb827139e Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 24 Jul 2026 12:34:35 -0400 Subject: [PATCH 208/219] Express the highstate schedule in minutes instead of hours salt.schedule.highstate_interval_hours could not express a sub-hour cadence, so an operator who disables salt.auto_apply had no way back to the legacy 15-minute highstate. Rename the setting to highstate_interval_minutes (default 120, behavior unchanged) and enforce a 15-minute floor in SOC. Non-manager splay is now a quarter of the interval clamped to [5, 30] minutes, so a short interval no longer gets jitter larger than itself; at the 120-minute default it stays 1800s. The so-salt-minion-check restart threshold keeps its interval-plus-one-hour grace, now in minute math. --- salt/common/tools/sbin_jinja/so-salt-minion-check | 2 +- salt/salt/defaults.yaml | 2 +- salt/salt/highstate_schedule.sls | 8 ++++++-- salt/salt/soc_salt.yaml | 6 ++++-- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/salt/common/tools/sbin_jinja/so-salt-minion-check b/salt/common/tools/sbin_jinja/so-salt-minion-check index c78846a5f..353448251 100755 --- a/salt/common/tools/sbin_jinja/so-salt-minion-check +++ b/salt/common/tools/sbin_jinja/so-salt-minion-check @@ -25,7 +25,7 @@ LAST_HIGHSTATE_END=$([ -e "/opt/so/log/salt/lasthighstate" ] && date -r /opt/so/ LAST_HEALTHCHECK_STATE_APPLY=$([ -e "/opt/so/log/salt/state-apply-test" ] && date -r /opt/so/log/salt/state-apply-test +%s || echo 0) # SETTING THRESHOLD TO ANYTHING UNDER 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default # THRESHOLD is derived from the salt schedule highstate interval + 1 hour, so the minion-check grace period tracks the schedule automatically. -THRESHOLD=$(( ({{ SCHEDULEMERGED.highstate_interval_hours }} + 1) * 3600 )) #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted +THRESHOLD=$(( ({{ SCHEDULEMERGED.highstate_interval_minutes }} + 60) * 60 )) #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted THRESHOLD_DATE=$((LAST_HEALTHCHECK_STATE_APPLY+THRESHOLD)) logCmd() { diff --git a/salt/salt/defaults.yaml b/salt/salt/defaults.yaml index 12d0373d5..140c61fcb 100644 --- a/salt/salt/defaults.yaml +++ b/salt/salt/defaults.yaml @@ -6,4 +6,4 @@ salt: batch: '25%' batch_wait: 15 schedule: - highstate_interval_hours: 2 + highstate_interval_minutes: 120 diff --git a/salt/salt/highstate_schedule.sls b/salt/salt/highstate_schedule.sls index ca2797387..5f5ff2fa9 100644 --- a/salt/salt/highstate_schedule.sls +++ b/salt/salt/highstate_schedule.sls @@ -1,11 +1,15 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'salt/schedule.map.jinja' import SCHEDULEMERGED %} +{# splay a quarter of the interval, clamped to [5 min, 30 min], so short intervals + don't get jitter larger than the interval itself #} +{% set SPLAY = [[(SCHEDULEMERGED.highstate_interval_minutes * 60 // 4) | int, 300] | max, 1800] | min %} + highstate_schedule: schedule.present: - function: state.highstate - - hours: {{ SCHEDULEMERGED.highstate_interval_hours }} + - minutes: {{ SCHEDULEMERGED.highstate_interval_minutes }} - maxrunning: 1 {% if not GLOBALS.is_manager %} - - splay: 1800 + - splay: {{ SPLAY }} {% endif %} diff --git a/salt/salt/soc_salt.yaml b/salt/salt/soc_salt.yaml index 0aa0aaf89..f3f2ff93a 100644 --- a/salt/salt/soc_salt.yaml +++ b/salt/salt/soc_salt.yaml @@ -31,9 +31,11 @@ salt: global: True advanced: True schedule: - highstate_interval_hours: - description: How often every minion in the grid runs a scheduled state.highstate, in hours. Lower values keep minions closer in sync at the cost of more load; higher values reduce load but increase worst-case latency for non-pushed changes. The salt-minion health check restarts a minion if its last highstate is older than this value plus one hour. + highstate_interval_minutes: + description: How often every minion in the grid runs a scheduled state.highstate, in minutes. Minimum 15 minutes. Lower values keep minions closer in sync at the cost of more load; higher values reduce load but increase worst-case latency for non-pushed changes. If Auto Apply is disabled, set this to the 15-minute minimum so changes are still picked up promptly. The salt-minion health check restarts a minion if its last state apply is older than this value plus one hour. forcedType: int helpLink: push global: True advanced: True + regex: '^(1[5-9]|[2-9][0-9]|[1-9][0-9]{2,4})$' + regexFailureMessage: The value must be an integer of at least 15 minutes (maximum 99999). From baca444a7e83d16643764e6b1f92f7121a13b470 Mon Sep 17 00:00:00 2001 From: Jorge Reyes <94730068+reyesj2@users.noreply.github.com> Date: Fri, 24 Jul 2026 13:18:48 -0500 Subject: [PATCH 209/219] Revert "elastic fleet server persistence" --- salt/common/tools/sbin/so-restart | 3 --- salt/common/tools/sbin/so-stop | 2 -- salt/elasticfleet/enabled.sls | 10 ---------- 3 files changed, 15 deletions(-) diff --git a/salt/common/tools/sbin/so-restart b/salt/common/tools/sbin/so-restart index b3864d2d1..14747d134 100755 --- a/salt/common/tools/sbin/so-restart +++ b/salt/common/tools/sbin/so-restart @@ -35,9 +35,6 @@ case $1 in "elastic-fleet"|"elasticfleet") docker_check_running "elastic-fleet" "--stop" docker rm "so-elastic-fleet" 2> /dev/null - # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy - rm -rf /opt/so/conf/elastic-fleet/state - salt-call state.apply elasticfleet queue=True ;; *) diff --git a/salt/common/tools/sbin/so-stop b/salt/common/tools/sbin/so-stop index 9218b44f7..d036a7b63 100755 --- a/salt/common/tools/sbin/so-stop +++ b/salt/common/tools/sbin/so-stop @@ -29,8 +29,6 @@ case $1 in "elasticfleet"|"elastic-fleet") docker_check_running "elastic-fleet" "--stop" docker rm "so-elastic-fleet" 2> /dev/null - # Removing the elastic fleet state directory, so that the next startup re-enrolls with a fresh policy - rm -rf /opt/so/conf/elastic-fleet/state ;; *) docker_check_running "$1" "--stop" diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index d9694f07d..feb83fc2f 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -11,10 +11,6 @@ {# This value is generated during node install and stored in minion pillar #} {% set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %} -{# Prevent Elastic Agent from re-enrolling with a new agent.id everytime the container starts up. - - if a fresh enrollment is needed use 'so-stop elasticfleet' - #} -{% set ENROLLED = salt['file.file_exists']('/opt/so/conf/elastic-fleet/state/fleet.enc') %} include: - ca @@ -70,7 +66,6 @@ so-elastic-fleet: - /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro - - /opt/so/conf/elastic-fleet/state:/usr/share/elastic-agent/state - /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs {% if DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %} {% for BIND in DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %} @@ -78,7 +73,6 @@ so-elastic-fleet: {% endfor %} {% endif %} - environment: - {% if not ENROLLED %} - FLEET_SERVER_ENABLE=true - FLEET_URL=https://{{ GLOBALS.hostname }}:8220 - FLEET_SERVER_ELASTICSEARCH_HOST=https://{{ GLOBALS.manager }}:9200 @@ -88,9 +82,6 @@ so-elastic-fleet: - FLEET_SERVER_CERT_KEY=/etc/pki/elasticfleet-server.key - FLEET_CA=/etc/pki/tls/certs/intca.crt - FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt - {% endif %} - - STATE_PATH=/usr/share/elastic-agent/state - - CONFIG_PATH=/usr/share/elastic-agent/state - LOGS_PATH=logs {% if DOCKERMERGED.containers['so-elastic-fleet'].extra_env %} {% for XTRAENV in DOCKERMERGED.containers['so-elastic-fleet'].extra_env %} @@ -109,7 +100,6 @@ so-elastic-fleet: - x509: etc_elasticfleet_crt - require: - file: trusttheca - - file: eastatedir - x509: etc_elasticfleet_key - x509: etc_elasticfleet_crt From d5fddafa6a826bca5bfc81ab463ab6b94d5f6d90 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 24 Jul 2026 13:33:48 -0500 Subject: [PATCH 210/219] run es-url-updae prior to upgrade --- salt/manager/tools/sbin/soup | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index cde6ae10a..30d26124d 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -980,6 +980,10 @@ up_to_3.2.0() { pin_elasticsearch_data_retention_method + # Run so-elastic-fleet-es-url update with --force to ensure eval/import have + # configured so-manager_elasicsearch as the default output for both monitoring and logs + /usr/sbin/so-elastic-fleet-es-url-update --force + INSTALLEDVERSION=3.2.0 } From 120b426a797b75f22711cf4964e64f33c40411d7 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 24 Jul 2026 17:23:00 -0400 Subject: [PATCH 211/219] Add already_running mapping --- .../files/soc/sigma_playbook_pipeline.yaml | 24 +++++++++++++++++++ salt/soc/files/soc/sigma_so_pipeline.yaml | 18 +++++++++++--- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/salt/soc/files/soc/sigma_playbook_pipeline.yaml b/salt/soc/files/soc/sigma_playbook_pipeline.yaml index 04bd2ffaf..a779c4dec 100644 --- a/salt/soc/files/soc/sigma_playbook_pipeline.yaml +++ b/salt/soc/files/soc/sigma_playbook_pipeline.yaml @@ -12,6 +12,30 @@ transformations: process.command_line: process.command_line.caseless process.parent.command_line: process.parent.command_line.caseless file.path: file.path.caseless + # entity_id pivots must also match processes that were already running when the + # agent started: Defend emits already_running (event.type:info), not start. + # Kept out of Playbook sigma query because Sysmon has no equivalent concept. + # contains_field is exact, so child pivots (process.parent.entity_id) stay + # start-only. Drop must precede add; + - id: playbook_process_lifecycle_drop_start_scope + type: drop_detection_item + field_name_conditions: + - type: include_fields + fields: ['event.type'] + rule_conditions: + - type: logsource + category: process_creation + - type: contains_field + field: process.entity_id + - id: playbook_process_lifecycle_add-fields + type: add_condition + conditions: + event.type: ['start', 'info'] + rule_conditions: + - type: logsource + category: process_creation + - type: contains_field + field: process.entity_id # file_activity: playbook-only pseudo-category spanning all file operations. - id: playbook_file_activity_add-fields type: add_condition diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index 724740673..64b691d8f 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -68,13 +68,25 @@ transformations: - type: logsource category: antivirus # OS-agnostic process_creation scoping for product-less (NIDS/host-pivot) rules. + # pySigma: rule_cond_expr requires rule_conditions as a mapping, not a list. - id: process_creation_os_agnostic type: add_condition conditions: event.category: process rule_conditions: - - type: logsource - category: process_creation + pc_cat: + type: logsource + category: process_creation + pc_win: + type: logsource + product: windows + pc_mac: + type: logsource + product: macos + pc_lin: + type: logsource + product: linux + rule_cond_expr: "pc_cat and not (pc_win or pc_mac or pc_lin)" # Transforms the `Hashes` field to ECS fields # ECS fields are used by the hash fields emitted by Elastic Defend # If shipped with Elastic Agent, sysmon logs will also have hashes mapped to ECS fields @@ -630,4 +642,4 @@ transformations: tags: '*file' rule_conditions: - type: logsource - category: file \ No newline at end of file + category: file From 14d11cc180e3bfb0e97072f76e90cd87c5699cde Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 24 Jul 2026 16:59:42 -0500 Subject: [PATCH 212/219] add sslverify=0 to minion repo config. Updates are pulled from the manager hosted repo --- salt/repo/client/oracle.sls | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/repo/client/oracle.sls b/salt/repo/client/oracle.sls index bf0a02751..bb4316879 100644 --- a/salt/repo/client/oracle.sls +++ b/salt/repo/client/oracle.sls @@ -60,6 +60,9 @@ so_repo: {% endif %} - enabled: 1 - gpgcheck: 1 + {% if not GLOBALS.is_manager %} + - sslverify: 0 + {% endif %} # Only assign the kernel repo once this node's running salt matches the version this # SO release ships. During a soup the grid is mid-salt-upgrade; gating here keeps the @@ -77,6 +80,9 @@ so_kernel_repo: {% endif %} - enabled: 1 - gpgcheck: 1 + {% if not GLOBALS.is_manager %} + - sslverify: 0 + {% endif %} # Supplementary kernel repo: tolerate it being empty/unreachable (e.g. before the # manager has populated /nsm/kernelrepo) so a missing repomd.xml can't make every # dnf/pkg operation on the grid fail. From 7f64f143d7f7b9dcc7c811c10e6d2dc989437172 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:25:51 -0500 Subject: [PATCH 213/219] ignore all TransformTask failures for so_kibana user --- salt/common/tools/sbin/so-log-check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 56c0278da..b1a64770d 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -231,7 +231,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|from NIC checksum offloading" # zeek reporter.log EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal" # docker container getting recycled EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459 - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-(tychon|aws_billing|microsoft_defender_endpoint|armis|o365_metrics|microsoft_sentinel|snyk|cyera|island_browser).*user so_kibana lacks the required permissions \[(logs|metrics)-\1" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user. This error should not be seen on fresh ES 9.3.3 installs or after SO 3.1.0 with soups addition of check_transform_health_and_reauthorize() + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown" # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1 fi From 57d629683d8cd8b2391bc264d45b66802fb7d362 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 27 Jul 2026 10:14:10 -0400 Subject: [PATCH 214/219] update vm state applid for pillar_push_map --- salt/reactor/pillar_push_map.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/reactor/pillar_push_map.yaml b/salt/reactor/pillar_push_map.yaml index c77f5e93c..993444bd2 100644 --- a/salt/reactor/pillar_push_map.yaml +++ b/salt/reactor/pillar_push_map.yaml @@ -242,7 +242,7 @@ versionlock: # grain (compound supports nested grain matching via G@::). # pillar/vm/soc_vm.sls write path is referenced at salt/_runners/setup_hypervisor.py:856. vm: - - state: vm + - state: vm.user tgt: 'G@salt-cloud:driver:libvirt' # zeek: sensor_roles + so-import (5 roles). From 812310088e415e889f459b8a12ebce44aa388a72 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 27 Jul 2026 10:30:50 -0400 Subject: [PATCH 215/219] Support sigma playbooks for airgap --- salt/soc/defaults.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index c89b39dcf..2553f17e7 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1512,6 +1512,10 @@ soc: rulesetName: sos-resources-ag branch: main folder: securityonion-normalized + - repo: file:///nsm/airgap-resources/playbooks/securityonion-resources-playbooks + rulesetName: sos-published-ag + branch: published + folder: sigma assistant: systemPromptAddendum: "" systemPromptAddendumMaxLength: 50000 From 4de8f0208fb1e3212d894d68a9fafd127fa980e1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 27 Jul 2026 12:15:21 -0400 Subject: [PATCH 216/219] Add Gemma configuration to defaults.yaml --- salt/soc/defaults.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 2553f17e7..848e6bc73 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -2718,4 +2718,14 @@ soc: enabled: true adapter: SOAI charsPerTokenEstimate: 4 + - id: gemma + displayName: Gemma + origin: USA + contextLimitSmall: 256000 + contextLimitLarge: 256000 + lowBalanceColorAlert: 500000 + enabled: true + adapter: SOAI + charsPerTokenEstimate: 4 + From a45ca1207685827161569328d1c4ce43935be60d Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 28 Jul 2026 10:19:18 -0600 Subject: [PATCH 217/219] Change defaults We're no longer using DisplayName as the model identifier. Refactored to use id@adapter. --- salt/soc/defaults.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 848e6bc73..d7ac5ac9e 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1534,9 +1534,9 @@ soc: healthTimeoutSeconds: 5 agentic: false agentMapping: - Orchestrator: Claude Sonnet - Investigator: Claude Sonnet - DetectionEngineer: Claude Sonnet + Orchestrator: sonnet@SOAI + Investigator: sonnet@SOAI + DetectionEngineer: sonnet@SOAI onionconfig: saltstackDir: /opt/so/saltstack bypassEnabled: false From d94c16eea1537ec523cd46f0170d6a870969f6bc Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 28 Jul 2026 10:26:29 -0600 Subject: [PATCH 218/219] Investigator and Engineer Use Gemma --- salt/soc/defaults.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index d7ac5ac9e..75fc1a597 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1535,8 +1535,8 @@ soc: agentic: false agentMapping: Orchestrator: sonnet@SOAI - Investigator: sonnet@SOAI - DetectionEngineer: sonnet@SOAI + Investigator: gemma@SOAI + DetectionEngineer: gemma@SOAI onionconfig: saltstackDir: /opt/so/saltstack bypassEnabled: false From c9642489d33160b4b7e23a14539373e315f423bf Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 29 Jul 2026 13:48:13 -0400 Subject: [PATCH 219/219] 3.2.0 --- DOWNLOAD_AND_VERIFY_ISO.md | 22 +++++++++++----------- sigs/securityonion-3.2.0-20260729.iso.sig | Bin 0 -> 566 bytes 2 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 sigs/securityonion-3.2.0-20260729.iso.sig diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index bae49c4ac..f3d496b41 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,17 +1,17 @@ -### 3.1.0-20260528 ISO image released on 2026/05/28 +### 3.2.0-20260729 ISO image released on 2026/07/29 ### Download and Verify -3.1.0-20260528 ISO image: -https://download.securityonion.net/file/securityonion/securityonion-3.1.0-20260528.iso +3.2.0-20260729 ISO image: +https://download.securityonion.net/file/securityonion/securityonion-3.2.0-20260729.iso -MD5: 9D6FF58DEEE24089D722C73169765B3E -SHA1: 2B8B816B6CEC3B7F96B3C5E040EBF502DD2C412F -SHA256: 62FAB57E247C843D6A04F0796D8162C732B65D82FC3E4A59D087135B9FD32912 +MD5: B1E10F46DF872B655C29325DF965A4DB +SHA1: 0F3C7ED80F6D326B7A993C2F899B986320C01BF9 +SHA256: 7465163C1D1ADFCDC3935530EAFB312E987C016941ADC11841B214553314D1FF Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.1.0-20260528.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.2.0-20260729.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS @@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/ Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.1.0-20260528.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.2.0-20260729.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-3.1.0-20260528.iso +wget https://download.securityonion.net/file/securityonion/securityonion-3.2.0-20260729.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-3.1.0-20260528.iso.sig securityonion-3.1.0-20260528.iso +gpg --verify securityonion-3.2.0-20260729.iso.sig securityonion-3.2.0-20260729.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Wed 27 May 2026 03:03:59 PM EDT using RSA key ID FE507013 +gpg: Signature made Tue 28 Jul 2026 06:17:34 PM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-3.2.0-20260729.iso.sig b/sigs/securityonion-3.2.0-20260729.iso.sig new file mode 100644 index 0000000000000000000000000000000000000000..f9324a9fdbd3b5a58ad6d1b5e838d966f1e59884 GIT binary patch literal 566 zcmV-60?GY}0y6{v0SEvc79j-41gSkXz6^6dp_W8^5Ma0dP;e6k0%~b0egFyy5PT3| zxBgIY6E_kM|3$vbpQ|$DK>6kVaW692ck*&lGV?!{{6fspIkw4LbLaFt^xbRsa;iy3 zxd=S4}5F5cWm#+DjfI(M{8gL3UOH{W6p#EQJo z75yyre@YC@?=Teeb=(WOaHu4O2RmQ-od!4*v>joKxzGUm_s))=4|wN< z$Qu6kKv!=CJ3K?ycO!>)Xw;LG>4+h7b8N7s=v$Dm3Rz@2di{$pqOX6_{|31rmV^bF z0Ida8;KQP(aXgI<9^=zVUOA;&cIU$=vlu`K#F>%?M3UJuYYB6RVVjf5#gQCskRlU- zBFNK@o;f`yB+(UX2FT>!n0s{@xiUFf2hQ6qdGa^4iE~?Na2OQBDfc|$g*;xDL`;>s zg!H*fc0z-9jMZ+9N8@MweuUZPiu3+a|KU$Btz1=*n+SLIJAb~(IPYHMqe`e^mV>Ql E-rN@rj{pDw literal 0 HcmV?d00001