ver

salt/_beacons/pillar_db.py

@@ -1,142 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Custom salt beacon that watches the SOC audit_settings table in postgres for
-# new settings changes and emits a beacon event per new row. This replaces the
-# inotify watch on /opt/so/saltstack/local/pillar -- instead of monitoring pillar
-# files on disk, we monitor the so_soc.audit_settings table that SOC writes to.
-#
-# Detection is poll-based with a monotonic `id` watermark persisted to
-# WATERMARK_FILE: each pass selects rows with id greater than the last id seen,
-# which makes it self-healing (a missed poll simply catches up on the next one).
-#
-# Each emitted event carries setting_id and node_id; the push_pillar reactor maps
-# setting_id -> app via pillar_push_map.yaml and writes a push intent, after which
-# the existing so-push-drainer / orch.push_batch pipeline takes over unchanged.
-
-import logging
-import os
-import subprocess
-
-log = logging.getLogger(__name__)
-
-WATERMARK_FILE = '/opt/so/state/pillar_db_watch.id'
-CONTAINER = 'so-postgres'
-DATABASE = 'so_soc'
-
-# Unaligned, tuples-only psql output with a field separator that cannot appear in
-# an id/setting_id/node_id, so we can split each row reliably.
-FIELD_SEP = '\x1f'
-
-
-def __virtual__():
-    return True
-
-
-def validate(config):
-    return True, 'valid'
-
-
-def _read_watermark():
-    # Returns the last processed id, or None if the watermark has not been seeded.
-    try:
-        with open(WATERMARK_FILE, 'r') as f:
-            return int((f.read() or '').strip())
-    except (IOError, ValueError):
-        return None
-
-
-def _write_watermark(value):
-    try:
-        os.makedirs(os.path.dirname(WATERMARK_FILE), exist_ok=True)
-        tmp = WATERMARK_FILE + '.tmp'
-        with open(tmp, 'w') as f:
-            f.write(str(int(value)))
-        os.rename(tmp, WATERMARK_FILE)
-    except OSError:
-        log.exception('pillar_db beacon: failed to persist watermark to %s', WATERMARK_FILE)
-
-
-def _query(sql):
-    # Run a query against so_soc inside the so-postgres container over the unix
-    # socket (trust auth, no password). Returns stdout on success, or None on any
-    # failure so the caller can no-op and retry on the next interval.
-    cmd = [
-        'docker', 'exec', CONTAINER,
-        'psql', '-U', 'postgres', '-d', DATABASE,
-        '-tA', '-F', FIELD_SEP, '-c', sql,
-    ]
-    try:
-        result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
-    except subprocess.TimeoutExpired:
-        log.warning('pillar_db beacon: psql timed out')
-        return None
-    except Exception:
-        log.exception('pillar_db beacon: failed to exec psql')
-        return None
-    if result.returncode != 0:
-        log.warning('pillar_db beacon: psql failed (rc=%s): %s',
-                    result.returncode, (result.stderr or '').strip())
-        return None
-    return result.stdout
-
-
-def beacon(config):
-    retval = []
-
-    watermark = _read_watermark()
-
-    # First run / missing watermark: seed to the current MAX(id) and emit nothing
-    # so we never replay the entire settings history into a fleetwide push.
-    if watermark is None:
-        seed = _query('SELECT COALESCE(MAX(id), 0) FROM audit_settings;')
-        if seed is None:
-            return retval  # postgres not ready yet; retry next interval
-        try:
-            _write_watermark(int((seed or '0').strip() or 0))
-        except ValueError:
-            log.warning('pillar_db beacon: could not parse MAX(id) seed: %r', seed)
-        return retval
-
-    rows = _query(
-        "SELECT id, setting_id, COALESCE(node_id, '') FROM audit_settings "
-        "WHERE id > %d ORDER BY id;" % watermark
-    )
-    if rows is None:
-        return retval
-
-    max_id = watermark
-    for line in rows.splitlines():
-        # Do NOT str.strip() the whole line: Python treats the \x1f field
-        # separator (and \x1c-\x1e) as whitespace, so stripping would eat an
-        # empty trailing node_id field and make the row look malformed.
-        if not line.strip():
-            continue
-        parts = line.split(FIELD_SEP)
-        if len(parts) < 3:
-            log.warning('pillar_db beacon: skipping malformed row: %r', line)
-            continue
-        try:
-            row_id = int(parts[0])
-        except ValueError:
-            log.warning('pillar_db beacon: skipping row with non-int id: %r', line)
-            continue
-        setting_id = parts[1]
-        node_id = parts[2]
-        retval.append({
-            'tag': 'audit_settings',
-            'id': row_id,
-            'setting_id': setting_id,
-            'node_id': node_id,
-        })
-        if row_id > max_id:
-            max_id = row_id
-
-    if max_id > watermark:
-        _write_watermark(max_id)
-        log.info('pillar_db beacon: emitted %d change(s), watermark %d -> %d',
-                 len(retval), watermark, max_id)
-
-    return retval

salt/common/tools/sbin_jinja/so-salt-minion-check

@@ -1,3 +1,5 @@
+{% import_yaml 'salt/minion.defaults.yaml' as SALT_MINION_DEFAULTS -%}
+
 #!/bin/bash
 #
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
@@ -23,8 +25,7 @@ SYSTEM_START_TIME=$(date -d "$(</proc/uptime awk '{print $1}') seconds ago" +%s)
 LAST_HIGHSTATE_END=$([ -e "/opt/so/log/salt/lasthighstate" ] && date -r /opt/so/log/salt/lasthighstate +%s || echo 0)
 LAST_HEALTHCHECK_STATE_APPLY=$([ -e "/opt/so/log/salt/state-apply-test" ] && date -r /opt/so/log/salt/state-apply-test +%s || echo 0)
 # SETTING THRESHOLD TO ANYTHING UNDER 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default
-# THRESHOLD is derived from the global push highstate interval + 1 hour, so the minion-check grace period tracks the schedule automatically.
-THRESHOLD=$(( ({{ salt['pillar.get']('global:push:highstate_interval_hours', 2) }} + 1) * 3600 )) #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted
+THRESHOLD={{SALT_MINION_DEFAULTS.salt.minion.check_threshold}} #within how many seconds the file /opt/so/log/salt/state-apply-test must have been touched/modified before the salt minion is restarted
 THRESHOLD_DATE=$((LAST_HEALTHCHECK_STATE_APPLY+THRESHOLD))
 
 logCmd() {

salt/docker_clean/init.sls

@@ -9,8 +9,7 @@
 prune_images:
   cmd.run:
     - name: so-docker-prune
-    - onlyif: command -v /usr/sbin/so-docker-prune >/dev/null 2>&1
-    - order: 9000
+    - order: last
 
 {% else %}
 

salt/elastalert/enabled.sls

@@ -19,7 +19,6 @@ wait_for_elasticsearch:
 so-elastalert:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastalert:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: elastalert
     - name: so-elastalert
     - user: so-elastalert

salt/elastic-fleet-package-registry/enabled.sls

@@ -15,7 +15,6 @@ include:
 so-elastic-fleet-package-registry:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-fleet-package-registry:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - name: so-elastic-fleet-package-registry
     - hostname: Fleet-package-reg-{{ GLOBALS.hostname }}
     - detach: True

salt/elasticagent/enabled.sls

@@ -16,7 +16,6 @@ include:
 so-elastic-agent:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - name: so-elastic-agent
     - hostname: {{ GLOBALS.hostname }}
     - detach: True

salt/elasticfleet/enabled.sls

@@ -42,7 +42,6 @@ elasticagent_syncartifacts:
 so-elastic-fleet:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - name: so-elastic-fleet
     - hostname: FleetServer-{{ GLOBALS.hostname }}
     - detach: True

salt/elasticfleet/manager.sls

@@ -11,15 +11,14 @@ include:
   - elasticfleet.config
 
 # If enabled, automatically update Fleet Logstash Outputs
-{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration %}
-{%   if grains.role not in ['so-import', 'so-eval']%}
+{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval'] %}
 so-elastic-fleet-auto-configure-logstash-outputs:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-outputs-update
     - retry:
         attempts: 4
         interval: 30
-{%   endif %}
+{% endif %}
 
 # If enabled, automatically update Fleet Server URLs & ES Connection
 so-elastic-fleet-auto-configure-server-urls:
@@ -28,7 +27,6 @@ so-elastic-fleet-auto-configure-server-urls:
     - retry:
         attempts: 4
         interval: 30
-{% endif %}
 
 # Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
 so-elastic-fleet-auto-configure-elasticsearch-urls:

salt/elasticsearch/cluster.sls

@@ -9,12 +9,9 @@
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %}
 {%   if GLOBALS.role != 'so-heavynode' %}
-{%     from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS, ADDON_INDICES %}
+{%     from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
 {%   endif %}
 
-include:
-  - elasticsearch.enabled
-
 escomponenttemplates:
   file.recurse:
     - name: /opt/so/conf/elasticsearch/templates/component
@@ -38,20 +35,6 @@ so_index_template_dir:
       {%- endfor %}
     {%- endif %}
 
-{%  if GLOBALS.role != "so-heavynode" %}
-# Clean up legacy and non-SO managed templates from the elasticsearch/templates/addon-index/ directory
-addon_index_template_dir:
-  file.directory:
-    - name: /opt/so/conf/elasticsearch/templates/addon-index
-    - clean: True
-    {%- if ADDON_INDICES %}
-    - require:
-      {%- for index in ADDON_INDICES %}
-      - file: addon_index_template_{{index}}
-      {%- endfor %}
-    {%- endif %}
-{%  endif %}
-
 # Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml)
 #   These index templates are for the core SO datasets and are always required
 {%  for index, settings in ES_INDEX_SETTINGS.items() %}

salt/elasticsearch/enabled.sls

@@ -24,7 +24,6 @@ include:
 so-elasticsearch:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:{{ ELASTICSEARCHMERGED.version }}
-    - restart_policy: unless-stopped
     - hostname: elasticsearch
     - name: so-elasticsearch
     - user: elasticsearch

salt/elasticsearch/template.map.jinja

@@ -61,25 +61,15 @@
 {% if ALL_ADDON_SETTINGS_ORIG.keys() | length > 0 %}
 {%   for index in ALL_ADDON_SETTINGS_ORIG.keys() %}
 {%     do ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ALL_ADDON_SETTINGS_ORIG[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}
-{#     Explicitly excluding addon indices from ES_INDEX_SETTINGS_ORIG
-         When manager.soc_managed_annotations runs, new entries are added to the salt/elasticsearch/defaults.yaml file to support 'revert to default' functionality.
-         Subsequent map renders will then incorrectly include 'integration X' in 'ES_INDEX_SETTINGS_ORIG' due to being in the defaults.yaml file. #}
-{%     if index in ES_INDEX_SETTINGS_ORIG.keys() %}
-{%       do ES_INDEX_SETTINGS_ORIG.pop(index) %}
-{%     endif %}
 {%   endfor %}
 {% endif %}
 
 {% set ES_INDEX_SETTINGS = {} %}
-{% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS, EXCLUDE_INDICES=[]) %}
+{% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS) %}
 
 {% do GLOBAL_OVERRIDES.update(salt['defaults.merge'](GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
 {% for index, settings in GLOBAL_OVERRIDES.items() %}
 
-{%   if index in EXCLUDE_INDICES %}
-{%     continue %}
-{%   endif %}
-
 {#   prevent this action from being performed on custom defined indices. #}
 {#   the custom defined index is not present in either of the dictionaries and fails to reder. #}
 {%   if index in DEFINED_SETTINGS and index in GLOBAL_OVERRIDES %}
@@ -160,19 +150,10 @@
 {% endfor %}
 {% endmacro %}
 
-{# Exclude addon integrations from final ES_INDEX_SETTINGS #}
-{{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS, ALL_ADDON_SETTINGS_ORIG.keys() | list ) }}
-
-{# Exclude SO managed indices, otherwise ALL_ADDON_SETTINGS will include pillar values
-  of core integrations without merging defaults, resulting in an overlapping, but bad index template being generated. #}
-{{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS, ES_INDEX_SETTINGS_ORIG.keys() | list ) }}
+{{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS) }}
+{{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS) }}
 
 {% set SO_MANAGED_INDICES = [] %}
 {% for index, settings in ES_INDEX_SETTINGS.items() %}
 {%   do SO_MANAGED_INDICES.append(index) %}
-{% endfor %}
-
-{% set ADDON_INDICES = [] %}
-{% for index, settings in ALL_ADDON_SETTINGS.items() %}
-{%   do ADDON_INDICES.append(index) %}
-{% endfor %}
+{% endfor %}

salt/global/defaults.yaml

@@ -1,10 +1,3 @@
 global:
   pcapengine: SURICATA
-  pipeline: REDIS
-  push:
-    enabled: true
-    highstate_interval_hours: 2
-    debounce_seconds: 30
-    drain_interval: 15
-    batch: '25%'
-    batch_wait: 15
+  pipeline: REDIS

salt/global/soc_global.yaml

@@ -59,41 +59,4 @@ global:
     description: Allows use of Endgame with Security Onion. This feature requires a license from Endgame.
     global: True
     advanced: True
-  push:
-    enabled:
-      description: Master kill-switch for the active push feature. When disabled, rule and pillar changes are picked up at the next scheduled highstate instead of being pushed immediately.
-      forcedType: bool
-      helpLink: push
-      global: True
-    highstate_interval_hours:
-      description: How often every minion in the grid runs a scheduled state.highstate, in hours. Lower values keep minions closer in sync at the cost of more load; higher values reduce load but increase worst-case latency for non-pushed changes. The salt-minion health check restarts a minion if its last highstate is older than this value plus one hour.
-      forcedType: int
-      helpLink: push
-      global: True
-      advanced: True
-    debounce_seconds:
-      description: Trailing-edge debounce window in seconds. A push intent must be quiet for this long before the drainer dispatches. Rapid bursts of edits within this window coalesce into one dispatch.
-      forcedType: int
-      helpLink: push
-      global: True
-      advanced: True
-    drain_interval:
-      description: How often the push drainer checks for ready intents, in seconds. Small values lower dispatch latency at the cost of more background work on the manager.
-      forcedType: int
-      helpLink: push
-      global: True
-      advanced: True
-    batch:
-      description: "Host batch size for push orchestrations. A number (e.g. '10') or a percentage (e.g. '25%'). Limits how many minions run the push state at once so large fleets don't thundering-herd."
-      helpLink: push
-      global: True
-      advanced: True
-      regex: '^([0-9]+%?)$'
-      regexFailureMessage: Enter a whole number or a whole-number percentage (e.g. 10 or 25%).
-    batch_wait:
-      description: Seconds to wait between host batches in a push orchestration. Gives the fleet time to breathe between waves.
-      forcedType: int
-      helpLink: push
-      global: True
-      advanced: True
 

salt/hydra/enabled.sls

@@ -58,7 +58,6 @@ so-hydra:
       - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
     {%   endfor %}
     {% endif %}
-    # Intentionally unless-stopped -- matches the fleet default.
     - restart_policy: unless-stopped
     - watch:
       - file: hydraconfig

salt/idh/enabled.sls

@@ -15,7 +15,6 @@ include:
 so-idh:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-idh:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - name: so-idh
     - detach: True
     - network_mode: host

salt/influxdb/enabled.sls

@@ -18,7 +18,6 @@ include:
 so-influxdb:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-influxdb:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: influxdb
     - networks:
       - sobridge:

salt/kafka/enabled.sls

@@ -27,7 +27,6 @@ include:
 so-kafka:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kafka:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: so-kafka
     - name: so-kafka
     - networks:

salt/kibana/enabled.sls

@@ -16,7 +16,6 @@ include:
 so-kibana:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kibana:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: kibana
     - user: kibana
     - networks:

salt/kratos/enabled.sls

@@ -51,7 +51,6 @@ so-kratos:
       - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
     {%   endfor %}
     {% endif %}
-    # Intentionally unless-stopped -- matches the fleet default.
     - restart_policy: unless-stopped
     - watch:
       - file: kratosschema

salt/logstash/enabled.sls

@@ -28,7 +28,6 @@ include:
 so-logstash:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-logstash:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: so-logstash
     - name: so-logstash
     - networks:

salt/manager/beacons.sls

@@ -1,21 +0,0 @@
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-{% from 'global/map.jinja' import GLOBALMERGED %}
-
-include:
-  - salt.minion
-
-{% if GLOBALS.is_manager and GLOBALMERGED.push.enabled %}
-salt_beacons_pushstate:
-  file.managed:
-    - name: /etc/salt/minion.d/beacons_pushstate.conf
-    - source: salt://manager/files/beacons_pushstate.conf.jinja
-    - template: jinja
-    - watch_in:
-      - service: salt_minion_service
-{% else %}
-salt_beacons_pushstate:
-  file.absent:
-    - name: /etc/salt/minion.d/beacons_pushstate.conf
-    - watch_in:
-      - service: salt_minion_service
-{% endif %}

salt/manager/files/beacons_pushstate.conf.jinja

@@ -1,41 +0,0 @@
-{% from 'global/map.jinja' import GLOBALMERGED %}
-beacons:
-  pillar_db:
-    - interval: {{ GLOBALMERGED.push.drain_interval }}
-    - disable_during_state_run: True
-  inotify:
-    - disable_during_state_run: True
-    - coalesce: True
-    - files:
-        /opt/so/saltstack/local/salt/suricata/rules:
-          mask:
-            - close_write
-            - moved_to
-            - delete
-          recurse: True
-          auto_add: True
-          exclude:
-            - '\.sw[a-z]$':
-                regex: True
-            - '~$':
-                regex: True
-            - '/4913$':
-                regex: True
-            - '/\.#':
-                regex: True
-        /opt/so/saltstack/local/salt/strelka/rules/compiled:
-          mask:
-            - close_write
-            - moved_to
-            - delete
-          recurse: True
-          auto_add: True
-          exclude:
-            - '\.sw[a-z]$':
-                regex: True
-            - '~$':
-                regex: True
-            - '/4913$':
-                regex: True
-            - '/\.#':
-                regex: True

salt/manager/init.sls

@@ -15,7 +15,6 @@ include:
   - manager.elasticsearch
   - manager.kibana
   - manager.managed_soc_annotations
-  - manager.beacons
 
 repo_log_dir:
   file.directory:
@@ -232,7 +231,6 @@ surifiltersrules:
     - user: 939
     - group: 939
 
-
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/manager/sync_es_users.sls

@@ -31,13 +31,11 @@ sync_es_users:
       - http: wait_for_kratos
       - file: so-user.lock # require so-user.lock file to be missing
 
-# we dont want this added too early in setup, so the onlyif gates on the
-# /opt/so/state/setup-complete marker. The marker is written by
-# mark_setup_complete in setup/so-functions just before the final setup
-# highstate (and by an upgrade-path state for systems set up under the old gate).
+# we dont want this added too early in setup, so we add the onlyif to verify 'startup_states: highstate'
+# is in the minion config. That line is added before the final highstate during setup
 so-user_sync:
   cron.present:
     - user: root
     - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log'
     - identifier: so-user_sync
-    - onlyif: "test -e /opt/so/state/setup-complete"
+    - onlyif: "grep -x 'startup_states: highstate' /etc/salt/minion"

salt/manager/tools/sbin/so-boot-mine-update

@@ -1,117 +0,0 @@
-#!/bin/bash
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Runs once per boot on managers (via so-boot-mine-update.service), before
-# so-boot-highstate.service. Waits for the responsive minion set to settle, pushes
-# mine.update, waits until every up minion has actually reported to the mine, then
-# warms the master's per-minion pillar cache so the mine-backed node pillars (node
-# IPs, ES/Redis/Logstash/hypervisor discovery -- some glob- and some pillar/grain-
-# targeted) are complete before the boot highstate renders them. Otherwise a node
-# that is up but not yet fully reported gets dropped from those pillars and torn
-# out of the configs they build (e.g. so-elasticsearch ExtraHosts -> container recreate).
-
-MAX_WAIT=${MINE_UPDATE_MAX_WAIT:-180}   # hard backstop only
-INTERVAL=10
-STABLE_CHECKS=3                          # up-count must hold steady this many polls
-elapsed=0
-prev=-1
-stable=0
-up=0
-
-# Wait for the *reachable* minion set to settle rather than for every accepted
-# key to report up: an operator may accept a minion's key and then intentionally
-# power off that host, so requiring up >= accepted would never be satisfied and
-# we'd always burn the full MAX_WAIT. Once the responsive count stops growing we
-# stop waiting and run mine.update against whoever is up.
-while [ "$elapsed" -lt "$MAX_WAIT" ]; do
-  up=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null \
-    | python3 -c 'import sys,json; print(len(json.load(sys.stdin)))' 2>/dev/null)
-  up=${up:-0}
-  if [ "$up" -gt 0 ] && [ "$up" -eq "$prev" ]; then
-    stable=$((stable + 1))
-    [ "$stable" -ge "$STABLE_CHECKS" ] && break
-  else
-    stable=0
-  fi
-  prev=$up
-  sleep "$INTERVAL"
-  elapsed=$((elapsed + INTERVAL))
-done
-
-echo "so-boot-mine-update: ${up} minions up (settled after ${elapsed}s); running mine.update"
-/usr/bin/salt '*' mine.update --out=txt
-
-# A node that is up but has not yet re-reported network.ip_addrs to the mine is
-# silently dropped from mine-backed pillars (elasticsearch:nodes, node_data, ...)
-# when highstate recompiles them -- which e.g. removes it from so-elasticsearch
-# ExtraHosts and forces a container recreate. After the broad mine.update above,
-# wait until every up minion actually has network.ip_addrs in the mine, re-pushing
-# mine.update to stragglers, before releasing the boot highstate. Bounded by the
-# same MAX_WAIT backstop so a slow/down node never blocks boot indefinitely.
-missing=""
-while [ "$elapsed" -lt "$MAX_WAIT" ]; do
-  up_json=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null)
-  mine_json=$(/usr/bin/salt-run mine.get '*' network.ip_addrs tgt_type=glob --out=json 2>/dev/null)
-  missing=$(printf '%s' "$up_json" | python3 -c '
-import sys, json
-up = set(json.load(sys.stdin) or [])
-mine = {k for k, v in (json.loads(sys.argv[1]) or {}).items() if v}
-print("\n".join(sorted(up - mine)))
-' "$mine_json" 2>/dev/null)
-  if [ -z "$missing" ]; then
-    echo "so-boot-mine-update: mine complete for all up minions after ${elapsed}s"
-    break
-  fi
-  echo "so-boot-mine-update: mine missing up minion(s): $(echo $missing); re-running mine.update"
-  for m in $missing; do /usr/bin/salt "$m" mine.update --out=txt; done
-  sleep "$INTERVAL"
-  elapsed=$((elapsed + INTERVAL))
-done
-[ -n "$missing" ] && echo "so-boot-mine-update: WARNING ${MAX_WAIT}s backstop hit; up minion(s) still absent from mine: $(echo $missing); highstate may drop them from configs"
-
-# The pillar/compound-targeted node pillars (elasticsearch:nodes, redis:nodes,
-# logstash:nodes, hypervisor:nodes) resolve their target against the master's
-# per-minion data cache (grains+pillar in .../minions/<id>/data.p), populated only
-# when a minion's pillar is (re)compiled -- separately from the mine. A freshly
-# booted node can be in the mine (glob/node_data sees it) yet absent from that
-# cache, so it is dropped from those pillars and from the configs they build (e.g.
-# so-elasticsearch ExtraHosts). Force a synchronous pillar refresh so the master
-# caches every up node's pillar; refresh_pillar wait=True returns only once the
-# pillar is recompiled (and thus cached for matching). Retry stragglers <= MAX_WAIT.
-echo "so-boot-mine-update: warming master pillar cache for pillar/grain-targeted node pillars"
-/usr/bin/salt '*' saltutil.refresh_pillar wait=True --out=txt
-missing=""
-while [ "$elapsed" -lt "$MAX_WAIT" ]; do
-  up_json=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null)
-  cached_json=$(/usr/bin/salt-run cache.pillar tgt='*' --out=json 2>/dev/null)
-  missing=$(printf '%s' "$up_json" | python3 -c '
-import sys, json
-up = set(json.load(sys.stdin) or [])
-cached = {k for k, v in (json.loads(sys.argv[1]) or {}).items() if v}
-print("\n".join(sorted(up - cached)))
-' "$cached_json" 2>/dev/null)
-  if [ -z "$missing" ]; then
-    echo "so-boot-mine-update: pillar cache warm for all up minions after ${elapsed}s"
-    break
-  fi
-  echo "so-boot-mine-update: pillar not yet cached for: $(echo $missing); refreshing"
-  for m in $missing; do /usr/bin/salt "$m" saltutil.refresh_pillar wait=True --out=txt; done
-  sleep "$INTERVAL"
-  elapsed=$((elapsed + INTERVAL))
-done
-[ -n "$missing" ] && echo "so-boot-mine-update: WARNING ${MAX_WAIT}s backstop hit; pillar not cached for: $(echo $missing); pillar-targeted pillars may drop them"
-
-# Log what the mine-backed pillars render so the boot-time state is inspectable.
-/usr/bin/salt-call saltutil.refresh_pillar >/dev/null 2>&1
-sleep 2
-for key in node_data elasticsearch:nodes; do
-  rendered=$(/usr/bin/salt-call --out=json pillar.get "$key" 2>/dev/null \
-    | python3 -c 'import sys,json; print(json.dumps(json.load(sys.stdin).get("local"), indent=2, sort_keys=True))' 2>/dev/null)
-  echo "so-boot-mine-update: ${key} rendered as:"
-  echo "${rendered:-null}"
-done
-exit 0

salt/manager/tools/sbin/so-push-drainer

@@ -1,232 +0,0 @@
-#!/opt/saltstack/salt/bin/python3
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-"""
-so-push-drainer
-===============
-
-Scheduled drainer for the active-push feature. Runs on the manager every
-drain_interval seconds (default 15) via a salt schedule in salt/schedule.sls.
-
-For each intent file under /opt/so/state/push_pending/*.json whose last_touch
-is older than debounce_seconds, this script:
-  * concatenates the actions lists from every ready intent
-  * dedupes by (state or __highstate__, tgt, tgt_type)
-  * dispatches a single `salt-run state.orchestrate orch.push_batch --async`
-    with the deduped actions list passed as pillar kwargs
-  * deletes the contributed intent files on successful dispatch
-
-Reactor sls files (push_suricata, push_strelka, push_pillar) write intents
-but never dispatch directly -- see plan
-/home/mreeves/.claude/plans/goofy-marinating-hummingbird.md for the full design.
-"""
-
-import fcntl
-import glob
-import json
-import logging
-import logging.handlers
-import os
-import subprocess
-import sys
-import time
-
-import salt.client
-
-PENDING_DIR = '/opt/so/state/push_pending'
-LOCK_FILE = os.path.join(PENDING_DIR, '.lock')
-LOG_FILE = '/opt/so/log/salt/so-push-drainer.log'
-
-HIGHSTATE_SENTINEL = '__highstate__'
-
-
-def _make_logger():
-    logger = logging.getLogger('so-push-drainer')
-    logger.setLevel(logging.INFO)
-    if not logger.handlers:
-        os.makedirs(os.path.dirname(LOG_FILE), exist_ok=True)
-        handler = logging.handlers.RotatingFileHandler(
-            LOG_FILE, maxBytes=5 * 1024 * 1024, backupCount=3,
-        )
-        handler.setFormatter(logging.Formatter(
-            '%(asctime)s | %(levelname)s | %(message)s',
-        ))
-        logger.addHandler(handler)
-    return logger
-
-
-def _load_push_cfg():
-    """Read the global:push pillar subtree via salt-call. Returns a dict."""
-    caller = salt.client.Caller()
-    cfg = caller.cmd('pillar.get', 'global:push', {})
-    return cfg if isinstance(cfg, dict) else {}
-
-
-def _read_intent(path, log):
-    try:
-        with open(path, 'r') as f:
-            return json.load(f)
-    except (IOError, ValueError) as exc:
-        log.warning('cannot read intent %s: %s', path, exc)
-        return None
-    except Exception:
-        log.exception('unexpected error reading %s', path)
-        return None
-
-
-def _dedupe_actions(actions):
-    seen = set()
-    deduped = []
-    for action in actions:
-        if not isinstance(action, dict):
-            continue
-        state_key = HIGHSTATE_SENTINEL if action.get('highstate') else action.get('state')
-        tgt = action.get('tgt')
-        tgt_type = action.get('tgt_type', 'compound')
-        if not state_key or not tgt:
-            continue
-        key = (state_key, tgt, tgt_type)
-        if key in seen:
-            continue
-        seen.add(key)
-        deduped.append(action)
-    return deduped
-
-
-def _dispatch(actions, log):
-    pillar_arg = json.dumps({'actions': actions})
-    cmd = [
-        'salt-run',
-        'state.orchestrate',
-        'orch.push_batch',
-        'pillar={}'.format(pillar_arg),
-        '--async',
-    ]
-    log.info('dispatching: %s', ' '.join(cmd[:3]) + ' pillar=<{} actions>'.format(len(actions)))
-    try:
-        result = subprocess.run(
-            cmd, check=True, capture_output=True, text=True, timeout=60,
-        )
-    except subprocess.CalledProcessError as exc:
-        log.error('dispatch failed (rc=%s): stdout=%s stderr=%s',
-                  exc.returncode, exc.stdout, exc.stderr)
-        return False
-    except subprocess.TimeoutExpired:
-        log.error('dispatch timed out after 60s')
-        return False
-    except Exception:
-        log.exception('dispatch raised')
-        return False
-    log.info('dispatch accepted: %s', (result.stdout or '').strip())
-    return True
-
-
-def main():
-    log = _make_logger()
-
-    if not os.path.isdir(PENDING_DIR):
-        # Nothing to do; reactors create the dir on first use.
-        return 0
-
-    try:
-        push = _load_push_cfg()
-    except Exception:
-        log.exception('failed to read global:push pillar; aborting drain pass')
-        return 1
-
-    if not push.get('enabled', True):
-        log.debug('push disabled; exiting')
-        return 0
-
-    debounce_seconds = int(push.get('debounce_seconds', 30))
-
-    os.makedirs(PENDING_DIR, exist_ok=True)
-    lock_fd = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR, 0o644)
-    try:
-        fcntl.flock(lock_fd, fcntl.LOCK_EX)
-
-        intent_files = [
-            p for p in sorted(glob.glob(os.path.join(PENDING_DIR, '*.json')))
-            if os.path.basename(p) != '.lock'
-        ]
-        if not intent_files:
-            return 0
-
-        now = time.time()
-        ready = []
-        skipped = 0
-        broken = []
-        for path in intent_files:
-            intent = _read_intent(path, log)
-            if not isinstance(intent, dict):
-                broken.append(path)
-                continue
-            last_touch = intent.get('last_touch', 0)
-            if now - last_touch < debounce_seconds:
-                skipped += 1
-                continue
-            ready.append((path, intent))
-
-        for path in broken:
-            try:
-                os.unlink(path)
-            except OSError:
-                pass
-
-        if not ready:
-            if skipped:
-                log.debug('no ready intents (%d still in debounce window)', skipped)
-            return 0
-
-        combined_actions = []
-        oldest_first_touch = now
-        all_paths = []
-        for path, intent in ready:
-            combined_actions.extend(intent.get('actions', []) or [])
-            first = intent.get('first_touch', now)
-            if first < oldest_first_touch:
-                oldest_first_touch = first
-            all_paths.extend(intent.get('paths', []) or [])
-
-        deduped = _dedupe_actions(combined_actions)
-        if not deduped:
-            log.warning('%d intent(s) had no usable actions; clearing', len(ready))
-            for path, _ in ready:
-                try:
-                    os.unlink(path)
-                except OSError:
-                    pass
-            return 0
-
-        debounce_duration = now - oldest_first_touch
-        log.info(
-            'draining %d intent(s): %d action(s) after dedupe (raw=%d), '
-            'debounce_duration=%.1fs, paths=%s',
-            len(ready), len(deduped), len(combined_actions),
-            debounce_duration, all_paths[:20],
-        )
-
-        if not _dispatch(deduped, log):
-            log.warning('dispatch failed; leaving intent files in place for retry')
-            return 1
-
-        for path, _ in ready:
-            try:
-                os.unlink(path)
-            except OSError:
-                log.exception('failed to remove drained intent %s', path)
-
-        return 0
-    finally:
-        try:
-            fcntl.flock(lock_fd, fcntl.LOCK_UN)
-        finally:
-            os.close(lock_fd)
-
-
-if __name__ == '__main__':
-    sys.exit(main())

salt/nginx/enabled.sls

@@ -34,7 +34,6 @@ make-rule-dir-nginx:
 so-nginx:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-nginx:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: so-nginx
     - networks:
       - sobridge:

salt/orch/push_batch.sls

@@ -1,37 +0,0 @@
-{% from 'global/map.jinja' import GLOBALMERGED %}
-{% set actions = salt['pillar.get']('actions', []) %}
-{% set BATCH = GLOBALMERGED.push.batch %}
-{% set BATCH_WAIT = GLOBALMERGED.push.batch_wait %}
-
-{% for action in actions %}
-{%   if action.get('highstate') %}
-apply_highstate_{{ loop.index }}:
-  salt.state:
-    - tgt: '{{ action.tgt }}'
-    - tgt_type: {{ action.get('tgt_type', 'compound') }}
-    - highstate: True
-    - batch: {{ action.get('batch', BATCH) }}
-    - batch_wait: {{ action.get('batch_wait', BATCH_WAIT) }}
-    - kwarg:
-        queue: 2
-{%   else %}
-refresh_pillar_{{ loop.index }}:
-  salt.function:
-    - name: saltutil.refresh_pillar
-    - tgt: '{{ action.tgt }}'
-    - tgt_type: {{ action.get('tgt_type', 'compound') }}
-
-apply_{{ action.state | replace('.', '_') }}_{{ loop.index }}:
-  salt.state:
-    - tgt: '{{ action.tgt }}'
-    - tgt_type: {{ action.get('tgt_type', 'compound') }}
-    - sls:
-      - {{ action.state }}
-    - batch: {{ action.get('batch', BATCH) }}
-    - batch_wait: {{ action.get('batch_wait', BATCH_WAIT) }}
-    - kwarg:
-        queue: 2
-    - require:
-      - salt: refresh_pillar_{{ loop.index }}
-{%   endif %}
-{% endfor %}

salt/reactor/pillar_push_map.yaml

@@ -1,240 +0,0 @@
-# One pillar directory can map to multiple (state, tgt) actions.
-# tgt is a raw salt compound expression. tgt_type is always "compound".
-# Per-action `batch` / `batch_wait` override the orch defaults (25% / 15s).
-# An action with `highstate: True` triggers state.highstate instead of
-# state.apply -- see salt/orch/push_batch.sls.
-#
-# Notes:
-#   - `bpf` is a pillar-only dir (no state of its own) consumed by both
-#     zeek and suricata via macros, so a bpf pillar change re-applies both.
-#   - suricata/strelka/zeek/elasticsearch/redis/kafka/logstash etc. have
-#     their own pillar dirs AND their own state, so they map 1:1 (or 1:2
-#     in strelka's case, because of the split init.sls / manager.sls).
-#
-# Intentional omissions (these will log a "not in pillar_push_map.yaml"
-# warning in push_pillar.sls and wait for the next scheduled highstate):
-#   - `data` and `node_data`: pillar-only data consumed by many states;
-#     handling them generically would amount to a fleetwide highstate.
-#   - `host`: soc_host describes mainint/mainip; a change is a re-IP and
-#     needs a coordinated procedure, not an immediate state push.
-#   - `hypervisor`: state changes touch libvirt and are disruptive; leave
-#     to the next scheduled highstate.
-#   - `sensor`: every field in soc_sensor.yaml is `readonly: True` or
-#     per-minion (`node: True`). Per-minion edits are persisted under
-#     pillar/minions/<id>.sls and are handled by Branch A of push_pillar.sls
-#     (per-minion highstate intent), not by this app-pillar map.
-#
-# The role sets here were verified line-by-line against salt/top.sls. If
-# salt/top.sls changes how an app is targeted, update the corresponding
-# compound here.
-
-# firewall: the one pillar everyone touches. Applied everywhere intentionally
-# because every host's iptables needs to know about every other host in the
-# grid. Salt's firewall state is idempotent (file.managed + iptables-restore
-# onchanges in salt/firewall/init.sls), so hosts whose rendered firewall is
-# unchanged do a file comparison and no-op without touching iptables -- actual
-# reload happens only on the hosts whose rules actually changed. Fleetwide
-# blast radius is intentional and matches the pre-plan behavior via highstate.
-# Adding N sensors in a burst coalesces into one dispatch via the drainer.
-firewall:
-  - state: firewall
-    tgt: '*'
-
-# backup: backup.config_backup runs on eval, standalone, manager, managerhype,
-# managersearch (NOT import -- the backup pillar is included on import per
-# pillar/top.sls but the backup state is not run there per salt/top.sls).
-backup:
-  - state: backup.config_backup
-    tgt: 'G@role:so-eval or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone'
-
-# bpf is pillar-only (no state); consumed by both zeek and suricata as macros.
-# Both states run on sensor_roles + so-import per salt/top.sls.
-bpf:
-  - state: zeek
-    tgt: 'G@role:so-eval or G@role:so-heavynode or G@role:so-import or G@role:so-sensor or G@role:so-standalone'
-  - state: suricata
-    tgt: 'G@role:so-eval or G@role:so-heavynode or G@role:so-import or G@role:so-sensor or G@role:so-standalone'
-
-# ca is applied universally.
-ca:
-  - state: ca
-    tgt: '*'
-
-# docker: universal. The docker state is in both the all-non-managers and
-# all-managers branches of salt/top.sls.
-docker:
-  - state: docker
-    tgt: '*'
-
-# elastalert: eval, standalone, manager, managerhype, managersearch (NOT import).
-elastalert:
-  - state: elastalert
-    tgt: 'G@role:so-eval or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone'
-
-# elastic-fleet-package-registry: manager_roles exactly.
-elastic-fleet-package-registry:
-  - state: elastic-fleet-package-registry
-    tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone'
-
-# elasticsearch: 8 roles.
-elasticsearch:
-  - state: elasticsearch
-    tgt: 'G@role:so-eval or G@role:so-heavynode or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-searchnode or G@role:so-standalone'
-
-# elasticagent: so-heavynode only.
-elasticagent:
-  - state: elasticagent
-    tgt: 'G@role:so-heavynode'
-
-# elasticfleet: base state only on pillar change. elasticfleet.install_agent_grid
-# is a deploy/enrollment step, not a config reload; leave it to the next highstate.
-elasticfleet:
-  - state: elasticfleet
-    tgt: 'G@role:so-eval or G@role:so-fleet or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone'
-
-# global: fanout to a fleetwide highstate. The global pillar (soc_global.sls)
-# carries cross-cutting settings (pipeline, url_base, imagerepo, mdengine, ...)
-# that are consumed by virtually every state, so a targeted re-apply isn't
-# meaningful. The drainer's batch/batch_wait throttling controls blast radius.
-global:
-  - highstate: True
-    tgt: '*'
-
-# healthcheck: eval, sensor, standalone only.
-healthcheck:
-  - state: healthcheck
-    tgt: 'G@role:so-eval or G@role:so-sensor or G@role:so-standalone'
-
-# hydra: manager_roles exactly.
-hydra:
-  - state: hydra
-    tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone'
-
-# idh: so-idh only.
-idh:
-  - state: idh
-    tgt: 'G@role:so-idh'
-
-# influxdb: manager_roles exactly.
-influxdb:
-  - state: influxdb
-    tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone'
-
-# kafka: standalone, manager, managerhype, managersearch, searchnode, receiver.
-kafka:
-  - state: kafka
-    tgt: 'G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode or G@role:so-standalone'
-
-# kibana: manager_roles exactly.
-kibana:
-  - state: kibana
-    tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone'
-
-# kratos: manager_roles exactly.
-kratos:
-  - state: kratos
-    tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone'
-
-# logrotate: universal (top-of-file '*' branch in salt/top.sls).
-logrotate:
-  - state: logrotate
-    tgt: '*'
-
-# logstash: 8 roles, no eval/import.
-logstash:
-  - state: logstash
-    tgt: 'G@role:so-fleet or G@role:so-heavynode or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode or G@role:so-standalone'
-
-# manager: manager_roles exactly. The manager state is also referenced under
-# *_sensor / *_heavynode top.sls blocks via `sensor`, but the standalone
-# `manager` state itself runs only on manager_roles.
-manager:
-  - state: manager
-    tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone'
-
-# nginx: 10 specific roles. NOT receiver, idh, hypervisor, desktop.
-nginx:
-  - state: nginx
-    tgt: 'G@role:so-eval or G@role:so-fleet or G@role:so-heavynode or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-searchnode or G@role:so-sensor or G@role:so-standalone'
-
-# ntp: universal (top-of-file '*' branch in salt/top.sls).
-ntp:
-  - state: ntp
-    tgt: '*'
-
-# patch: universal. soc_patch carries the OS update schedule, applied via
-# patch.os.schedule on every node (it's in both the all-non-managers and
-# all-managers branches of salt/top.sls).
-patch:
-  - state: patch.os.schedule
-    tgt: '*'
-
-# postgres: manager_roles exactly.
-postgres:
-  - state: postgres
-    tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone'
-
-# redis: 6 roles. standalone, manager, managerhype, managersearch, heavynode, receiver.
-# (NOT eval, NOT import, NOT searchnode.)
-redis:
-  - state: redis
-    tgt: 'G@role:so-heavynode or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-receiver or G@role:so-standalone'
-
-# registry: manager_roles exactly.
-registry:
-  - state: registry
-    tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone'
-
-# sensoroni: universal.
-sensoroni:
-  - state: sensoroni
-    tgt: '*'
-
-# soc: manager_roles exactly.
-soc:
-  - state: soc
-    tgt: 'G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone'
-
-# stig: broad. Runs on standalone, manager, managerhype, managersearch,
-# searchnode, sensor, receiver, fleet, hypervisor, desktop.
-# NOT eval, NOT import, NOT heavynode, NOT idh (the *_idh block in
-# salt/top.sls intentionally omits stig).
-stig:
-  - state: stig
-    tgt: 'G@role:so-desktop or G@role:so-fleet or G@role:so-hypervisor or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode or G@role:so-sensor or G@role:so-standalone'
-
-# strelka: sensor-side only on pillar change (sensor_roles). strelka.manager is
-# intentionally NOT fired on pillar changes -- YARA rule and strelka config
-# pillar changes are consumed by the sensor-side strelka backend, and re-running
-# strelka.manager on managers is both unnecessary and disruptive. strelka.manager
-# is left to the 2-hour highstate.
-strelka:
-  - state: strelka
-    tgt: 'G@role:so-eval or G@role:so-heavynode or G@role:so-sensor or G@role:so-standalone'
-
-# suricata: sensor_roles + so-import (5 roles).
-suricata:
-  - state: suricata
-    tgt: 'G@role:so-eval or G@role:so-heavynode or G@role:so-import or G@role:so-sensor or G@role:so-standalone'
-
-# telegraf: universal.
-telegraf:
-  - state: telegraf
-    tgt: '*'
-
-# versionlock: universal (top-of-file '*' branch in salt/top.sls).
-versionlock:
-  - state: versionlock
-    tgt: '*'
-
-# vm: libvirt-driver hypervisors only. Matched by the salt-cloud:driver:libvirt
-# grain (compound supports nested grain matching via G@<key>:<subkey>:<value>).
-# pillar/vm/soc_vm.sls write path is referenced at salt/_runners/setup_hypervisor.py:856.
-vm:
-  - state: vm
-    tgt: 'G@salt-cloud:driver:libvirt'
-
-# zeek: sensor_roles + so-import (5 roles).
-zeek:
-  - state: zeek
-    tgt: 'G@role:so-eval or G@role:so-heavynode or G@role:so-import or G@role:so-sensor or G@role:so-standalone'

salt/reactor/push_pillar.sls

@@ -1,176 +0,0 @@
-#!py
-
-# Reactor invoked by the pillar_db beacon when SOC records settings changes in
-# the so_soc.audit_settings table (see salt/_beacons/pillar_db.py). The beacon
-# emits one event per new row carrying setting_id and node_id.
-#
-# Two branches, keyed on node_id:
-#   A) node_id populated -> the change is scoped to that one minion. Look up the
-#      app in pillar_push_map.yaml and write an intent that runs the app's mapped
-#      state(s) targeted to just that node.
-#   B) node_id empty -> grid-wide app change. Look up the app in
-#      pillar_push_map.yaml and write an intent with the entry's actions as-is.
-#
-# The app name is the first dotted segment of setting_id (e.g. "telegraf.output"
-# -> "telegraf"), which matches the pillar_push_map.yaml keys 1:1.
-#
-# Reactors never dispatch directly. The so-push-drainer schedule picks up
-# ready intents, dedupes across pending files, and dispatches orch.push_batch.
-
-import fcntl
-import json
-import logging
-import os
-import time
-
-from salt.client import Caller
-import yaml
-
-LOG = logging.getLogger(__name__)
-
-PENDING_DIR = '/opt/so/state/push_pending'
-LOCK_FILE = os.path.join(PENDING_DIR, '.lock')
-MAX_PATHS = 20
-
-# The pillar_push_map.yaml is shipped via salt:// but the reactor runs on the
-# master, which mounts the default saltstack tree at this path.
-PUSH_MAP_PATH = '/opt/so/saltstack/default/salt/reactor/pillar_push_map.yaml'
-
-_PUSH_MAP_CACHE = {'mtime': 0, 'data': None}
-
-
-def _load_push_map():
-    try:
-        st = os.stat(PUSH_MAP_PATH)
-    except OSError:
-        LOG.warning('push_pillar: %s not found', PUSH_MAP_PATH)
-        return {}
-    if _PUSH_MAP_CACHE['mtime'] != st.st_mtime:
-        try:
-            with open(PUSH_MAP_PATH, 'r') as f:
-                _PUSH_MAP_CACHE['data'] = yaml.safe_load(f) or {}
-        except Exception:
-            LOG.exception('push_pillar: failed to load %s', PUSH_MAP_PATH)
-            _PUSH_MAP_CACHE['data'] = {}
-        _PUSH_MAP_CACHE['mtime'] = st.st_mtime
-    return _PUSH_MAP_CACHE['data'] or {}
-
-
-def _push_enabled():
-    try:
-        caller = Caller()
-        return bool(caller.cmd('pillar.get', 'global:push:enabled', True))
-    except Exception:
-        LOG.exception('push_pillar: pillar.get global:push:enabled failed, assuming enabled')
-        return True
-
-
-def _write_intent(key, actions, path):
-    now = time.time()
-    try:
-        os.makedirs(PENDING_DIR, exist_ok=True)
-    except OSError:
-        LOG.exception('push_pillar: cannot create %s', PENDING_DIR)
-        return
-
-    intent_path = os.path.join(PENDING_DIR, '{}.json'.format(key))
-    lock_fd = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR, 0o644)
-    try:
-        fcntl.flock(lock_fd, fcntl.LOCK_EX)
-
-        intent = {}
-        if os.path.exists(intent_path):
-            try:
-                with open(intent_path, 'r') as f:
-                    intent = json.load(f)
-            except (IOError, ValueError):
-                intent = {}
-
-        intent.setdefault('first_touch', now)
-        intent['last_touch'] = now
-        intent['actions'] = actions
-        paths = intent.get('paths', [])
-        if path and path not in paths:
-            paths.append(path)
-            paths = paths[-MAX_PATHS:]
-        intent['paths'] = paths
-
-        tmp_path = intent_path + '.tmp'
-        with open(tmp_path, 'w') as f:
-            json.dump(intent, f)
-        os.rename(tmp_path, intent_path)
-    except Exception:
-        LOG.exception('push_pillar: failed to write intent %s', intent_path)
-    finally:
-        try:
-            fcntl.flock(lock_fd, fcntl.LOCK_UN)
-        finally:
-            os.close(lock_fd)
-
-
-def _app_from_setting(setting_id):
-    # setting_id is e.g. 'telegraf.output' -> 'telegraf', 'ntp.config.servers' -> 'ntp'
-    if not setting_id:
-        return None
-    return setting_id.split('.', 1)[0] or None
-
-
-def _node_actions(entry, node_id):
-    # Copy the app's mapped actions but retarget each one to the single node.
-    # Preserves the state/highstate selection and any batch/batch_wait overrides.
-    actions = []
-    for action in entry:
-        if not isinstance(action, dict):
-            continue
-        node_action = dict(action)
-        node_action['tgt'] = node_id
-        node_action['tgt_type'] = 'glob'
-        actions.append(node_action)
-    return actions
-
-
-def run():
-    if not _push_enabled():
-        LOG.info('push_pillar: push disabled, skipping')
-        return {}
-
-    # The pillar_db beacon nests its payload under data['data']; fall back to the
-    # top level so the reactor is robust to either shape.
-    event = data.get('data', data)  # noqa: F821 -- data provided by reactor
-    setting_id = event.get('setting_id', '')
-    node_id = (event.get('node_id') or '').strip()
-
-    app = _app_from_setting(setting_id)
-    if not app:
-        LOG.debug('push_pillar: ignoring event with no app segment: setting_id=%s', setting_id)
-        return {}
-
-    push_map = _load_push_map()
-    entry = push_map.get(app)
-    if not entry:
-        LOG.warning(
-            'push_pillar: app "%s" is not in pillar_push_map.yaml; change will be '
-            'picked up at the next scheduled highstate (setting_id=%s)',
-            app, setting_id,
-        )
-        return {}
-
-    # Branch A: per-node change -> retarget the app's states to just that node.
-    if node_id:
-        actions = _node_actions(entry, node_id)
-        if not actions:
-            LOG.warning('push_pillar: no usable actions for app "%s" (setting_id=%s)', app, setting_id)
-            return {}
-        _write_intent(
-            'node_{}_{}'.format(node_id, app), actions,
-            'audit:{}@{}'.format(setting_id, node_id),
-        )
-        LOG.info('push_pillar: per-node intent updated for %s on %s (setting_id=%s)',
-                 app, node_id, setting_id)
-        return {}
-
-    # Branch B: grid-wide app change -> use the map entry's actions as-is.
-    actions = list(entry)  # copy to avoid mutating the cache
-    _write_intent('pillar_{}'.format(app), actions, 'audit:{}'.format(setting_id))
-    LOG.info('push_pillar: app intent updated for %s (setting_id=%s)', app, setting_id)
-    return {}

salt/reactor/push_strelka.sls

@@ -1,96 +0,0 @@
-#!py
-
-# Reactor invoked by the inotify beacon on rule file changes under
-# /opt/so/saltstack/local/salt/strelka/rules/compiled/.
-#
-# Writes (or updates) a push intent at /opt/so/state/push_pending/rules_strelka.json
-# and returns {}. The so-push-drainer schedule picks up ready intents, dedupes
-# across pending files, and dispatches orch.push_batch. Reactors never dispatch
-# directly -- see plan /home/mreeves/.claude/plans/goofy-marinating-hummingbird.md.
-
-import fcntl
-import json
-import logging
-import os
-import time
-
-from salt.client import Caller
-
-LOG = logging.getLogger(__name__)
-
-PENDING_DIR = '/opt/so/state/push_pending'
-LOCK_FILE = os.path.join(PENDING_DIR, '.lock')
-MAX_PATHS = 20
-
-# Mirrors GLOBALS.sensor_roles in salt/vars/globals.map.jinja. Sensor-side
-# strelka runs on exactly these four roles; so-import gets strelka.manager
-# instead, which is not fired on pillar changes.
-SENSOR_ROLES = ['so-eval', 'so-heavynode', 'so-sensor', 'so-standalone']
-
-
-def _sensor_compound():
-    return ' or '.join('G@role:{}'.format(r) for r in SENSOR_ROLES)
-
-
-def _push_enabled():
-    try:
-        caller = Caller()
-        return bool(caller.cmd('pillar.get', 'global:push:enabled', True))
-    except Exception:
-        LOG.exception('push_strelka: pillar.get global:push:enabled failed, assuming enabled')
-        return True
-
-
-def _write_intent(key, actions, path):
-    now = time.time()
-    try:
-        os.makedirs(PENDING_DIR, exist_ok=True)
-    except OSError:
-        LOG.exception('push_strelka: cannot create %s', PENDING_DIR)
-        return
-
-    intent_path = os.path.join(PENDING_DIR, '{}.json'.format(key))
-    lock_fd = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR, 0o644)
-    try:
-        fcntl.flock(lock_fd, fcntl.LOCK_EX)
-
-        intent = {}
-        if os.path.exists(intent_path):
-            try:
-                with open(intent_path, 'r') as f:
-                    intent = json.load(f)
-            except (IOError, ValueError):
-                intent = {}
-
-        intent.setdefault('first_touch', now)
-        intent['last_touch'] = now
-        intent['actions'] = actions
-        paths = intent.get('paths', [])
-        if path and path not in paths:
-            paths.append(path)
-            paths = paths[-MAX_PATHS:]
-        intent['paths'] = paths
-
-        tmp_path = intent_path + '.tmp'
-        with open(tmp_path, 'w') as f:
-            json.dump(intent, f)
-        os.rename(tmp_path, intent_path)
-    except Exception:
-        LOG.exception('push_strelka: failed to write intent %s', intent_path)
-    finally:
-        try:
-            fcntl.flock(lock_fd, fcntl.LOCK_UN)
-        finally:
-            os.close(lock_fd)
-
-
-def run():
-    if not _push_enabled():
-        LOG.info('push_strelka: push disabled, skipping')
-        return {}
-
-    path = data.get('path', '')  # noqa: F821 -- data provided by reactor
-    actions = [{'state': 'strelka', 'tgt': _sensor_compound()}]
-    _write_intent('rules_strelka', actions, path)
-    LOG.info('push_strelka: intent updated for path=%s', path)
-    return {}

salt/reactor/push_suricata.sls

@@ -1,95 +0,0 @@
-#!py
-
-# Reactor invoked by the inotify beacon on rule file changes under
-# /opt/so/saltstack/local/salt/suricata/rules/.
-#
-# Writes (or updates) a push intent at /opt/so/state/push_pending/rules_suricata.json
-# and returns {}. The so-push-drainer schedule picks up ready intents, dedupes
-# across pending files, and dispatches orch.push_batch. Reactors never dispatch
-# directly -- see plan /home/mreeves/.claude/plans/goofy-marinating-hummingbird.md.
-
-import fcntl
-import json
-import logging
-import os
-import time
-
-from salt.client import Caller
-
-LOG = logging.getLogger(__name__)
-
-PENDING_DIR = '/opt/so/state/push_pending'
-LOCK_FILE = os.path.join(PENDING_DIR, '.lock')
-MAX_PATHS = 20
-
-# Mirrors GLOBALS.sensor_roles in salt/vars/globals.map.jinja. Suricata also
-# runs on so-import per salt/top.sls, so that role is appended below.
-SENSOR_ROLES = ['so-eval', 'so-heavynode', 'so-sensor', 'so-standalone']
-
-
-def _sensor_compound_plus_import():
-    return ' or '.join('G@role:{}'.format(r) for r in SENSOR_ROLES) + ' or G@role:so-import'
-
-
-def _push_enabled():
-    try:
-        caller = Caller()
-        return bool(caller.cmd('pillar.get', 'global:push:enabled', True))
-    except Exception:
-        LOG.exception('push_suricata: pillar.get global:push:enabled failed, assuming enabled')
-        return True
-
-
-def _write_intent(key, actions, path):
-    now = time.time()
-    try:
-        os.makedirs(PENDING_DIR, exist_ok=True)
-    except OSError:
-        LOG.exception('push_suricata: cannot create %s', PENDING_DIR)
-        return
-
-    intent_path = os.path.join(PENDING_DIR, '{}.json'.format(key))
-    lock_fd = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR, 0o644)
-    try:
-        fcntl.flock(lock_fd, fcntl.LOCK_EX)
-
-        intent = {}
-        if os.path.exists(intent_path):
-            try:
-                with open(intent_path, 'r') as f:
-                    intent = json.load(f)
-            except (IOError, ValueError):
-                intent = {}
-
-        intent.setdefault('first_touch', now)
-        intent['last_touch'] = now
-        intent['actions'] = actions
-        paths = intent.get('paths', [])
-        if path and path not in paths:
-            paths.append(path)
-            paths = paths[-MAX_PATHS:]
-        intent['paths'] = paths
-
-        tmp_path = intent_path + '.tmp'
-        with open(tmp_path, 'w') as f:
-            json.dump(intent, f)
-        os.rename(tmp_path, intent_path)
-    except Exception:
-        LOG.exception('push_suricata: failed to write intent %s', intent_path)
-    finally:
-        try:
-            fcntl.flock(lock_fd, fcntl.LOCK_UN)
-        finally:
-            os.close(lock_fd)
-
-
-def run():
-    if not _push_enabled():
-        LOG.info('push_suricata: push disabled, skipping')
-        return {}
-
-    path = data.get('path', '')  # noqa: F821 -- data provided by reactor
-    actions = [{'state': 'suricata', 'tgt': _sensor_compound_plus_import()}]
-    _write_intent('rules_suricata', actions, path)
-    LOG.info('push_suricata: intent updated for path=%s', path)
-    return {}

salt/redis/enabled.sls

@@ -17,7 +17,6 @@ include:
 so-redis:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-redis:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: so-redis
     - user: socore
     - networks:

salt/registry/enabled.sls

@@ -21,9 +21,6 @@ so-dockerregistry:
     - networks:
       - sobridge:
         - ipv4_address: {{ DOCKERMERGED.containers['so-dockerregistry'].ip }}
-    # Intentionally `always` (not unless-stopped) -- registry is critical infra
-    # and must come back up even if it was manually stopped. Do not homogenize
-    # to unless-stopped; see the container auto-restart section of the plan.
     - restart_policy: always
     - port_bindings:
       {% for BINDING in DOCKERMERGED.containers['so-dockerregistry'].port_bindings %}

salt/salt/beacons.sls

@@ -3,7 +3,7 @@
 {% set SCHEDULE = salt['pillar.get']('healthcheck:schedule', 30) %}
 
 include:
-  - salt.minion
+  - salt
 
 {% if CHECKS and ENABLED %}
 salt_beacons:
@@ -14,13 +14,12 @@ salt_beacons:
     - defaults:
         CHECKS: {{ CHECKS }}
         SCHEDULE: {{ SCHEDULE }}
-    - watch_in:
+    - watch_in: 
       - service: salt_minion_service
 {% else %}
 salt_beacons:
   file.absent:
     - name: /etc/salt/minion.d/beacons.conf
-    - watch_in:
+    - watch_in: 
       - service: salt_minion_service
 {% endif %}
-

salt/salt/files/reactor_pushstate.conf

@@ -1,11 +0,0 @@
-reactor:
-  - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/suricata/rules':
-    - salt://reactor/push_suricata.sls
-  - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/suricata/rules/*':
-    - salt://reactor/push_suricata.sls
-  - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/strelka/rules/compiled':
-    - salt://reactor/push_strelka.sls
-  - 'salt/beacon/*/inotify//opt/so/saltstack/local/salt/strelka/rules/compiled/*':
-    - salt://reactor/push_strelka.sls
-  - 'salt/beacon/*/pillar_db/audit_settings':
-    - salt://reactor/push_pillar.sls

salt/salt/init.sls

@@ -5,11 +5,3 @@ salt_bootstrap:
     - source: salt://salt/scripts/bootstrap-salt.sh
     - mode: 755
     - show_changes: False
-
-salt_sbin:
-  file.recurse:
-    - name: /usr/sbin
-    - source: salt://salt/tools/sbin
-    - user: 939
-    - group: 939
-    - file_mode: 755

salt/salt/lasthighstate.sls

@@ -1,4 +1,4 @@
 lasthighstate:
   file.touch:
     - name: /opt/so/log/salt/lasthighstate
-    - order: 9001
+    - order: last

salt/salt/master.sls

@@ -10,13 +10,10 @@
 #    software that is protected by the license key."
 
 {% from 'allowed_states.map.jinja' import allowed_states %}
-{% from 'global/map.jinja' import GLOBALMERGED %}
 {% if sls in allowed_states %}
 
 include:
   - salt.minion
-  - salt.master.pyinotify
-  - salt.master.boot_mine_update
 {%   if 'vrt' in salt['pillar.get']('features', []) %}
   - salt.cloud
   - salt.cloud.reactor_config_hypervisor
@@ -65,21 +62,6 @@ engines_config:
     - name: /etc/salt/master.d/engines.conf
     - source: salt://salt/files/engines.conf
 
-{% if GLOBALMERGED.push.enabled %}
-reactor_pushstate_config:
-  file.managed:
-    - name: /etc/salt/master.d/reactor_pushstate.conf
-    - source: salt://salt/files/reactor_pushstate.conf
-    - watch_in:
-      - service: salt_master_service
-{% else %}
-reactor_pushstate_config:
-  file.absent:
-    - name: /etc/salt/master.d/reactor_pushstate.conf
-    - watch_in:
-      - service: salt_master_service
-{% endif %}
-
 # update the bootstrap script when used for salt-cloud
 salt_bootstrap_cloud:
   file.managed:
@@ -95,7 +77,7 @@ salt_master_service:
       - file: checkmine_engine
       - file: pillarWatch_engine
       - file: engines_config
-    - order: 9002
+    - order: last
 
 {% else %}
 

salt/salt/master/boot_mine_update.sls

@@ -1,29 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Manages /etc/systemd/system/so-boot-mine-update.service, a manager-only
-# Type=oneshot unit that pushes `salt '*' mine.update` once per boot, ordered
-# before so-boot-highstate.service so mine-backed pillars (node IPs, ES/Redis/
-# Logstash discovery) are fresh before the boot highstate renders them.
-
-include:
-  - systemd.reload
-
-so_boot_mine_update_unit_file:
-  file.managed:
-    - name: /etc/systemd/system/so-boot-mine-update.service
-    - source: salt://salt/service/so-boot-mine-update.service
-    - onchanges_in:
-      - module: systemd_reload
-
-# Only enable once setup is complete. Until then the gate file is missing and
-# the unit's own ConditionPathExists would no-op it anyway.
-so_boot_mine_update_service:
-  service.enabled:
-    - name: so-boot-mine-update.service
-    - onlyif: test -e /opt/so/state/setup-complete
-    - require:
-      - file: so_boot_mine_update_unit_file
-      - module: systemd_reload

salt/salt/master/pyinotify.sls

@@ -1,20 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-pyinotify_module_package:
-  file.recurse:
-    - name: /opt/so/conf/salt/module_packages/pyinotify
-    - source: salt://salt/module_packages/pyinotify
-    - clean: True
-    - makedirs: True
-
-pyinotify_python_module_install:
-  cmd.run:
-    - name: /opt/saltstack/salt/bin/python3.10 -m pip install pyinotify --no-index --find-links=/opt/so/conf/salt/module_packages/pyinotify/ --upgrade
-    - onchanges:
-      - file: pyinotify_module_package
-    - failhard: True
-    - watch_in:
-      - service: salt_minion_service

salt/salt/minion.defaults.yaml

@@ -2,3 +2,4 @@
 salt:
   minion:
     version: '3006.19'
+    check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default

salt/salt/minion/boot_highstate.sls

@@ -1,31 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Manages /etc/systemd/system/so-boot-highstate.service, a Type=oneshot
-# RemainAfterExit=yes unit that runs `salt-call state.highstate` exactly once
-# per system boot. Replaces the legacy `startup_states: highstate` minion
-# config, which fired on every salt-minion service restart (causing a redundant
-# highstate whenever a highstate itself restarted salt-minion).
-
-include:
-  - systemd.reload
-
-so_boot_highstate_unit_file:
-  file.managed:
-    - name: /etc/systemd/system/so-boot-highstate.service
-    - source: salt://salt/service/so-boot-highstate.service
-    - onchanges_in:
-      - module: systemd_reload
-
-# Only enable once setup is complete. Until then the gate file is missing and
-# the unit's own ConditionPathExists would no-op it anyway -- this just keeps
-# `systemctl is-enabled` honest for the sync_es_users gate.
-so_boot_highstate_service:
-  service.enabled:
-    - name: so-boot-highstate.service
-    - onlyif: test -e /opt/so/state/setup-complete
-    - require:
-      - file: so_boot_highstate_unit_file
-      - module: systemd_reload

salt/salt/minion/init.sls

@@ -17,7 +17,6 @@ include:
   - repo.client
   - salt.mine_functions
   - salt.minion.service_file
-  - salt.minion.boot_highstate
 {% if GLOBALS.is_manager %}
   - ca.signing_policy
 {% endif %}
@@ -81,47 +80,21 @@ set_log_levels:
       - "log_level: info"
       - "log_level_logfile: info"
 
-# startup_states: highstate caused a full highstate to run on every
-# salt-minion service start, including the restart triggered when a highstate
-# itself modified the minion config (beacons, mine, unit file). Replaced by
-# so-boot-highstate.service (managed in salt.minion.boot_highstate), which
-# runs once per system boot only. Strip the line from /etc/salt/minion on
-# upgrade; both the commented and uncommented forms historically existed.
-remove_startup_states:
-  file.line:
+enable_startup_states:
+  file.uncomment:
     - name: /etc/salt/minion
-    - match: 'startup_states: highstate'
-    - mode: delete
-
-# Upgrade-path bridge: systems that already passed setup under the old gate
-# (`grep -x 'startup_states: highstate' /etc/salt/minion`) get a /opt/so/state/setup-complete
-# marker so so-boot-highstate.service can be enabled and the so-user_sync cron
-# in sync_es_users.sls keeps installing. Setup-in-progress systems instead get
-# the marker from `mark_setup_complete` in setup/so-functions at the right
-# moment. `replace: false` means we never overwrite a marker once written.
-mark_setup_complete_for_upgrades:
-  file.managed:
-    - name: /opt/so/state/setup-complete
-    - replace: false
-    - makedirs: True
-    - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion"
-    - require_in:
-      - file: remove_startup_states
-      - service: so_boot_highstate_service
+    - regex: '^startup_states: highstate$'
+    - unless: pgrep so-setup
 
 {% endif %}
 
-# this has to be outside the if statement above since there are <requisite>_in calls to this state.
-# uses watch (not listen) so the restart fires in-state and its result lands on this state's
-# running entry; that is what lets wait_for_salt_minion_ready below detect any restart
-# uniformly via onchanges, regardless of whether the trigger came from these files or from
-# external watch_in's (e.g. beacons, master/pyinotify).
+# this has to be outside the if statement above since there are <requisite>_in calls to this state
 salt_minion_service:
   service.running:
     - name: salt-minion
     - enable: True
     - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}"
-    - watch:
+    - listen:
       - file: mine_functions
 {% if INSTALLEDSALTVERSION|string == SALTVERSION|string %}
       - file: set_log_levels
@@ -130,17 +103,3 @@ salt_minion_service:
       - file: signing_policy
 {% endif %}
     - order: last
-
-# block until the just-restarted salt-minion is back and can execute modules locally, so
-# follow-on jobs and the next highstate iteration do not race the restart. onchanges +
-# require on salt_minion_service catches every restart trigger uniformly because watch
-# mod_watch results replace the service state's running entry. wait logic lives in
-# /usr/sbin/so-salt-minion-wait (deployed by common_sbin from common/tools/sbin/).
-wait_for_salt_minion_ready:
-  cmd.run:
-    - name: /usr/sbin/so-salt-minion-wait
-    - onchanges:
-      - service: salt_minion_service
-    - require:
-      - service: salt_minion_service
-    - order: last

salt/salt/service/so-boot-highstate.service

@@ -1,14 +0,0 @@
-[Unit]
-Description=Security Onion boot-time highstate (runs once per boot)
-After=salt-minion.service network-online.target docker.service
-Wants=network-online.target docker.service
-Requires=salt-minion.service
-ConditionPathExists=/opt/so/state/setup-complete
-
-[Service]
-Type=oneshot
-RemainAfterExit=yes
-ExecStart=/usr/bin/salt-call state.highstate -l info queue=True
-
-[Install]
-WantedBy=multi-user.target

salt/salt/service/so-boot-mine-update.service

@@ -1,15 +0,0 @@
-[Unit]
-Description=Security Onion boot-time grid mine.update (managers, runs once per boot before highstate)
-After=salt-master.service salt-minion.service network-online.target
-Wants=network-online.target
-Requires=salt-master.service salt-minion.service
-Before=so-boot-highstate.service
-ConditionPathExists=/opt/so/state/setup-complete
-
-[Service]
-Type=oneshot
-RemainAfterExit=yes
-ExecStart=/usr/sbin/so-boot-mine-update
-
-[Install]
-WantedBy=multi-user.target

salt/salt/tools/sbin/so-salt-minion-wait

@@ -1,35 +0,0 @@
-#!/bin/bash
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Block until the local salt-minion service is back up and can execute modules locally.
-# Invoked from the wait_for_salt_minion_ready state in salt/minion/init.sls after
-# salt_minion_service fires its watch-driven mod_watch (a non-blocking systemctl restart),
-# so follow-on jobs and the next highstate iteration do not race the in-flight restart.
-
-. /usr/sbin/so-common
-
-# Initial sleep gives the systemctl restart (--no-block by default for salt-minion on
-# >=3006.15) time to begin tearing down the old process before we probe for readiness.
-INITIAL_SLEEP=3
-TIMEOUT=120
-PING_TIMEOUT=5
-
-sleep "$INITIAL_SLEEP"
-
-elapsed="$INITIAL_SLEEP"
-while [ "$elapsed" -lt "$TIMEOUT" ]; do
-  if systemctl is-active --quiet salt-minion \
-     && salt-call --local --timeout="$PING_TIMEOUT" --out=quiet test.ping >/dev/null 2>&1; then
-    echo "salt-minion ready after ${elapsed}s"
-    exit 0
-  fi
-  sleep 1
-  elapsed=$((elapsed + 1))
-done
-
-echo "salt-minion did not become ready within ${TIMEOUT}s" >&2
-exit 1

salt/schedule.sls

@@ -1,26 +1,10 @@
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-{% from 'global/map.jinja' import GLOBALMERGED %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
 
 highstate_schedule:
   schedule.present:
     - function: state.highstate
-    - hours: {{ GLOBALMERGED.push.highstate_interval_hours }}
+    - minutes: 15
     - maxrunning: 1
 {% if not GLOBALS.is_manager %}
-    - splay: 1800
-{% endif %}
-
-{% if GLOBALS.is_manager and GLOBALMERGED.push.enabled %}
-push_drain_schedule:
-  schedule.present:
-    - function: cmd.run
-    - job_args:
-      - /usr/sbin/so-push-drainer
-    - seconds: {{ GLOBALMERGED.push.drain_interval }}
-    - maxrunning: 1
-    - return_job: False
-{% elif GLOBALS.is_manager %}
-push_drain_schedule:
-  schedule.absent:
-    - name: push_drain_schedule
+    - splay: 120
 {% endif %}

salt/sensoroni/enabled.sls

@@ -14,7 +14,6 @@ include:
 so-sensoroni:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-soc:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - network_mode: host
     - binds:
       - /nsm/import:/nsm/import:rw

salt/setup/virt/setSalt.sls

@@ -8,6 +8,11 @@ set_role_grain:
     - name: role
     - value: so-{{ grains.id.split("_") | last }}
 
+set_highstate:
+  file.append:
+    - name: /etc/salt/minion
+    - text: 'startup_states: highstate'
+
 enable_salt_minion:
   service.enabled:
     - name: salt-minion

salt/soc/enabled.sls

@@ -18,7 +18,6 @@ include:
 so-soc:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-soc:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - hostname: soc
     - name: so-soc
     - networks:

salt/strelka/backend/enabled.sls

@@ -47,10 +47,6 @@ strelka_backend:
       - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
     {%   endfor %}
     {% endif %}
-    # Intentionally `on-failure` (not unless-stopped) -- strelka backend shuts
-    # down cleanly during rule reloads and we do not want those clean exits to
-    # trigger an auto-restart. Do not homogenize; see the container
-    # auto-restart section of the plan.
     - restart_policy: on-failure
     - watch:
       - file: strelkasensorcompiledrules

salt/strelka/coordinator/enabled.sls

@@ -15,7 +15,6 @@ include:
 strelka_coordinator:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-redis:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - name: so-strelka-coordinator
     - networks:
       - sobridge:

salt/strelka/filestream/enabled.sls

@@ -15,7 +15,6 @@ include:
 strelka_filestream:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-strelka-manager:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - binds:
       - /opt/so/conf/strelka/filestream/:/etc/strelka/:ro
       - /nsm/strelka:/nsm/strelka

salt/strelka/frontend/enabled.sls

@@ -15,7 +15,6 @@ include:
 strelka_frontend:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-strelka-manager:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - binds:
       - /opt/so/conf/strelka/frontend/:/etc/strelka/:ro
       - /nsm/strelka/log/:/var/log/strelka/:rw

salt/strelka/gatekeeper/enabled.sls

@@ -15,7 +15,6 @@ include:
 strelka_gatekeeper:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-redis:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - name: so-strelka-gatekeeper
     - networks:
       - sobridge:

salt/strelka/manager/enabled.sls

@@ -15,7 +15,6 @@ include:
 strelka_manager:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-strelka-manager:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - binds:
       - /opt/so/conf/strelka/manager/:/etc/strelka/:ro
       {% if DOCKERMERGED.containers['so-strelka-manager'].custom_bind_mounts %}

salt/suricata/enabled.sls

@@ -18,7 +18,6 @@ so-suricata:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-suricata:{{ GLOBALS.so_version }}
     - privileged: True
-    - restart_policy: unless-stopped
     - environment:
       - INTERFACE={{ GLOBALS.sensor.interface }}
       {% if DOCKERMERGED.containers['so-suricata'].extra_env %}

salt/tcpreplay/init.sls

@@ -7,7 +7,6 @@ so-tcpreplay:
   docker_container.running:
     - network_mode: "host"
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-tcpreplay:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - name: so-tcpreplay
     - user: root
     - interactive: True

salt/telegraf/enabled.sls

@@ -18,7 +18,6 @@ include:
 so-telegraf:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-telegraf:{{ GLOBALS.so_version }}
-    - restart_policy: unless-stopped
     - user: 939
     - group_add: 939,920
     - environment:

salt/zeek/enabled.sls

@@ -18,7 +18,6 @@ so-zeek:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-zeek:{{ GLOBALS.so_version }}
     - start: True
     - privileged: True
-    - restart_policy: unless-stopped
     {% if DOCKERMERGED.containers['so-zeek'].ulimits %}
     - ulimits:
     {%   for ULIMIT in DOCKERMERGED.containers['so-zeek'].ulimits %}

setup/so-functions

@@ -539,19 +539,16 @@ configure_minion() {
 		"  x509_v2: true"\
 		"log_level: info"\
 		"log_level_logfile: info"\
-		"log_file: /opt/so/log/salt/minion" >> "$minion_config"
+		"log_file: /opt/so/log/salt/minion"\
+		"#startup_states: highstate" >> "$minion_config"
 
 }
 
-mark_setup_complete() {
-	# Writes the setup-complete marker. Salt's so-boot-highstate.service
-	# (boot-time oneshot) and the so-user_sync cron gate in
-	# salt/manager/sync_es_users.sls both key off this file.
-	local marker=/opt/so/state/setup-complete
+checkin_at_boot() {
+	local minion_config=/etc/salt/minion
 
-	info "Marking setup as complete"
-	mkdir -p "$(dirname "$marker")"
-	touch "$marker"
+	info "Enabling checkin at boot"
+	sed -i 's/#startup_states: highstate/startup_states: highstate/' "$minion_config"
 }
 
 check_requirements() {
@@ -980,8 +977,6 @@ docker_seed_registry() {
 		docker_seed_update_percent=25
 
 		update_docker_containers 'netinstall' '' 'docker_seed_update' '/dev/stdout' 2>&1 | tee -a "$setup_log"
-        # Use pipe exit status of 'update_docker_containers' for return code
-		return ${PIPESTATUS[0]}
 	fi
 }
 

setup/so-setup

@@ -223,6 +223,8 @@ if [ -n "$test_profile" ]; then
 	WEBPASSWD1=0n10nus3r
 	WEBPASSWD2=0n10nus3r
 	NODE_DESCRIPTION="${HOSTNAME} - ${install_type} - ${MSRVIP_OFFSET}"
+	# opt out of telemetry for automated testing
+	telemetry=1
 
 	update_sudoers_for_testing
 fi
@@ -767,10 +769,7 @@ if ! [[ -f $install_opt_file ]]; then
 		title "Applying the registry state"
 		logCmd "salt-call state.apply -l info registry"
 		title "Seeding the docker registry"
-		if ! docker_seed_registry; then
-			error "Failed to seed the docker registry"
-			fail_setup
-		fi
+		docker_seed_registry
 		title "Applying the manager state"
 		logCmd "salt-call state.apply -l info manager"
 		logCmd "salt-call state.apply influxdb -l info"
@@ -795,7 +794,7 @@ if ! [[ -f $install_opt_file ]]; then
 			error "Failed to run so-elastic-fleet-setup"
 			fail_setup
 		fi
-		mark_setup_complete
+		checkin_at_boot
 		set_initial_firewall_access
         initialize_elasticsearch_indices "so-case so-casehistory so-assistant-session so-assistant-chat"
 		# run a final highstate before enabling scheduled highstates.