From 9580976ba2a1317d1b99ef4091a964248170d8bb Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Mon, 8 Jun 2026 11:05:13 -0400 Subject: [PATCH] Add manager boot-time grid mine.update oneshot before highstate so-boot-mine-update.service is a manager-only Type=oneshot unit that runs once per boot after salt-master/salt-minion start and before so-boot-highstate.service. It pushes mine.update to all reachable minions so mine-backed pillars (node IPs, ES/Redis/Logstash discovery) are fresh before the boot highstate renders them. The helper waits for the responsive minion set to settle (plateau) rather than for every accepted key to report up, so an intentionally powered-off minion doesn't block the update; MAX_WAIT remains as a backstop. --- salt/manager/tools/sbin/so-boot-mine-update | 42 +++++++++++++++++++ salt/salt/master.sls | 1 + salt/salt/master/boot_mine_update.sls | 29 +++++++++++++ salt/salt/service/so-boot-mine-update.service | 15 +++++++ 4 files changed, 87 insertions(+) create mode 100755 salt/manager/tools/sbin/so-boot-mine-update create mode 100644 salt/salt/master/boot_mine_update.sls create mode 100644 salt/salt/service/so-boot-mine-update.service diff --git a/salt/manager/tools/sbin/so-boot-mine-update b/salt/manager/tools/sbin/so-boot-mine-update new file mode 100755 index 000000000..f497d891f --- /dev/null +++ b/salt/manager/tools/sbin/so-boot-mine-update @@ -0,0 +1,42 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Runs once per boot on managers (via so-boot-mine-update.service), before +# so-boot-highstate.service. Waits for the responsive minion set to settle, then +# pushes mine.update to all minions so mine-backed pillars (node IPs, ES/Redis/ +# Logstash discovery) are fresh before the boot highstate renders them. + +MAX_WAIT=${MINE_UPDATE_MAX_WAIT:-180} # hard backstop only +INTERVAL=10 +STABLE_CHECKS=3 # up-count must hold steady this many polls +elapsed=0 +prev=-1 +stable=0 +up=0 + +# Wait for the *reachable* minion set to settle rather than for every accepted +# key to report up: an operator may accept a minion's key and then intentionally +# power off that host, so requiring up >= accepted would never be satisfied and +# we'd always burn the full MAX_WAIT. Once the responsive count stops growing we +# stop waiting and run mine.update against whoever is up. +while [ "$elapsed" -lt "$MAX_WAIT" ]; do + up=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null \ + | python3 -c 'import sys,json; print(len(json.load(sys.stdin)))' 2>/dev/null) + up=${up:-0} + if [ "$up" -gt 0 ] && [ "$up" -eq "$prev" ]; then + stable=$((stable + 1)) + [ "$stable" -ge "$STABLE_CHECKS" ] && break + else + stable=0 + fi + prev=$up + sleep "$INTERVAL" + elapsed=$((elapsed + INTERVAL)) +done + +echo "so-boot-mine-update: ${up} minions up (settled after ${elapsed}s); running mine.update" +/usr/bin/salt '*' mine.update --out=txt diff --git a/salt/salt/master.sls b/salt/salt/master.sls index 895150cd7..c62bd20f3 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -14,6 +14,7 @@ include: - salt.minion + - salt.master.boot_mine_update {% if 'vrt' in salt['pillar.get']('features', []) %} - salt.cloud - salt.cloud.reactor_config_hypervisor diff --git a/salt/salt/master/boot_mine_update.sls b/salt/salt/master/boot_mine_update.sls new file mode 100644 index 000000000..9f96c0ddf --- /dev/null +++ b/salt/salt/master/boot_mine_update.sls @@ -0,0 +1,29 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Manages /etc/systemd/system/so-boot-mine-update.service, a manager-only +# Type=oneshot unit that pushes `salt '*' mine.update` once per boot, ordered +# before so-boot-highstate.service so mine-backed pillars (node IPs, ES/Redis/ +# Logstash discovery) are fresh before the boot highstate renders them. + +include: + - systemd.reload + +so_boot_mine_update_unit_file: + file.managed: + - name: /etc/systemd/system/so-boot-mine-update.service + - source: salt://salt/service/so-boot-mine-update.service + - onchanges_in: + - module: systemd_reload + +# Only enable once setup is complete. Until then the gate file is missing and +# the unit's own ConditionPathExists would no-op it anyway. +so_boot_mine_update_service: + service.enabled: + - name: so-boot-mine-update.service + - onlyif: test -e /opt/so/state/setup-complete + - require: + - file: so_boot_mine_update_unit_file + - module: systemd_reload diff --git a/salt/salt/service/so-boot-mine-update.service b/salt/salt/service/so-boot-mine-update.service new file mode 100644 index 000000000..c5c6cdf7b --- /dev/null +++ b/salt/salt/service/so-boot-mine-update.service @@ -0,0 +1,15 @@ +[Unit] +Description=Security Onion boot-time grid mine.update (managers, runs once per boot before highstate) +After=salt-master.service salt-minion.service network-online.target +Wants=network-online.target +Requires=salt-master.service salt-minion.service +Before=so-boot-highstate.service +ConditionPathExists=/opt/so/state/setup-complete + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/sbin/so-boot-mine-update + +[Install] +WantedBy=multi-user.target