diff --git a/salt/manager/tools/sbin/so-boot-mine-update b/salt/manager/tools/sbin/so-boot-mine-update new file mode 100755 index 000000000..f497d891f --- /dev/null +++ b/salt/manager/tools/sbin/so-boot-mine-update @@ -0,0 +1,42 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Runs once per boot on managers (via so-boot-mine-update.service), before +# so-boot-highstate.service. Waits for the responsive minion set to settle, then +# pushes mine.update to all minions so mine-backed pillars (node IPs, ES/Redis/ +# Logstash discovery) are fresh before the boot highstate renders them. + +MAX_WAIT=${MINE_UPDATE_MAX_WAIT:-180} # hard backstop only +INTERVAL=10 +STABLE_CHECKS=3 # up-count must hold steady this many polls +elapsed=0 +prev=-1 +stable=0 +up=0 + +# Wait for the *reachable* minion set to settle rather than for every accepted +# key to report up: an operator may accept a minion's key and then intentionally +# power off that host, so requiring up >= accepted would never be satisfied and +# we'd always burn the full MAX_WAIT. Once the responsive count stops growing we +# stop waiting and run mine.update against whoever is up. +while [ "$elapsed" -lt "$MAX_WAIT" ]; do + up=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null \ + | python3 -c 'import sys,json; print(len(json.load(sys.stdin)))' 2>/dev/null) + up=${up:-0} + if [ "$up" -gt 0 ] && [ "$up" -eq "$prev" ]; then + stable=$((stable + 1)) + [ "$stable" -ge "$STABLE_CHECKS" ] && break + else + stable=0 + fi + prev=$up + sleep "$INTERVAL" + elapsed=$((elapsed + INTERVAL)) +done + +echo "so-boot-mine-update: ${up} minions up (settled after ${elapsed}s); running mine.update" +/usr/bin/salt '*' mine.update --out=txt diff --git a/salt/salt/master.sls b/salt/salt/master.sls index 895150cd7..c62bd20f3 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -14,6 +14,7 @@ include: - salt.minion + - salt.master.boot_mine_update {% if 'vrt' in salt['pillar.get']('features', []) %} - salt.cloud - salt.cloud.reactor_config_hypervisor diff --git a/salt/salt/master/boot_mine_update.sls b/salt/salt/master/boot_mine_update.sls new file mode 100644 index 000000000..9f96c0ddf --- /dev/null +++ b/salt/salt/master/boot_mine_update.sls @@ -0,0 +1,29 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Manages /etc/systemd/system/so-boot-mine-update.service, a manager-only +# Type=oneshot unit that pushes `salt '*' mine.update` once per boot, ordered +# before so-boot-highstate.service so mine-backed pillars (node IPs, ES/Redis/ +# Logstash discovery) are fresh before the boot highstate renders them. + +include: + - systemd.reload + +so_boot_mine_update_unit_file: + file.managed: + - name: /etc/systemd/system/so-boot-mine-update.service + - source: salt://salt/service/so-boot-mine-update.service + - onchanges_in: + - module: systemd_reload + +# Only enable once setup is complete. Until then the gate file is missing and +# the unit's own ConditionPathExists would no-op it anyway. +so_boot_mine_update_service: + service.enabled: + - name: so-boot-mine-update.service + - onlyif: test -e /opt/so/state/setup-complete + - require: + - file: so_boot_mine_update_unit_file + - module: systemd_reload diff --git a/salt/salt/service/so-boot-mine-update.service b/salt/salt/service/so-boot-mine-update.service new file mode 100644 index 000000000..c5c6cdf7b --- /dev/null +++ b/salt/salt/service/so-boot-mine-update.service @@ -0,0 +1,15 @@ +[Unit] +Description=Security Onion boot-time grid mine.update (managers, runs once per boot before highstate) +After=salt-master.service salt-minion.service network-online.target +Wants=network-online.target +Requires=salt-master.service salt-minion.service +Before=so-boot-highstate.service +ConditionPathExists=/opt/so/state/setup-complete + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/sbin/so-boot-mine-update + +[Install] +WantedBy=multi-user.target