Files
securityonion/salt/salt/minion/init.sls
T
Josh Patterson 141116f550 Make so-salt-minion-wait work without requiring a restart
The wait required both a socket gate and a log gate to pass. The log gate
greps the minion log for salt's one-time startup line "Minion is ready to
receive requests!", which scrolls out of the log tail on a minion that has
not restarted recently. On such a minion the log gate could never pass, so
the script ran to its full 120s timeout and exited 1 even though the minion
was healthy and connected. This also false-timed-out when salt_minion_service
reported a non-restart change (e.g. an enable toggle).

The log gate's only remaining purpose is closing the ~2.8s post-connect window
where the master sockets are up but _post_master_init() is still loading. Gate
it on the current pid's uptime: enforce the ready line only within
READY_LINE_WINDOW (90s) of (re)start, and let the already restart-independent
socket gate be the steady-state authority past that. The fresh-restart path is
unchanged, and if uptime can't be read the strict behavior is kept.
2026-07-16 16:52:06 -04:00

154 lines
5.7 KiB
YAML+Jinja

# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'salt/map.jinja' import UPGRADECOMMAND with context %}
{% from 'salt/map.jinja' import SALTVERSION %}
{% from 'salt/map.jinja' import INSTALLEDSALTVERSION %}
{% from 'salt/map.jinja' import SALTPACKAGES %}
{% import_yaml 'salt/minion.defaults.yaml' as SALTMINION %}
include:
- salt.python_modules
- salt.patch.x509_v2
- salt
- repo.client
- salt.mine_functions
- salt.minion.service_file
- salt.minion.boot_highstate
{% if GLOBALS.is_manager %}
- ca.signing_policy
{% endif %}
{% if INSTALLEDSALTVERSION|string != SALTVERSION|string %}
unhold_salt_packages:
pkg.unheld:
- pkgs:
{% for package in SALTPACKAGES %}
- {{ package }}
{% endfor %}
install_salt_minion:
cmd.run:
- name: /bin/sh -c '{{ UPGRADECOMMAND }}'
# minion service is in failed state after upgrade. this command will start it after the state run for the upgrade completes
start_minion_post_upgrade:
cmd.run:
- name: |
exec 0>&- # close stdin
exec 1>&- # close stdout
exec 2>&- # close stderr
nohup /bin/sh -c 'sleep 30; systemctl start salt-minion' &
- require:
- cmd: install_salt_minion
- watch:
- cmd: install_salt_minion
- order: last
{% endif %}
{% if INSTALLEDSALTVERSION|string == SALTVERSION|string %}
{% for package in SALTPACKAGES %}
# only hold the package if it is already installed
{% if salt['pkg.version'](package) %}
hold_{{ package }}_package:
pkg.held:
- name: {{ package }}
- version: {{SALTVERSION}}-0.*
{% endif %}
{% endfor %}
remove_error_log_level_logfile:
file.line:
- name: /etc/salt/minion
- match: "log_level_logfile: error"
- mode: delete
remove_error_log_level:
file.line:
- name: /etc/salt/minion
- match: "log_level: error"
- mode: delete
set_log_levels:
file.append:
- name: /etc/salt/minion
- text:
- "log_level: info"
- "log_level_logfile: info"
# startup_states: highstate caused a full highstate to run on every
# salt-minion service start, including the restart triggered when a highstate
# itself modified the minion config (beacons, mine, unit file). Replaced by
# so-boot-highstate.service (managed in salt.minion.boot_highstate), which
# runs once per system boot only. Strip the line from /etc/salt/minion on
# upgrade; both the commented and uncommented forms historically existed.
remove_startup_states:
file.line:
- name: /etc/salt/minion
- match: 'startup_states: highstate'
- mode: delete
# Upgrade-path bridge: systems that already passed setup under the old gate
# (`grep -x 'startup_states: highstate' /etc/salt/minion`) get a /opt/so/state/setup-complete
# marker so so-boot-highstate.service can be enabled and the so-user_sync cron
# in sync_es_users.sls keeps installing. Setup-in-progress systems instead get
# the marker from `mark_setup_complete` in setup/so-functions at the right
# moment. `replace: false` means we never overwrite a marker once written.
mark_setup_complete_for_upgrades:
file.managed:
- name: /opt/so/state/setup-complete
- replace: false
- makedirs: True
- onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion"
- require_in:
- file: remove_startup_states
- service: so_boot_highstate_service
{% endif %}
# this has to be outside the if statement above since there are <requisite>_in calls to this state.
# uses watch (not listen) so the restart fires in-state and its result lands on this state's
# running entry; that is what lets wait_for_salt_minion_ready below detect any restart
# uniformly via onchanges, regardless of whether the trigger came from these files or from
# external watch_in's (e.g. beacons, master/pyinotify).
salt_minion_service:
service.running:
- name: salt-minion
- enable: True
- onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}"
- watch:
- file: mine_functions
{% if INSTALLEDSALTVERSION|string == SALTVERSION|string %}
- file: set_log_levels
{% endif %}
{% if GLOBALS.is_manager %}
- file: signing_policy
{% endif %}
- order: last
# block until the salt-minion daemon is ready for the current instance, so follow-on jobs and the
# next highstate iteration do not race the restart. onchanges + require on salt_minion_service
# catches every restart trigger uniformly because watch mod_watch results replace the service
# state's running entry. wait logic lives in /usr/sbin/so-salt-minion-wait (deployed by salt_sbin
# from salt/tools/sbin/); its steady-state authority is the master req/publish sockets for the
# current daemon pid (resolved via systemd, not the pidfile), and it corroborates a just-restarted
# instance with the pid-tagged "Minion is ready to receive requests!" log line only within a short
# window of startup. Because that socket signal does not require a recent restart, the wait also
# succeeds cleanly when salt_minion_service reports a non-restart change (e.g. an enable toggle)
# rather than false-timing-out. set_log_levels above enforces the log_level_logfile: info that the
# ready line depends on. salt restarts this unit with --no-block, so mod_watch returns while the old
# daemon is still up; the script waits for systemd's restart job to drain before it reads MainPID.
wait_for_salt_minion_ready:
cmd.run:
- name: /usr/sbin/so-salt-minion-wait
- onchanges:
- service: salt_minion_service
- require:
- service: salt_minion_service
- order: last