From fabecb82885d80efe96edf239f8b463f0103b8ef Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 14 May 2026 13:57:40 -0400 Subject: [PATCH 1/4] remove highstate from startup_states. highstate on system start --- salt/manager/sync_es_users.sls | 8 +++-- salt/salt/minion/boot_highstate.sls | 32 +++++++++++++++++++ salt/salt/minion/init.sls | 31 +++++++++++++++--- .../service/so-boot-highstate.service.jinja | 14 ++++++++ salt/setup/virt/setSalt.sls | 5 --- setup/so-functions | 15 +++++---- setup/so-setup | 2 +- 7 files changed, 88 insertions(+), 19 deletions(-) create mode 100644 salt/salt/minion/boot_highstate.sls create mode 100644 salt/salt/service/so-boot-highstate.service.jinja diff --git a/salt/manager/sync_es_users.sls b/salt/manager/sync_es_users.sls index 29b090e18..f452ff5fe 100644 --- a/salt/manager/sync_es_users.sls +++ b/salt/manager/sync_es_users.sls @@ -31,11 +31,13 @@ sync_es_users: - http: wait_for_kratos - file: so-user.lock # require so-user.lock file to be missing -# we dont want this added too early in setup, so we add the onlyif to verify 'startup_states: highstate' -# is in the minion config. That line is added before the final highstate during setup +# we dont want this added too early in setup, so the onlyif gates on the +# /opt/so/conf/setup-complete marker. The marker is written by +# mark_setup_complete in setup/so-functions just before the final setup +# highstate (and by an upgrade-path state for systems set up under the old gate). so-user_sync: cron.present: - user: root - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log' - identifier: so-user_sync - - onlyif: "grep -x 'startup_states: highstate' /etc/salt/minion" + - onlyif: "test -e /opt/so/conf/setup-complete" diff --git a/salt/salt/minion/boot_highstate.sls b/salt/salt/minion/boot_highstate.sls new file mode 100644 index 000000000..13bfbda65 --- /dev/null +++ b/salt/salt/minion/boot_highstate.sls @@ -0,0 +1,32 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Manages /etc/systemd/system/so-boot-highstate.service, a Type=oneshot +# RemainAfterExit=yes unit that runs `salt-call state.highstate` exactly once +# per system boot. Replaces the legacy `startup_states: highstate` minion +# config, which fired on every salt-minion service restart (causing a redundant +# highstate whenever a highstate itself restarted salt-minion). + +include: + - systemd.reload + +so_boot_highstate_unit_file: + file.managed: + - name: /etc/systemd/system/so-boot-highstate.service + - source: salt://salt/service/so-boot-highstate.service.jinja + - template: jinja + - onchanges_in: + - module: systemd_reload + +# Only enable once setup is complete. Until then the gate file is missing and +# the unit's own ConditionPathExists would no-op it anyway -- this just keeps +# `systemctl is-enabled` honest for the sync_es_users gate. +so_boot_highstate_service: + service.enabled: + - name: so-boot-highstate.service + - onlyif: test -e /opt/so/conf/setup-complete + - require: + - file: so_boot_highstate_unit_file + - module: systemd_reload diff --git a/salt/salt/minion/init.sls b/salt/salt/minion/init.sls index eb7018aed..0d0eed22c 100644 --- a/salt/salt/minion/init.sls +++ b/salt/salt/minion/init.sls @@ -17,6 +17,7 @@ include: - repo.client - salt.mine_functions - salt.minion.service_file + - salt.minion.boot_highstate {% if GLOBALS.is_manager %} - ca.signing_policy {% endif %} @@ -80,11 +81,33 @@ set_log_levels: - "log_level: info" - "log_level_logfile: info" -enable_startup_states: - file.uncomment: +# startup_states: highstate caused a full highstate to run on every +# salt-minion service start, including the restart triggered when a highstate +# itself modified the minion config (beacons, mine, unit file). Replaced by +# so-boot-highstate.service (managed in salt.minion.boot_highstate), which +# runs once per system boot only. Strip the line from /etc/salt/minion on +# upgrade; both the commented and uncommented forms historically existed. +remove_startup_states: + file.line: - name: /etc/salt/minion - - regex: '^startup_states: highstate$' - - unless: pgrep so-setup + - match: 'startup_states: highstate' + - mode: delete + +# Upgrade-path bridge: systems that already passed setup under the old gate +# (`grep -x 'startup_states: highstate' /etc/salt/minion`) get a setup-complete +# marker so so-boot-highstate.service can be enabled and the so-user_sync cron +# in sync_es_users.sls keeps installing. Setup-in-progress systems instead get +# the marker from `mark_setup_complete` in setup/so-functions at the right +# moment. `replace: false` means we never overwrite a marker once written. +mark_setup_complete_for_upgrades: + file.managed: + - name: /opt/so/conf/setup-complete + - replace: false + - makedirs: True + - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion" + - require_in: + - file: remove_startup_states + - service: so_boot_highstate_service {% endif %} diff --git a/salt/salt/service/so-boot-highstate.service.jinja b/salt/salt/service/so-boot-highstate.service.jinja new file mode 100644 index 000000000..f3ec950c3 --- /dev/null +++ b/salt/salt/service/so-boot-highstate.service.jinja @@ -0,0 +1,14 @@ +[Unit] +Description=Security Onion boot-time highstate (runs once per boot) +After=salt-minion.service network-online.target +Wants=network-online.target +Requires=salt-minion.service +ConditionPathExists=/opt/so/conf/setup-complete + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/bin/salt-call state.highstate -l info queue=True + +[Install] +WantedBy=multi-user.target diff --git a/salt/setup/virt/setSalt.sls b/salt/setup/virt/setSalt.sls index 69c8795de..59ab9e1e3 100644 --- a/salt/setup/virt/setSalt.sls +++ b/salt/setup/virt/setSalt.sls @@ -8,11 +8,6 @@ set_role_grain: - name: role - value: so-{{ grains.id.split("_") | last }} -set_highstate: - file.append: - - name: /etc/salt/minion - - text: 'startup_states: highstate' - enable_salt_minion: service.enabled: - name: salt-minion diff --git a/setup/so-functions b/setup/so-functions index c94b8eee7..da8e31d73 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -539,16 +539,19 @@ configure_minion() { " x509_v2: true"\ "log_level: info"\ "log_level_logfile: info"\ - "log_file: /opt/so/log/salt/minion"\ - "#startup_states: highstate" >> "$minion_config" + "log_file: /opt/so/log/salt/minion" >> "$minion_config" } -checkin_at_boot() { - local minion_config=/etc/salt/minion +mark_setup_complete() { + # Writes the setup-complete marker. Salt's so-boot-highstate.service + # (boot-time oneshot) and the so-user_sync cron gate in + # salt/manager/sync_es_users.sls both key off this file. + local marker=/opt/so/conf/setup-complete - info "Enabling checkin at boot" - sed -i 's/#startup_states: highstate/startup_states: highstate/' "$minion_config" + info "Marking setup as complete" + mkdir -p "$(dirname "$marker")" + touch "$marker" } check_requirements() { diff --git a/setup/so-setup b/setup/so-setup index 6c77e781c..c11d287eb 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -792,7 +792,7 @@ if ! [[ -f $install_opt_file ]]; then error "Failed to run so-elastic-fleet-setup" fail_setup fi - checkin_at_boot + mark_setup_complete set_initial_firewall_access initialize_elasticsearch_indices "so-case so-casehistory so-assistant-session so-assistant-chat" # run a final highstate before enabling scheduled highstates. From 2d653b6f1bd7111f1e4f02f9463742f3bce7a77a Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 3 Jun 2026 15:46:58 -0400 Subject: [PATCH 2/4] does not need to be jinja template --- salt/salt/minion/boot_highstate.sls | 3 +-- ...-boot-highstate.service.jinja => so-boot-highstate.service} | 0 2 files changed, 1 insertion(+), 2 deletions(-) rename salt/salt/service/{so-boot-highstate.service.jinja => so-boot-highstate.service} (100%) diff --git a/salt/salt/minion/boot_highstate.sls b/salt/salt/minion/boot_highstate.sls index 13bfbda65..e489210f6 100644 --- a/salt/salt/minion/boot_highstate.sls +++ b/salt/salt/minion/boot_highstate.sls @@ -15,8 +15,7 @@ include: so_boot_highstate_unit_file: file.managed: - name: /etc/systemd/system/so-boot-highstate.service - - source: salt://salt/service/so-boot-highstate.service.jinja - - template: jinja + - source: salt://salt/service/so-boot-highstate.service - onchanges_in: - module: systemd_reload diff --git a/salt/salt/service/so-boot-highstate.service.jinja b/salt/salt/service/so-boot-highstate.service similarity index 100% rename from salt/salt/service/so-boot-highstate.service.jinja rename to salt/salt/service/so-boot-highstate.service From 13f8be40b59e04d2de171b7ea93185dceddd5a32 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 4 Jun 2026 08:46:35 -0400 Subject: [PATCH 3/4] so-boot-highstate: wait for docker before running highstate Add docker.service to After= and Wants= so the boot-time highstate starts after docker is up. Uses Wants (soft) so highstate still runs if docker fails to start. --- salt/salt/service/so-boot-highstate.service | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/service/so-boot-highstate.service b/salt/salt/service/so-boot-highstate.service index f3ec950c3..a770122d6 100644 --- a/salt/salt/service/so-boot-highstate.service +++ b/salt/salt/service/so-boot-highstate.service @@ -1,7 +1,7 @@ [Unit] Description=Security Onion boot-time highstate (runs once per boot) -After=salt-minion.service network-online.target -Wants=network-online.target +After=salt-minion.service network-online.target docker.service +Wants=network-online.target docker.service Requires=salt-minion.service ConditionPathExists=/opt/so/conf/setup-complete From cb3631da818d1be7cb3a8c1e495751c2af0a8575 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 4 Jun 2026 15:07:27 -0400 Subject: [PATCH 4/4] Move setup-complete marker from /opt/so/conf to /opt/so/state The setup-complete marker is a runtime-state file, not config, so move it to /opt/so/state/setup-complete. Updates both writers (mark_setup_complete in setup/so-functions and the upgrade-path state in minion/init.sls) and the three readers (so-boot-highstate.service ConditionPathExists, boot_highstate.sls enable gate, and the so-user_sync cron gate). --- salt/manager/sync_es_users.sls | 4 ++-- salt/salt/minion/boot_highstate.sls | 2 +- salt/salt/minion/init.sls | 4 ++-- salt/salt/service/so-boot-highstate.service | 2 +- setup/so-functions | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/manager/sync_es_users.sls b/salt/manager/sync_es_users.sls index f452ff5fe..8fc9c6bb4 100644 --- a/salt/manager/sync_es_users.sls +++ b/salt/manager/sync_es_users.sls @@ -32,7 +32,7 @@ sync_es_users: - file: so-user.lock # require so-user.lock file to be missing # we dont want this added too early in setup, so the onlyif gates on the -# /opt/so/conf/setup-complete marker. The marker is written by +# /opt/so/state/setup-complete marker. The marker is written by # mark_setup_complete in setup/so-functions just before the final setup # highstate (and by an upgrade-path state for systems set up under the old gate). so-user_sync: @@ -40,4 +40,4 @@ so-user_sync: - user: root - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log' - identifier: so-user_sync - - onlyif: "test -e /opt/so/conf/setup-complete" + - onlyif: "test -e /opt/so/state/setup-complete" diff --git a/salt/salt/minion/boot_highstate.sls b/salt/salt/minion/boot_highstate.sls index e489210f6..eb2596dad 100644 --- a/salt/salt/minion/boot_highstate.sls +++ b/salt/salt/minion/boot_highstate.sls @@ -25,7 +25,7 @@ so_boot_highstate_unit_file: so_boot_highstate_service: service.enabled: - name: so-boot-highstate.service - - onlyif: test -e /opt/so/conf/setup-complete + - onlyif: test -e /opt/so/state/setup-complete - require: - file: so_boot_highstate_unit_file - module: systemd_reload diff --git a/salt/salt/minion/init.sls b/salt/salt/minion/init.sls index 0d0eed22c..59dd0289c 100644 --- a/salt/salt/minion/init.sls +++ b/salt/salt/minion/init.sls @@ -94,14 +94,14 @@ remove_startup_states: - mode: delete # Upgrade-path bridge: systems that already passed setup under the old gate -# (`grep -x 'startup_states: highstate' /etc/salt/minion`) get a setup-complete +# (`grep -x 'startup_states: highstate' /etc/salt/minion`) get a /opt/so/state/setup-complete # marker so so-boot-highstate.service can be enabled and the so-user_sync cron # in sync_es_users.sls keeps installing. Setup-in-progress systems instead get # the marker from `mark_setup_complete` in setup/so-functions at the right # moment. `replace: false` means we never overwrite a marker once written. mark_setup_complete_for_upgrades: file.managed: - - name: /opt/so/conf/setup-complete + - name: /opt/so/state/setup-complete - replace: false - makedirs: True - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion" diff --git a/salt/salt/service/so-boot-highstate.service b/salt/salt/service/so-boot-highstate.service index a770122d6..cc8c6a1c6 100644 --- a/salt/salt/service/so-boot-highstate.service +++ b/salt/salt/service/so-boot-highstate.service @@ -3,7 +3,7 @@ Description=Security Onion boot-time highstate (runs once per boot) After=salt-minion.service network-online.target docker.service Wants=network-online.target docker.service Requires=salt-minion.service -ConditionPathExists=/opt/so/conf/setup-complete +ConditionPathExists=/opt/so/state/setup-complete [Service] Type=oneshot diff --git a/setup/so-functions b/setup/so-functions index da8e31d73..5ce9a8fdc 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -547,7 +547,7 @@ mark_setup_complete() { # Writes the setup-complete marker. Salt's so-boot-highstate.service # (boot-time oneshot) and the so-user_sync cron gate in # salt/manager/sync_es_users.sls both key off this file. - local marker=/opt/so/conf/setup-complete + local marker=/opt/so/state/setup-complete info "Marking setup as complete" mkdir -p "$(dirname "$marker")"