Write setup-complete marker on non-manager nodes

so-boot-highstate.service was never enabled outside managers: only the
manager branch of so-setup called mark_setup_complete, so the marker its
service.enabled gates on never existed on sensors, search nodes, receivers,
etc.

Move the marker state into salt.minion.boot_highstate as the sole owner
within a highstate. Non-managers never apply salt.minion during setup, so
reaching it means setup is done and the marker is unconditional -- this also
heals already-installed nodes. Managers keep the legacy startup_states gate,
since they do highstate mid-setup.

Also add the marker to setup.virt for salt-cloud guests (replacing the
startup_states line removed in fabecb82) and to so-setup's non-manager branch.

salt/salt/minion/boot_highstate.sls

@@ -3,6 +3,8 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
 # Manages /etc/systemd/system/so-boot-highstate.service, a Type=oneshot
 # RemainAfterExit=yes unit that runs `salt-call state.highstate` exactly once
 # per system boot. Replaces the legacy `startup_states: highstate` minion
@@ -19,9 +21,45 @@ so_boot_highstate_unit_file:
     - onchanges_in:
       - module: systemd_reload
 
-# Only enable once setup is complete. Until then the gate file is missing and
-# the unit's own ConditionPathExists would no-op it anyway -- this just keeps
-# `systemctl is-enabled` honest for the sync_es_users gate.
+# Sole owner of the /opt/so/state/setup-complete marker within a highstate run.
+# (mark_setup_complete in setup/so-functions writes it directly during setup,
+# and setup.virt writes it for hypervisor guests, but neither of those is ever
+# part of a highstate.)
+#
+# Non-manager: salt.minion is never applied during setup. The non-manager branch
+# of setup/so-setup only reaches salt through --local applies of
+# salt.mine_functions, salt.python_modules, salt.patch.x509_v2 and libvirt.*;
+# the first highstate is pushed later by the manager via orch.deploy_newnode.
+# So if this state is rendering on a non-manager, setup is finished by
+# definition and the marker is unconditionally correct. This is what heals
+# nodes installed after the startup_states removal but before this fix, which
+# have neither the marker nor the legacy line the upgrade gate below keys off.
+#
+# Manager: managers DO apply salt.minion mid-setup (so-setup applies salt.master,
+# then the manager state whose init.sls includes salt.minion, then a full
+# highstate -- all before setup completes). Marking a manager complete there
+# would install the every-minute so-user_sync cron from manager/sync_es_users.sls
+# against a half-configured manager, racing so-user add for the so-user.lock.
+# Managers therefore only get the marker from the legacy upgrade signal; fresh
+# manager installs get it from mark_setup_complete in setup/so-functions at the
+# right moment.
+#
+# replace: false means an existing marker is never rewritten.
+mark_setup_complete:
+  file.managed:
+    - name: /opt/so/state/setup-complete
+    - replace: false
+    - makedirs: True
+{% if GLOBALS.is_manager %}
+    - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion"
+{% endif %}
+    - require_in:
+      - service: so_boot_highstate_service
+
+# Only enable once setup is complete. The unit's own ConditionPathExists already
+# no-ops it before then, so this is belt-and-braces; it also preserves the
+# operator-facing invariant that an enabled so-boot-highstate.service means the
+# node finished setup.
 so_boot_highstate_service:
   service.enabled:
     - name: so-boot-highstate.service

salt/salt/minion/init.sls

@@ -87,27 +87,17 @@ set_log_levels:
 # so-boot-highstate.service (managed in salt.minion.boot_highstate), which
 # runs once per system boot only. Strip the line from /etc/salt/minion on
 # upgrade; both the commented and uncommented forms historically existed.
+#
+# Ordered after mark_setup_complete (salt.minion.boot_highstate) because on a
+# manager that state's gate greps for this very line -- deleting it first would
+# lose the upgrade bridge that lets an upgraded manager enable the boot unit.
 remove_startup_states:
   file.line:
     - name: /etc/salt/minion
     - match: 'startup_states: highstate'
     - mode: delete
-
-# Upgrade-path bridge: systems that already passed setup under the old gate
-# (`grep -x 'startup_states: highstate' /etc/salt/minion`) get a /opt/so/state/setup-complete
-# marker so so-boot-highstate.service can be enabled and the so-user_sync cron
-# in sync_es_users.sls keeps installing. Setup-in-progress systems instead get
-# the marker from `mark_setup_complete` in setup/so-functions at the right
-# moment. `replace: false` means we never overwrite a marker once written.
-mark_setup_complete_for_upgrades:
-  file.managed:
-    - name: /opt/so/state/setup-complete
-    - replace: false
-    - makedirs: True
-    - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion"
-    - require_in:
-      - file: remove_startup_states
-      - service: so_boot_highstate_service
+    - require:
+      - file: mark_setup_complete
 
 {% endif %}
 

salt/setup/virt/setSalt.sls

@@ -8,6 +8,18 @@ set_role_grain:
     - name: role
     - value: so-{{ grains.id.split("_") | last }}
 
+# Hypervisor guests provisioned by salt-cloud never run setup/so-setup, so
+# nothing else marks them setup-complete at provisioning time. Replaces the
+# 'startup_states: highstate' line this state used to append; the marker gates
+# so-boot-highstate.service (see salt/salt/minion/boot_highstate.sls).
+# Deliberately no GLOBALS import -- this state tree runs before the guest's
+# pillars exist (see the comment at the top of salt/common/packages.sls).
+mark_setup_complete_vm_guest:
+  file.managed:
+    - name: /opt/so/state/setup-complete
+    - replace: false
+    - makedirs: True
+
 enable_salt_minion:
   service.enabled:
     - name: salt-minion

setup/so-setup

@@ -833,6 +833,7 @@ if ! [[ -f $install_opt_file ]]; then
 		check_sos_appliance
 		drop_install_options
 		hypervisor_local_states
+		mark_setup_complete
 		verify_setup
 	fi