add logrotate for virtual_node_manager and so-salt-cloud

.github/DISCUSSION_TEMPLATE/3-0.yml

@@ -11,7 +11,6 @@ body:
         -
         - 3.0.0
         - 3.1.0
-        - 3.2.0
         - Other (please provide detail below)
     validations:
       required: true

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 3.1.0-20260528 ISO image released on 2026/05/28
+### 3.0.0-20260331 ISO image released on 2026/03/31
 
 
 ### Download and Verify
 
-3.1.0-20260528 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-3.1.0-20260528.iso
+3.0.0-20260331 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
  
-MD5: 9D6FF58DEEE24089D722C73169765B3E  
-SHA1: 2B8B816B6CEC3B7F96B3C5E040EBF502DD2C412F  
-SHA256: 62FAB57E247C843D6A04F0796D8162C732B65D82FC3E4A59D087135B9FD32912  
+MD5: ECD318A1662A6FDE0EF213F5A9BD4B07  
+SHA1: E55BE314440CCF3392DC0B06BC5E270B43176D9C  
+SHA256: 7FC47405E335CBE5C2B6C51FE7AC60248F35CBE504907B8B5A33822B23F8F4D5  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.1.0-20260528.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.1.0-20260528.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-3.1.0-20260528.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-3.1.0-20260528.iso.sig securityonion-3.1.0-20260528.iso
+gpg --verify securityonion-3.0.0-20260331.iso.sig securityonion-3.0.0-20260331.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Wed 27 May 2026 03:03:59 PM EDT using RSA key ID FE507013
+gpg: Signature made Mon 30 Mar 2026 06:22:14 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

HOTFIX

@@ -1 +0,0 @@
-

VERSION

@@ -1 +1 @@
-3.2.0
+3.1.0

salt/backup/tools/sbin/so-config-backup.jinja

@@ -25,11 +25,9 @@ if [ ! -f $BACKUPFILE ]; then
   # Create empty backup file
   tar -cf $BACKUPFILE -T /dev/null
 
-  # Loop through all paths defined in global.sls, and append them to backup file if they exist
+  # Loop through all paths defined in global.sls, and append them to backup file
   {%- for LOCATION in BACKUPLOCATIONS %}
-  if [[ -d {{ LOCATION }} || -f {{ LOCATION }} ]]; then
-    tar -rf $BACKUPFILE "${EXCLUSIONS[@]}" {{ LOCATION }}
-  fi
+  tar -rf $BACKUPFILE "${EXCLUSIONS[@]}" {{ LOCATION }}
   {%- endfor %}
 
 fi

salt/common/init.sls

@@ -130,17 +130,6 @@ common_sbin:
       - so-pcap-import
 {% endif %}
 
-# Pin physical NIC names by MAC (run-once) so a kernel upgrade can't renumber the
-# interfaces SO binds by name. The marker keeps it a one-time setup; an admin can
-# pre-create the marker to opt out.
-pin_nic_names:
-  cmd.run:
-    - name: /usr/sbin/so-nic-pin
-    - unless: 'test -e /opt/so/state/nic_names_pinned'
-    - require:
-      - file: common_sbin
-      - file: statedir
-
 common_sbin_jinja:
   file.recurse:
     - name: /usr/sbin

salt/common/tools/sbin/so-nic-pin

@@ -1,76 +0,0 @@
-#!/bin/bash
-#
-# so-nic-pin — pin physical NIC names by permanent MAC via classic by-MAC udev
-#              rules, so a kernel upgrade can't renumber them.
-#
-# Security Onion binds its management and monitor interfaces BY NAME in pillar
-# (host:mainint, sensor:mainint, and bond0 is built on a specific physical NIC).
-# A kernel upgrade can change the kernel/systemd-udevd predictable-naming output
-# and renumber those NICs (e.g. enp1s0 -> enp2s0), which breaks the grid: the
-# pillar references a name that no longer exists and bond/bridge bring-up fails.
-#
-# This writes /etc/udev/rules.d/70-persistent-net.rules pinning each PHYSICAL NIC
-# to its CURRENT name by its PERMANENT MAC, freezing the names across future kernel
-# changes. It only writes the rules file; it does NOT live-trigger a rename (the
-# rules apply on the next boot/kernel, and a live rename would be disruptive).
-#
-# Run-once: gated by the drop file /opt/so/state/nic_names_pinned. If the marker is
-# present the script does nothing, so an admin can pre-create it to opt out. Invoked
-# from the common state on every highstate; the marker keeps it a one-time setup.
-
-NET_RULES_FILE="/etc/udev/rules.d/70-persistent-net.rules"
-MARKER="/opt/so/state/nic_names_pinned"
-
-log() { echo -e "[so-nic-pin] $*"; }
-
-# Echo "<name> <permanent-mac>" for every PHYSICAL NIC. A physical NIC is backed by a
-# real device (has device/driver), which excludes bond0/sobridge/docker0/veth*/lo whose
-# MACs are dynamic and must never be pinned. The PERMANENT MAC is used (ethtool -P, with
-# fallbacks), not the current one: an enslaved bond member's current MAC is rewritten to
-# the bond's, so matching on it would be wrong/ambiguous.
-physical_nics() {
-    local path n mac
-    for path in /sys/class/net/*; do
-        n="${path##*/}"
-        [ "$n" = "lo" ] && continue
-        [ -e "${path}/device/driver" ] || continue          # real device only
-        mac="$(ethtool -P "$n" 2>/dev/null | awk '/Permanent address/{print $NF}')"
-        case "$mac" in ""|00:00:00:00:00:00) mac="$(cat "${path}/bonding_slave/perm_hwaddr" 2>/dev/null)" ;; esac
-        case "$mac" in ""|00:00:00:00:00:00) mac="$(cat "${path}/address" 2>/dev/null)" ;; esac
-        case "$mac" in ""|00:00:00:00:00:00) continue ;; esac
-        echo "$n $mac"
-    done
-}
-
-# Turn "<name> <mac>" lines on stdin into classic by-MAC persistent-net udev rules.
-render_net_rules() {
-    echo "# Generated by so-nic-pin: pin NIC names by MAC so kernel upgrades can't renumber them."
-    echo "# Security Onion binds its management/monitor interfaces by name; do not hand-edit."
-    local n mac
-    while read -r n mac; do
-        [ -n "$n" ] || continue
-        printf 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="%s", NAME="%s"\n' \
-            "$mac" "$n"
-    done
-}
-
-[ "$(id -u)" -eq 0 ] || exit 0                   # salt runs us as root; bail quietly otherwise
-[ -e "${MARKER}" ] && exit 0                      # run-once guard (mirrors the state's unless)
-
-nics="$(physical_nics)"
-if [ -z "${nics}" ]; then
-    log "no physical NICs detected — nothing to pin (will retry on next highstate)"
-    exit 0                                         # do NOT drop the marker; let it retry later
-fi
-
-log "pinning physical NICs by permanent MAC:"
-echo "${nics}" | sed 's/^/    /'
-
-[ -f "${NET_RULES_FILE}" ] && cp -f "${NET_RULES_FILE}" "${NET_RULES_FILE}.bak"
-echo "${nics}" | render_net_rules > "${NET_RULES_FILE}" || {
-    log "ERROR: failed to write ${NET_RULES_FILE}"
-    exit 1
-}
-
-mkdir -p "$(dirname "${MARKER}")" && touch "${MARKER}"
-log "wrote ${NET_RULES_FILE} ($(grep -c '^SUBSYSTEM' "${NET_RULES_FILE}") NIC(s) pinned); dropped ${MARKER}"

salt/elasticfleet/enabled.sls

@@ -26,9 +26,7 @@ include:
 wait_for_elasticsearch_elasticfleet:
   cmd.run:
     - name: so-elasticsearch-wait
-{% endif %}
 
-{% if GLOBALS.role == "so-fleet" %}
 # Sync Elastic Agent artifacts to Fleet Node
 elasticagent_syncartifacts:
   file.recurse:
@@ -101,17 +99,6 @@ so-elastic-fleet:
       - file: trusttheca
       - x509: etc_elasticfleet_key
       - x509: etc_elasticfleet_crt
-
-wait_for_so-elastic-fleet:
-  http.wait_for_successful_query:
-    - name: "https://localhost:8220/api/status"
-    - ssl: True
-    - verify_ssl: False
-    - status: 200
-    - wait_for: 300
-    - request_interval: 15
-    - require:
-      - docker_container: so-elastic-fleet
 {%   endif %}
 
 delete_so-elastic-fleet_so-status.disabled:

salt/elasticfleet/manager.sls

@@ -9,20 +9,16 @@
 
 include:
   - elasticfleet.config
-  - kibana.enabled
 
 # If enabled, automatically update Fleet Logstash Outputs
-{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration %}
-{%   if grains.role not in ['so-import', 'so-eval']%}
+{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval'] %}
 so-elastic-fleet-auto-configure-logstash-outputs:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-outputs-update
     - retry:
         attempts: 4
         interval: 30
-    - require:
-      - http: wait_for_so-kibana
-{%   endif %}
+{% endif %}
 
 # If enabled, automatically update Fleet Server URLs & ES Connection
 so-elastic-fleet-auto-configure-server-urls:
@@ -31,9 +27,6 @@ so-elastic-fleet-auto-configure-server-urls:
     - retry:
         attempts: 4
         interval: 30
-    - require:
-      - http: wait_for_so-kibana
-{% endif %}
 
 # Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
 so-elastic-fleet-auto-configure-elasticsearch-urls:
@@ -42,8 +35,6 @@ so-elastic-fleet-auto-configure-elasticsearch-urls:
     - retry:
         attempts: 4
         interval: 30
-    - require:
-      - http: wait_for_so-kibana
 
 so-elastic-fleet-auto-configure-artifact-urls:
   cmd.run:
@@ -51,8 +42,6 @@ so-elastic-fleet-auto-configure-artifact-urls:
     - retry:
         attempts: 4
         interval: 30
-    - require:
-      - http: wait_for_so-kibana
 
 so-elastic-fleet-package-statefile:
   file.managed:
@@ -64,9 +53,7 @@ so-elastic-fleet-package-upgrade:
     - name: /usr/sbin/so-elastic-fleet-package-upgrade
     - retry:
         attempts: 3
-        interval: 30
-    - require:
-      - http: wait_for_so-kibana
+        interval: 10
     - onchanges:
       - file: /opt/so/state/elastic_fleet_packages.txt
 
@@ -76,8 +63,6 @@ so-elastic-fleet-integrations:
     - retry:
         attempts: 3
         interval: 10
-    - require:
-      - http: wait_for_so-kibana
 
 so-elastic-agent-grid-upgrade:
   cmd.run:
@@ -85,8 +70,6 @@ so-elastic-agent-grid-upgrade:
     - retry:
         attempts: 12
         interval: 5
-    - require:
-      - http: wait_for_so-kibana
 
 so-elastic-fleet-integration-upgrade:
   cmd.run:
@@ -94,22 +77,16 @@ so-elastic-fleet-integration-upgrade:
     - retry:
         attempts: 3
         interval: 10
-    - require:
-      - http: wait_for_so-kibana
 
 {# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
 so-elastic-fleet-addon-integrations:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
-    - require:
-      - http: wait_for_so-kibana
 
 {% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
 so-elastic-defend-manage-filters-file-watch:
   cmd.run:
     - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log
-    - require:
-      - http: wait_for_so-kibana
     - onchanges:
       - file: elasticdefendcustom
       - file: elasticdefenddisabled

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -108,12 +108,9 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
   done
 
   # Only create the state file if all policies were created/updated successfully
-  if [[ $RETURN_CODE -eq 0 ]]; then
+  if [[ "$RETURN_CODE" != "1" ]]; then
     touch /opt/so/state/eaintegrations.txt
-  else
-    exit 1
   fi
 else
-  echo "Fleet integration policies already loaded."
-  exit 0
+  exit $RETURN_CODE
 fi

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade

@@ -8,33 +8,18 @@
 
 . /usr/sbin/so-elastic-fleet-common
 
-PKG_LOAD_FAILURES=0
-PKG_LOAD_FAILURES_NAMES=()
-
 {%- for PACKAGE in SUPPORTED_PACKAGES %}
 echo "Upgrading {{ PACKAGE }} package..."
 if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
     if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
-        PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
-        PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
+        # exit 1 on failure to upgrade a default package, allow salt to handle retries
+        echo -e "\nERROR: Failed to upgrade $PACKAGE to version: $VERSION"
+        exit 1
     fi
 else
-    PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
-    PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
+    echo -e "\nERROR: Failed to get version information for integration $PACKAGE"
 fi
 echo
 {%- endfor %}
-
-if [ $PKG_LOAD_FAILURES -gt 0 ]; then
-    echo "ERROR: Failed to upgrade $PKG_LOAD_FAILURES package(s):"
-    for PKG in "${PKG_LOAD_FAILURES_NAMES[@]}"; do
-        echo " - $PKG"
-    done
-    # exit 1 on failure to upgrade a default package, allow salt to handle retries
-    exit 1
-else
-    echo "Successfully upgraded all packages."
-fi
-
 echo
 /usr/sbin/so-elasticsearch-templates-load

salt/elasticsearch/cluster.sls

@@ -9,12 +9,9 @@
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %}
 {%   if GLOBALS.role != 'so-heavynode' %}
-{%     from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS, ADDON_INDICES %}
+{%     from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
 {%   endif %}
 
-include:
-  - elasticsearch.enabled
-
 escomponenttemplates:
   file.recurse:
     - name: /opt/so/conf/elasticsearch/templates/component
@@ -38,20 +35,6 @@ so_index_template_dir:
       {%- endfor %}
     {%- endif %}
 
-{%  if GLOBALS.role != "so-heavynode" %}
-# Clean up legacy and non-SO managed templates from the elasticsearch/templates/addon-index/ directory
-addon_index_template_dir:
-  file.directory:
-    - name: /opt/so/conf/elasticsearch/templates/addon-index
-    - clean: True
-    {%- if ADDON_INDICES %}
-    - require:
-      {%- for index in ADDON_INDICES %}
-      - file: addon_index_template_{{index}}
-      {%- endfor %}
-    {%- endif %}
-{%  endif %}
-
 # Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml)
 #   These index templates are for the core SO datasets and are always required
 {%  for index, settings in ES_INDEX_SETTINGS.items() %}

salt/elasticsearch/template.map.jinja

@@ -61,25 +61,15 @@
 {% if ALL_ADDON_SETTINGS_ORIG.keys() | length > 0 %}
 {%   for index in ALL_ADDON_SETTINGS_ORIG.keys() %}
 {%     do ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ALL_ADDON_SETTINGS_ORIG[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}
-{#     Explicitly excluding addon indices from ES_INDEX_SETTINGS_ORIG
-         When manager.soc_managed_annotations runs, new entries are added to the salt/elasticsearch/defaults.yaml file to support 'revert to default' functionality.
-         Subsequent map renders will then incorrectly include 'integration X' in 'ES_INDEX_SETTINGS_ORIG' due to being in the defaults.yaml file. #}
-{%     if index in ES_INDEX_SETTINGS_ORIG.keys() %}
-{%       do ES_INDEX_SETTINGS_ORIG.pop(index) %}
-{%     endif %}
 {%   endfor %}
 {% endif %}
 
 {% set ES_INDEX_SETTINGS = {} %}
-{% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS, EXCLUDE_INDICES=[]) %}
+{% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS) %}
 
 {% do GLOBAL_OVERRIDES.update(salt['defaults.merge'](GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %}
 {% for index, settings in GLOBAL_OVERRIDES.items() %}
 
-{%   if index in EXCLUDE_INDICES %}
-{%     continue %}
-{%   endif %}
-
 {#   prevent this action from being performed on custom defined indices. #}
 {#   the custom defined index is not present in either of the dictionaries and fails to reder. #}
 {%   if index in DEFINED_SETTINGS and index in GLOBAL_OVERRIDES %}
@@ -160,19 +150,10 @@
 {% endfor %}
 {% endmacro %}
 
-{# Exclude addon integrations from final ES_INDEX_SETTINGS #}
-{{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS, ALL_ADDON_SETTINGS_ORIG.keys() | list ) }}
-
-{# Exclude SO managed indices, otherwise ALL_ADDON_SETTINGS will include pillar values
-  of core integrations without merging defaults, resulting in an overlapping, but bad index template being generated. #}
-{{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS, ES_INDEX_SETTINGS_ORIG.keys() | list ) }}
+{{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS) }}
+{{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS) }}
 
 {% set SO_MANAGED_INDICES = [] %}
 {% for index, settings in ES_INDEX_SETTINGS.items() %}
 {%   do SO_MANAGED_INDICES.append(index) %}
-{% endfor %}
-
-{% set ADDON_INDICES = [] %}
-{% for index, settings in ALL_ADDON_SETTINGS.items() %}
-{%   do ADDON_INDICES.append(index) %}
-{% endfor %}
+{% endfor %}

salt/kibana/enabled.sls

@@ -6,7 +6,6 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
-{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 
 include:
@@ -61,19 +60,6 @@ so-kibana:
     - watch:
       - file: kibanaconfig
 
-wait_for_so-kibana:
-  http.wait_for_successful_query:
-    - name: "http://localhost:5601/api/status"
-    - username: 'so_elastic'
-    - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}'
-    - ssl: True
-    - verify_ssl: False
-    - status: 200
-    - wait_for: 300
-    - request_interval: 15
-    - require:
-      - docker_container: so-kibana
-
 delete_so-kibana_so-status.disabled:
   file.uncomment:
     - name: /opt/so/conf/so-status/so-status.conf

salt/kratos/soc_kratos.yaml

@@ -103,7 +103,7 @@ kratos:
   config:
     session:
       lifespan: 
-        description: Defines the length of a login session before it will timeout, and require a new login.
+        description: Defines the length of a login session.
         global: True
         helpLink: kratos
       whoami:

salt/logrotate/defaults.yaml

@@ -210,6 +210,26 @@ logrotate:
       - extension .log
       - dateext
       - dateyesterday
+    /opt/so/log/salt/virtual_node_manager:
+      - daily
+      - rotate 14
+      - missingok
+      - copytruncate
+      - compress
+      - create
+      - extension .log
+      - dateext
+      - dateyesterday
+    /opt/so/log/salt/so-salt-cloud:
+      - daily
+      - rotate 14
+      - missingok
+      - copytruncate
+      - compress
+      - create
+      - extension .log
+      - dateext
+      - dateyesterday
     /nsm/idh/*_x_log:
       - daily
       - rotate 14

salt/logrotate/soc_logrotate.yaml

@@ -133,6 +133,20 @@ logrotate:
       multiline: True
       global: True
       forcedType: "[]string"
+    "/opt/so/log/salt/virtual_node_manager":
+      description: List of logrotate options for this file.
+      title: /opt/so/log/salt/virtual_node_manager
+      advanced: True
+      multiline: True
+      global: True
+      forcedType: "[]string"
+    "/opt/so/log/salt/so-salt-cloud":
+      description: List of logrotate options for this file.
+      title: /opt/so/log/salt/so-salt-cloud
+      advanced: True
+      multiline: True
+      global: True
+      forcedType: "[]string"
     "/nsm/idh/*_x_log":
       description: List of logrotate options for this file.
       title: /nsm/idh/*.log

salt/manager/sync_es_users.sls

@@ -31,13 +31,11 @@ sync_es_users:
       - http: wait_for_kratos
       - file: so-user.lock # require so-user.lock file to be missing
 
-# we dont want this added too early in setup, so the onlyif gates on the
-# /opt/so/state/setup-complete marker. The marker is written by
-# mark_setup_complete in setup/so-functions just before the final setup
-# highstate (and by an upgrade-path state for systems set up under the old gate).
+# we dont want this added too early in setup, so we add the onlyif to verify 'startup_states: highstate'
+# is in the minion config. That line is added before the final highstate during setup
 so-user_sync:
   cron.present:
     - user: root
     - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log'
     - identifier: so-user_sync
-    - onlyif: "test -e /opt/so/state/setup-complete"
+    - onlyif: "grep -x 'startup_states: highstate' /etc/salt/minion"

salt/manager/tools/sbin/so-boot-mine-update

@@ -1,117 +0,0 @@
-#!/bin/bash
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Runs once per boot on managers (via so-boot-mine-update.service), before
-# so-boot-highstate.service. Waits for the responsive minion set to settle, pushes
-# mine.update, waits until every up minion has actually reported to the mine, then
-# warms the master's per-minion pillar cache so the mine-backed node pillars (node
-# IPs, ES/Redis/Logstash/hypervisor discovery -- some glob- and some pillar/grain-
-# targeted) are complete before the boot highstate renders them. Otherwise a node
-# that is up but not yet fully reported gets dropped from those pillars and torn
-# out of the configs they build (e.g. so-elasticsearch ExtraHosts -> container recreate).
-
-MAX_WAIT=${MINE_UPDATE_MAX_WAIT:-180}   # hard backstop only
-INTERVAL=10
-STABLE_CHECKS=3                          # up-count must hold steady this many polls
-elapsed=0
-prev=-1
-stable=0
-up=0
-
-# Wait for the *reachable* minion set to settle rather than for every accepted
-# key to report up: an operator may accept a minion's key and then intentionally
-# power off that host, so requiring up >= accepted would never be satisfied and
-# we'd always burn the full MAX_WAIT. Once the responsive count stops growing we
-# stop waiting and run mine.update against whoever is up.
-while [ "$elapsed" -lt "$MAX_WAIT" ]; do
-  up=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null \
-    | python3 -c 'import sys,json; print(len(json.load(sys.stdin)))' 2>/dev/null)
-  up=${up:-0}
-  if [ "$up" -gt 0 ] && [ "$up" -eq "$prev" ]; then
-    stable=$((stable + 1))
-    [ "$stable" -ge "$STABLE_CHECKS" ] && break
-  else
-    stable=0
-  fi
-  prev=$up
-  sleep "$INTERVAL"
-  elapsed=$((elapsed + INTERVAL))
-done
-
-echo "so-boot-mine-update: ${up} minions up (settled after ${elapsed}s); running mine.update"
-/usr/bin/salt '*' mine.update --out=txt
-
-# A node that is up but has not yet re-reported network.ip_addrs to the mine is
-# silently dropped from mine-backed pillars (elasticsearch:nodes, node_data, ...)
-# when highstate recompiles them -- which e.g. removes it from so-elasticsearch
-# ExtraHosts and forces a container recreate. After the broad mine.update above,
-# wait until every up minion actually has network.ip_addrs in the mine, re-pushing
-# mine.update to stragglers, before releasing the boot highstate. Bounded by the
-# same MAX_WAIT backstop so a slow/down node never blocks boot indefinitely.
-missing=""
-while [ "$elapsed" -lt "$MAX_WAIT" ]; do
-  up_json=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null)
-  mine_json=$(/usr/bin/salt-run mine.get '*' network.ip_addrs tgt_type=glob --out=json 2>/dev/null)
-  missing=$(printf '%s' "$up_json" | python3 -c '
-import sys, json
-up = set(json.load(sys.stdin) or [])
-mine = {k for k, v in (json.loads(sys.argv[1]) or {}).items() if v}
-print("\n".join(sorted(up - mine)))
-' "$mine_json" 2>/dev/null)
-  if [ -z "$missing" ]; then
-    echo "so-boot-mine-update: mine complete for all up minions after ${elapsed}s"
-    break
-  fi
-  echo "so-boot-mine-update: mine missing up minion(s): $(echo $missing); re-running mine.update"
-  for m in $missing; do /usr/bin/salt "$m" mine.update --out=txt; done
-  sleep "$INTERVAL"
-  elapsed=$((elapsed + INTERVAL))
-done
-[ -n "$missing" ] && echo "so-boot-mine-update: WARNING ${MAX_WAIT}s backstop hit; up minion(s) still absent from mine: $(echo $missing); highstate may drop them from configs"
-
-# The pillar/compound-targeted node pillars (elasticsearch:nodes, redis:nodes,
-# logstash:nodes, hypervisor:nodes) resolve their target against the master's
-# per-minion data cache (grains+pillar in .../minions/<id>/data.p), populated only
-# when a minion's pillar is (re)compiled -- separately from the mine. A freshly
-# booted node can be in the mine (glob/node_data sees it) yet absent from that
-# cache, so it is dropped from those pillars and from the configs they build (e.g.
-# so-elasticsearch ExtraHosts). Force a synchronous pillar refresh so the master
-# caches every up node's pillar; refresh_pillar wait=True returns only once the
-# pillar is recompiled (and thus cached for matching). Retry stragglers <= MAX_WAIT.
-echo "so-boot-mine-update: warming master pillar cache for pillar/grain-targeted node pillars"
-/usr/bin/salt '*' saltutil.refresh_pillar wait=True --out=txt
-missing=""
-while [ "$elapsed" -lt "$MAX_WAIT" ]; do
-  up_json=$(/usr/bin/salt-run manage.up --out=json 2>/dev/null)
-  cached_json=$(/usr/bin/salt-run cache.pillar tgt='*' --out=json 2>/dev/null)
-  missing=$(printf '%s' "$up_json" | python3 -c '
-import sys, json
-up = set(json.load(sys.stdin) or [])
-cached = {k for k, v in (json.loads(sys.argv[1]) or {}).items() if v}
-print("\n".join(sorted(up - cached)))
-' "$cached_json" 2>/dev/null)
-  if [ -z "$missing" ]; then
-    echo "so-boot-mine-update: pillar cache warm for all up minions after ${elapsed}s"
-    break
-  fi
-  echo "so-boot-mine-update: pillar not yet cached for: $(echo $missing); refreshing"
-  for m in $missing; do /usr/bin/salt "$m" saltutil.refresh_pillar wait=True --out=txt; done
-  sleep "$INTERVAL"
-  elapsed=$((elapsed + INTERVAL))
-done
-[ -n "$missing" ] && echo "so-boot-mine-update: WARNING ${MAX_WAIT}s backstop hit; pillar not cached for: $(echo $missing); pillar-targeted pillars may drop them"
-
-# Log what the mine-backed pillars render so the boot-time state is inspectable.
-/usr/bin/salt-call saltutil.refresh_pillar >/dev/null 2>&1
-sleep 2
-for key in node_data elasticsearch:nodes; do
-  rendered=$(/usr/bin/salt-call --out=json pillar.get "$key" 2>/dev/null \
-    | python3 -c 'import sys,json; print(json.dumps(json.load(sys.stdin).get("local"), indent=2, sort_keys=True))' 2>/dev/null)
-  echo "so-boot-mine-update: ${key} rendered as:"
-  echo "${rendered:-null}"
-done
-exit 0

salt/manager/tools/sbin/soup

@@ -188,6 +188,13 @@ airgap_update_dockers() {
   fi
 }
 
+backup_old_states_pillars() {
+
+	tar czf /nsm/backup/$(echo $INSTALLEDVERSION)_$(date +%Y%m%d-%H%M%S)_soup_default_states_pillars.tar.gz /opt/so/saltstack/default/
+	tar czf /nsm/backup/$(echo $INSTALLEDVERSION)_$(date +%Y%m%d-%H%M%S)_soup_local_states_pillars.tar.gz /opt/so/saltstack/local/
+
+}
+
 update_registry() {
   docker stop so-dockerregistry
   docker rm so-dockerregistry
@@ -363,9 +370,8 @@ preupgrade_changes() {
     # This function is to add any new pillar items if needed.
     echo "Checking to see if changes are needed."
 
-    [[ "$INSTALLEDVERSION" =~ ^2\.4\.21[0-9]+$ ]] && up_to_3.0.0
+    [[ "$INSTALLEDVERSION" =~ ^2\.4\.21[0-9]+$ ]] && up_to_3.0.0   
     [[ "$INSTALLEDVERSION" == "3.0.0" ]] && up_to_3.1.0
-    [[ "$INSTALLEDVERSION" == "3.1.0" ]] && up_to_3.2.0
     true
 }
 
@@ -375,7 +381,6 @@ postupgrade_changes() {
 
     [[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0
     [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0
-    [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0
     true
 }
 
@@ -528,23 +533,6 @@ elasticfleet_set_agent_logging_level_warn() {
     done <<< "$policies_to_update"
 }
 
-update_logstash_pipeline_name() {
-    local original_pipeline_name="$1"
-    local new_pipeline_name="$2"
-
-    echo "Checking for conflicting logstash defined_pipelines pillar value."
-    local LOGSTASH_FILE=/opt/so/saltstack/local/pillar/logstash/soc_logstash.sls
-    local MINIONDIR=/opt/so/saltstack/local/pillar/minions
-    for pillar_file in "$LOGSTASH_FILE" "$MINIONDIR"/*.sls; do
-        [[ -f "$pillar_file" ]] || continue
-        if grep -q "$original_pipeline_name$" "$pillar_file"; then
-            echo "Found conflicting defined_pipeline pillar value in $pillar_file. Updating to use the new logstash pipeline name."
-            sed -i "s#$original_pipeline_name\$#$new_pipeline_name#g" "$pillar_file"
-            chown socore:socore "$pillar_file"
-        fi
-    done
-}
-
 check_transform_health_and_reauthorize() {
     . /usr/sbin/so-elastic-fleet-common
 
@@ -688,10 +676,6 @@ rename_strelka_scan_lnk() {
   rm -f "$TMP_VALUE_FILE"
 }
 
-fix_logstash_0013_lumberjack_pipeline_name() {
-    update_logstash_pipeline_name "so/0013_input_lumberjack_fleet.conf" "so/0013_input_lumberjack_fleet.conf.jinja"
-}
-
 up_to_3.1.0() {
   ensure_postgres_local_pillar
   ensure_postgres_secret
@@ -700,7 +684,6 @@ up_to_3.1.0() {
   # Clear existing component template state file.
   rm -f /opt/so/state/esfleet_component_templates.json
   rename_strelka_scan_lnk
-  fix_logstash_0013_lumberjack_pipeline_name
 
   INSTALLEDVERSION=3.1.0
 }
@@ -737,48 +720,6 @@ post_to_3.1.0() {
 
 ### 3.1.0 End ###
 
-### 3.2.0 Scripts ###
-
-bootstrap_so_soc_database() {
-  # init-db.sh is mounted into so-postgres at /docker-entrypoint-initdb.d/init-db.sh
-  # and runs automatically only on a fresh data directory. Hosts upgrading from
-  # 3.1.0 already have /nsm/postgres populated, so the so_soc bootstrap block
-  # added in 3.2 never fires. Re-run the script explicitly; it's idempotent.
-  echo "Bootstrapping so_soc database via init-db.sh."
-  # The postgres image has no USER directive, so `docker exec` defaults to
-  # root, and the container env intentionally omits POSTGRES_USER (the upstream
-  # entrypoint defaults it transiently during first-init only). Recreate both
-  # so psql inside init-db.sh resolves the connect user correctly.
-  local exec_cmd="docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh"
-  if ! /usr/sbin/so-postgres-wait; then
-    FINAL_MESSAGE_QUEUE+=("WARNING: so-postgres was not ready during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd")
-    return 0
-  fi
-  if ! $exec_cmd; then
-    FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd")
-    return 0
-  fi
-  echo "so_soc bootstrap complete."
-}
-
-up_to_3.2.0() {
-  fix_logstash_0013_lumberjack_pipeline_name
-
-  INSTALLEDVERSION=3.2.0
-}
-
-post_to_3.2.0() {
-  bootstrap_so_soc_database
-
-  # Including agent regen script here since it was missed in post_to_3.1.0
-  echo "Regenerating Elastic Agent Installers"
-  /sbin/so-elastic-agent-gen-installers
-
-  POSTVERSION=3.2.0
-}
-
-### 3.2.0 End ###
-
 
 repo_sync() {
   echo "Sync the local repo."
@@ -1030,9 +971,6 @@ verify_es_version_compatibility() {
     local is_active_intermediate_upgrade=1
     # supported upgrade paths for SO-ES versions
     declare -A es_upgrade_map=(
-        ["8.18.4"]="8.18.6 8.18.8 9.0.8"
-	    ["8.18.6"]="8.18.8 9.0.8"
-	    ["8.18.8"]="9.0.8"
         ["9.0.8"]="9.3.3"
     )
 
@@ -1056,171 +994,6 @@ verify_es_version_compatibility() {
         exit 160
     fi
 
-    compatible_es_versions="$target_es_version"
-    for current_version in "${!es_upgrade_map[@]}"; do
-        # shellcheck disable=SC2076
-        if [[ " ${es_upgrade_map[$current_version]} " =~ " $target_es_version " ]]; then
-            compatible_es_versions+=" $current_version"
-        fi
-    done
-
-    # Check if the given ES version can directly upgrade to the target ES version. Used to assist with catching lagging nodes during the upgrade process
-    es_version_can_upgrade_to_target() {
-        local current_version="$1"
-        # shellcheck disable=SC2076
-        if [[ -n "$current_version" && " $compatible_es_versions " =~ " $current_version " ]]; then
-            return 0
-        fi
-
-        return 1
-    }
-
-    # Gather Elasticsearch cluster version info and verify that each node in the cluster is running a version compatible with the target ES version.
-    verify_searchnodes_es_target_compatibility() {
-        local retries=20
-        local retry_count=0
-        local delay=180
-        local expected_es_nodes searchnode_minions attempt
-        local searchnode_discovery_success=false
-        SEARCHNODE_ES_VERSIONS=""
-
-        for attempt in {1..3}; do
-            if searchnode_minions=$(set -o pipefail; salt-key --out=json --list=accepted 2> /dev/null | jq -r '.minions[]? | select(endswith("searchnode"))'); then
-                searchnode_discovery_success=true
-                break
-            fi
-
-            echo "Failed to retrieve grid searchnodes via salt-key... Retrying in 30 seconds. Attempt $attempt of 3."
-            sleep 30
-        done
-
-        if [[ "$searchnode_discovery_success" != "true" ]]; then
-            echo "Failed to retrieve grid searchnodes via salt-key."
-            return 1
-        fi
-
-        # Always add node running soup to expected es nodes
-        expected_es_nodes="${MINIONID%_*}"
-        while IFS= read -r searchnode_minion; do
-            [[ -z "$searchnode_minion" ]] && continue
-            expected_es_nodes+=$'\n'"${searchnode_minion%_searchnode}"
-        done <<< "$searchnode_minions"
-
-        while [[ $retry_count -lt $retries ]]; do
-            SEARCHNODE_ES_VERSIONS=$(so-elasticsearch-query _nodes/_all/version --retry 5 --retry-delay 10 --fail 2>&1)
-            local exit_status=$?
-
-            if [[ $exit_status -ne 0 ]]; then
-                echo "Failed to retrieve Elasticsearch versions from searchnodes... Retrying in $delay seconds. Attempt $((retry_count + 1)) of $retries."
-                ((retry_count++))
-                sleep $delay
-                continue
-            fi
-
-            local all_searchnodes_compatible=true
-            while IFS=$'\t' read -r node current_version; do
-                [[ -z "$node" ]] && continue
-                if ! es_version_can_upgrade_to_target "$current_version"; then
-                    echo "Searchnode $node is running Elasticsearch $current_version, which is not directly upgradable to Elasticsearch $target_es_version."
-                    all_searchnodes_compatible=false
-                fi
-            done < <(echo "$SEARCHNODE_ES_VERSIONS" | jq -r '.nodes | to_entries[] | [.value.name, .value.version] | @tsv')
-
-            while IFS= read -r expected_es_node; do
-                [[ -z "$expected_es_node" ]] && continue
-                if ! echo "$SEARCHNODE_ES_VERSIONS" | jq -e --arg node "$expected_es_node" '.nodes | to_entries | any(.value.name == $node)' > /dev/null; then
-                    echo "Searchnode $expected_es_node did not report an Elasticsearch version. It may be offline or still upgrading."
-                    all_searchnodes_compatible=false
-                fi
-            done <<< "$expected_es_nodes"
-
-            if [[ "$all_searchnodes_compatible" == true ]]; then
-                echo "All Searchnodes are upgradable to Elasticsearch $target_es_version."
-                return 0
-            fi
-
-            echo "One or more Searchnodes cannot upgrade directly to Elasticsearch $target_es_version. Rechecking in $delay seconds. Attempt $((retry_count + 1)) of $retries."
-            ((retry_count++))
-            sleep $delay
-        done
-
-        return 1
-    }
-
-    # Gather heavynode version info and verify that each node is running a version compatible with the target ES version.
-    verify_heavynodes_es_target_compatibility() {
-        local heavynode_minions attempt
-        local retries=20
-        local retry_count=0
-        local delay=180
-        local heavynode_discovery_success=false
-        HEAVYNODE_ES_VERSIONS=""
-
-        for attempt in {1..3}; do
-            if heavynode_minions=$(set -o pipefail; salt-key --out=json --list=accepted 2> /dev/null | jq -r '.minions[]? | select(endswith("heavynode"))'); then
-                heavynode_discovery_success=true
-                break
-            fi
-
-            echo "Failed to retrieve grid heavynodes via salt-key... Retrying in 30 seconds. Attempt $attempt of 3."
-            sleep 30
-        done
-
-        if [[ "$heavynode_discovery_success" != "true" ]]; then
-            echo "Failed to retrieve grid heavynodes via salt-key."
-            return 1
-        fi
-
-        if [[ -z "$heavynode_minions" ]]; then
-            echo "No heavynodes detected. Skipping heavynode Elasticsearch version compatibility check."
-            return 0
-        fi
-
-        while [[ $retry_count -lt $retries ]]; do
-            HEAVYNODE_ES_VERSIONS=$(salt -C 'G@role:so-heavynode' cmd.run 'set -o pipefail; so-elasticsearch-query / --retry 5 --retry-delay 10 | jq -er ".version.number"' shell=/bin/bash --out=json 2> /dev/null)
-            local exit_status=$?
-
-            if [[ $exit_status -ne 0 ]]; then
-                echo "Failed to retrieve Elasticsearch version from one or more heavynodes... Retrying in $delay seconds. Attempt $((retry_count + 1)) of $retries."
-                ((retry_count++))
-                sleep $delay
-                continue
-            fi
-
-            local all_heavynodes_compatible=true
-            while IFS=$'\t' read -r node current_version; do
-                [[ -z "$node" ]] && continue
-                if ! es_version_can_upgrade_to_target "$current_version"; then
-                    echo "Heavynode $node is running Elasticsearch $current_version, which is not directly upgradable to Elasticsearch $target_es_version."
-                    all_heavynodes_compatible=false
-                fi
-            done < <(echo "$HEAVYNODE_ES_VERSIONS" | jq -r 'to_entries[] | [.key, .value] | @tsv')
-
-            while IFS= read -r heavynode_minion; do
-                [[ -z "$heavynode_minion" ]] && continue
-                if ! echo "$HEAVYNODE_ES_VERSIONS" | jq -se --arg minion "$heavynode_minion" 'add | has($minion)' > /dev/null; then
-                    echo "Heavynode $heavynode_minion did not report an Elasticsearch version. It may be offline or still upgrading."
-                    all_heavynodes_compatible=false
-                fi
-            done <<< "$heavynode_minions"
-
-            if [[ "$all_heavynodes_compatible" == true ]]; then
-                echo -e "\nAll heavynodes can upgrade to Elasticsearch $target_es_version."
-                return 0
-            fi
-
-            echo "One or more heavynodes cannot upgrade directly to Elasticsearch $target_es_version. Rechecking in $delay seconds. Attempt $((retry_count + 1)) of $retries."
-            ((retry_count++))
-            sleep $delay
-        done
-
-        return 1
-    }
-
-    if [[ ! -f "$es_verification_script" ]]; then
-        create_intermediate_upgrade_verification_script "$es_verification_script"
-    fi
-
     for statefile in "${es_required_version_statefile_base}"-*; do
         [[ -f $statefile ]] || continue
 
@@ -1239,6 +1012,10 @@ verify_es_version_compatibility() {
             continue
         fi
 
+        if [[ ! -f "$es_verification_script" ]]; then
+            create_intermediate_upgrade_verification_script "$es_verification_script"
+        fi
+
         echo -e "\n##############################################################################################################################\n"
         echo "A previously required intermediate Elasticsearch upgrade was detected. Verifying that all Searchnodes/Heavynodes have successfully upgraded Elasticsearch to $es_required_version_statefile_value before proceeding with soup to avoid potential data loss! This command can take up to an hour to complete."
         if ! timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$statefile"; then
@@ -1260,26 +1037,6 @@ verify_es_version_compatibility() {
 
     # shellcheck disable=SC2076 # Do not want a regex here eg usage " 8.18.8 9.0.8 " =~ " 9.0.8 "
     if [[ " ${es_upgrade_map[$es_version]} " =~ " $target_es_version " || "$es_version" == "$target_es_version" ]]; then
-        if ! verify_searchnodes_es_target_compatibility || ! verify_heavynodes_es_target_compatibility; then
-            echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
-
-            echo "One or more Searchnode(s)/Heavynode(s) cannot upgrade directly to Elasticsearch $target_es_version. This can happen with soups that include Elasticsearch upgrades being run in quick succession. Typically, this will resolve itself as the grid synchronizes. Please allow time for all Searchnodes/Heavynodes to have upgraded Elasticsearch to a compatible version with $target_es_version before running soup again to avoid potential data loss!"
-
-            if [[ -n "$HEAVYNODE_ES_VERSIONS" ]]; then
-                echo "Current heavynode Elasticsearch versions:"
-                echo "$HEAVYNODE_ES_VERSIONS" | jq '.'
-            fi
-
-            if [[ -n "$SEARCHNODE_ES_VERSIONS" ]]; then
-                echo "Current searchnode Elasticsearch versions:"
-                echo "$SEARCHNODE_ES_VERSIONS" | jq '.nodes | to_entries | map({(.value.name): .value.version}) | sort | add'
-            fi
-
-            echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
-
-            exit 161
-        fi
-
         # supported upgrade
         return 0
     else
@@ -1565,7 +1322,7 @@ EOF
 
 # Keeping this block in case we need to do a hotfix that requires salt update
 apply_hotfix() {
-    echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)"
+   echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)"
 }
 
 failed_soup_restore_items() {
@@ -1637,13 +1394,13 @@ main() {
   echo "Verifying we have the latest soup script."
   verify_latest_update_script
 
+  echo "Verifying Elasticsearch version compatibility before upgrading."
+  verify_es_version_compatibility
+
   echo "Let's see if we need to update Security Onion."
   upgrade_check
   upgrade_space
 
-  echo "Verifying Elasticsearch version compatibility across the grid before upgrading."
-  verify_es_version_compatibility
-
   echo "Checking for Salt Master and Minion updates."
   upgrade_check_salt
   set -e
@@ -1663,8 +1420,7 @@ main() {
     echo "Applying $HOTFIXVERSION hotfix"
     # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars
     if [[ ! "$MINION_ROLE" == "import" ]]; then
-        echo "Running so-config-backup script."
-        /sbin/so-config-backup
+      backup_old_states_pillars
     fi
     copy_new_files
     create_local_directories "/opt/so/saltstack/default"
@@ -1720,8 +1476,8 @@ main() {
     # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars
     if [[ ! "$MINION_ROLE" == "import" ]]; then
       echo ""
-      echo "Running so-config-backup script."
-      /sbin/so-config-backup
+      echo "Creating snapshots of default and local Salt states and pillars and saving to /nsm/backup/"
+      backup_old_states_pillars
     fi
 
     echo ""

salt/postgres/files/init-db.sh

@@ -17,7 +17,6 @@ psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-E
         END IF;
     END
     \$\$;
-    GRANT ALL ON SCHEMA public TO "$SO_POSTGRES_USER";
     GRANT ALL PRIVILEGES ON DATABASE "$POSTGRES_DB" TO "$SO_POSTGRES_USER";
     -- Lock the SOC database down at the connect layer; PUBLIC gets CONNECT
     -- by default, which would let per-minion telegraf roles open sessions
@@ -32,4 +31,4 @@ EOSQL
 # only ensures the shared database exists on first initialization.
 if ! psql -U "$POSTGRES_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1; then
     psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE DATABASE so_telegraf"
-fi
+fi

salt/postgres/telegraf_users.sls

@@ -18,12 +18,26 @@ include:
 {% set TG_OUT = TELEGRAFMERGED.output | upper %}
 {% if TG_OUT in ['POSTGRES', 'BOTH'] %}
 
+# docker_container.running returns as soon as the container starts, but on
+# first-init docker-entrypoint.sh starts a temporary postgres with
+# `listen_addresses=''` to run /docker-entrypoint-initdb.d scripts, then
+# shuts it down before exec'ing the real CMD. A default pg_isready check
+# (Unix socket) passes during that ephemeral phase and races the shutdown
+# with "the database system is shutting down". Checking TCP readiness on
+# 127.0.0.1 only succeeds after the final postgres binds the port.
 postgres_wait_ready:
   cmd.run:
-    - name: /usr/sbin/so-postgres-wait
+    - name: |
+        for i in $(seq 1 60); do
+          if docker exec so-postgres pg_isready -h 127.0.0.1 -U postgres -q 2>/dev/null; then
+            exit 0
+          fi
+          sleep 2
+        done
+        echo "so-postgres did not accept TCP connections within 120s" >&2
+        exit 1
     - require:
       - docker_container: so-postgres
-      - file: postgres_sbin
 
 # Ensure the shared Telegraf database exists. init-db.sh only runs on a
 # fresh data dir, so hosts upgraded onto an existing /nsm/postgres volume

salt/postgres/tools/sbin/so-postgres-wait

@@ -1,32 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Wait for the so-postgres container to accept TCP connections.
-#
-# docker_container.running returns as soon as the container starts, but on
-# first-init docker-entrypoint.sh starts a temporary postgres with
-# `listen_addresses=''` to run /docker-entrypoint-initdb.d scripts, then
-# shuts it down before exec'ing the real CMD. A default pg_isready check
-# (Unix socket) passes during that ephemeral phase and races the shutdown
-# with "the database system is shutting down". Checking TCP readiness on
-# 127.0.0.1 only succeeds after the final postgres binds the port.
-#
-# Usage: so-postgres-wait [iterations] [sleep_seconds]
-# Default: 60 iterations, 2s sleep (~120s total).
-
-ITERATIONS=${1:-60}
-SLEEP_SECONDS=${2:-2}
-
-for i in $(seq 1 "$ITERATIONS"); do
-  if docker exec so-postgres pg_isready -h 127.0.0.1 -U postgres -q 2>/dev/null; then
-    exit 0
-  fi
-  sleep "$SLEEP_SECONDS"
-done
-
-echo "so-postgres did not accept TCP connections within $((ITERATIONS * SLEEP_SECONDS))s" >&2
-exit 1

salt/salt/master.sls

@@ -14,7 +14,6 @@
 
 include:
   - salt.minion
-  - salt.master.boot_mine_update
 {%   if 'vrt' in salt['pillar.get']('features', []) %}
   - salt.cloud
   - salt.cloud.reactor_config_hypervisor

salt/salt/master/boot_mine_update.sls

@@ -1,29 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Manages /etc/systemd/system/so-boot-mine-update.service, a manager-only
-# Type=oneshot unit that pushes `salt '*' mine.update` once per boot, ordered
-# before so-boot-highstate.service so mine-backed pillars (node IPs, ES/Redis/
-# Logstash discovery) are fresh before the boot highstate renders them.
-
-include:
-  - systemd.reload
-
-so_boot_mine_update_unit_file:
-  file.managed:
-    - name: /etc/systemd/system/so-boot-mine-update.service
-    - source: salt://salt/service/so-boot-mine-update.service
-    - onchanges_in:
-      - module: systemd_reload
-
-# Only enable once setup is complete. Until then the gate file is missing and
-# the unit's own ConditionPathExists would no-op it anyway.
-so_boot_mine_update_service:
-  service.enabled:
-    - name: so-boot-mine-update.service
-    - onlyif: test -e /opt/so/state/setup-complete
-    - require:
-      - file: so_boot_mine_update_unit_file
-      - module: systemd_reload

salt/salt/minion/boot_highstate.sls

@@ -1,31 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Manages /etc/systemd/system/so-boot-highstate.service, a Type=oneshot
-# RemainAfterExit=yes unit that runs `salt-call state.highstate` exactly once
-# per system boot. Replaces the legacy `startup_states: highstate` minion
-# config, which fired on every salt-minion service restart (causing a redundant
-# highstate whenever a highstate itself restarted salt-minion).
-
-include:
-  - systemd.reload
-
-so_boot_highstate_unit_file:
-  file.managed:
-    - name: /etc/systemd/system/so-boot-highstate.service
-    - source: salt://salt/service/so-boot-highstate.service
-    - onchanges_in:
-      - module: systemd_reload
-
-# Only enable once setup is complete. Until then the gate file is missing and
-# the unit's own ConditionPathExists would no-op it anyway -- this just keeps
-# `systemctl is-enabled` honest for the sync_es_users gate.
-so_boot_highstate_service:
-  service.enabled:
-    - name: so-boot-highstate.service
-    - onlyif: test -e /opt/so/state/setup-complete
-    - require:
-      - file: so_boot_highstate_unit_file
-      - module: systemd_reload

salt/salt/minion/init.sls

@@ -17,7 +17,6 @@ include:
   - repo.client
   - salt.mine_functions
   - salt.minion.service_file
-  - salt.minion.boot_highstate
 {% if GLOBALS.is_manager %}
   - ca.signing_policy
 {% endif %}
@@ -81,33 +80,11 @@ set_log_levels:
       - "log_level: info"
       - "log_level_logfile: info"
 
-# startup_states: highstate caused a full highstate to run on every
-# salt-minion service start, including the restart triggered when a highstate
-# itself modified the minion config (beacons, mine, unit file). Replaced by
-# so-boot-highstate.service (managed in salt.minion.boot_highstate), which
-# runs once per system boot only. Strip the line from /etc/salt/minion on
-# upgrade; both the commented and uncommented forms historically existed.
-remove_startup_states:
-  file.line:
+enable_startup_states:
+  file.uncomment:
     - name: /etc/salt/minion
-    - match: 'startup_states: highstate'
-    - mode: delete
-
-# Upgrade-path bridge: systems that already passed setup under the old gate
-# (`grep -x 'startup_states: highstate' /etc/salt/minion`) get a /opt/so/state/setup-complete
-# marker so so-boot-highstate.service can be enabled and the so-user_sync cron
-# in sync_es_users.sls keeps installing. Setup-in-progress systems instead get
-# the marker from `mark_setup_complete` in setup/so-functions at the right
-# moment. `replace: false` means we never overwrite a marker once written.
-mark_setup_complete_for_upgrades:
-  file.managed:
-    - name: /opt/so/state/setup-complete
-    - replace: false
-    - makedirs: True
-    - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion"
-    - require_in:
-      - file: remove_startup_states
-      - service: so_boot_highstate_service
+    - regex: '^startup_states: highstate$'
+    - unless: pgrep so-setup
 
 {% endif %}
 

salt/salt/service/so-boot-highstate.service

@@ -1,14 +0,0 @@
-[Unit]
-Description=Security Onion boot-time highstate (runs once per boot)
-After=salt-minion.service network-online.target docker.service
-Wants=network-online.target docker.service
-Requires=salt-minion.service
-ConditionPathExists=/opt/so/state/setup-complete
-
-[Service]
-Type=oneshot
-RemainAfterExit=yes
-ExecStart=/usr/bin/salt-call state.highstate -l info queue=True
-
-[Install]
-WantedBy=multi-user.target

salt/salt/service/so-boot-mine-update.service

@@ -1,15 +0,0 @@
-[Unit]
-Description=Security Onion boot-time grid mine.update (managers, runs once per boot before highstate)
-After=salt-master.service salt-minion.service network-online.target
-Wants=network-online.target
-Requires=salt-master.service salt-minion.service
-Before=so-boot-highstate.service
-ConditionPathExists=/opt/so/state/setup-complete
-
-[Service]
-Type=oneshot
-RemainAfterExit=yes
-ExecStart=/usr/sbin/so-boot-mine-update
-
-[Install]
-WantedBy=multi-user.target

salt/setup/virt/setSalt.sls

@@ -8,6 +8,11 @@ set_role_grain:
     - name: role
     - value: so-{{ grains.id.split("_") | last }}
 
+set_highstate:
+  file.append:
+    - name: /etc/salt/minion
+    - text: 'startup_states: highstate'
+
 enable_salt_minion:
   service.enabled:
     - name: salt-minion

salt/soc/defaults.yaml

@@ -1519,16 +1519,6 @@ soc:
               serviceAccountJSON: ""
               serviceAccountLocation: ""
               healthTimeoutSeconds: 5
-        onionconfig:
-          saltstackDir: /opt/so/saltstack
-          bypassEnabled: false
-        postgres:
-          host: ""
-          port: 5432
-          sslMode: "allow"
-          database: securityonion
-          user: ""
-          password: ""
         salt:
           queueDir: /opt/sensoroni/queue
           timeoutMs: 45000

salt/soc/merged.map.jinja

@@ -16,14 +16,6 @@
 {% do SOCMERGED.config.server.update({'additionalCA': MANAGERMERGED.additionalCA}) %}
 {% do SOCMERGED.config.server.update({'insecureSkipVerify': MANAGERMERGED.insecureSkipVerify}) %}
 
-{% if not SOCMERGED.config.server.modules.postgres.host %}
-{%   do SOCMERGED.config.server.modules.postgres.update({'host': GLOBALS.manager}) %}
-{% endif %}
-{% if not SOCMERGED.config.server.modules.postgres.password %}
-{%   do SOCMERGED.config.server.modules.postgres.update({'password': salt['pillar.get']('postgres:auth:users:so_postgres_user:pass', '')}) %}
-{%   do SOCMERGED.config.server.modules.postgres.update({'user': salt['pillar.get']('postgres:auth:users:so_postgres_user:user', 'so_postgres')}) %}
-{% endif %}
-
 {# if SOCMERGED.config.server.modules.cases == httpcase details come from the soc pillar #}
 {% if SOCMERGED.config.server.modules.cases != 'soc' %}
 {%   do SOCMERGED.config.server.modules.elastic.update({'casesEnabled': false}) %}

salt/soc/soc_soc.yaml

@@ -453,42 +453,6 @@ soc:
             description: Duration (in milliseconds) that must elapse after a grid node fails to check-in before the node will be marked offline (fault).
             global: True
             advanced: True
-        onionconfig:
-          saltstackDir:
-            description: Root directory containing the SaltStack tree that SOC reads and writes configuration from. Should not be changed under normal circumstances.
-            global: True
-            advanced: True
-          bypassEnabled:
-            description: When enabled, errors encountered while reading the SaltStack pillar tree (missing files, unreadable directories, etc.) are logged but do not prevent SOC from starting or serving settings. Intended for advanced troubleshooting and recovery scenarios when the pillar tree is partially unreadable.
-            global: True
-            advanced: True
-            forcedType: bool
-        postgres:
-          host:
-            description: Hostname or IP address of the PostgreSQL server used by SOC. Defaults to the manager hostname.
-            global: True
-            advanced: True
-          port:
-            description: Port of the PostgreSQL server used by SOC.
-            global: True
-            advanced: True
-          sslMode:
-            description: "Use encrypted connections to the PostgreSQL server. Must be one of the following values: disable, allow, prefer, require, verify-ca, verify-full.  Defaults to allow."
-            global: True
-            advanced: True
-          database:
-            description: Database used by SOC to authenticate to the PostgreSQL server.
-            global: True
-            advanced: True
-          user:
-            description: Username used by SOC to authenticate to the PostgreSQL server.
-            global: True
-            advanced: True
-          password:
-            description: Password used by SOC to authenticate to the PostgreSQL server.
-            global: True
-            sensitive: True
-            advanced: True
         salt:
           longRelayTimeoutMs:
             description: Duration (in milliseconds) to wait for a response from the Salt API when executing tasks known for being long running before giving up and showing an error on the SOC UI.
@@ -854,7 +818,6 @@ soc:
           description: List of available external tools visible in the SOC UI. Each tool is defined in JSON object notation, and must include the "name" key and "link" key, where the link is the tool's URL.
           global: True
           advanced: True
-          multiline: True
           forcedType: "[]{}"
         exportNodeId:
           description: The node ID on which export jobs will be executed.

setup/so-functions

@@ -539,19 +539,16 @@ configure_minion() {
 		"  x509_v2: true"\
 		"log_level: info"\
 		"log_level_logfile: info"\
-		"log_file: /opt/so/log/salt/minion" >> "$minion_config"
+		"log_file: /opt/so/log/salt/minion"\
+		"#startup_states: highstate" >> "$minion_config"
 
 }
 
-mark_setup_complete() {
-	# Writes the setup-complete marker. Salt's so-boot-highstate.service
-	# (boot-time oneshot) and the so-user_sync cron gate in
-	# salt/manager/sync_es_users.sls both key off this file.
-	local marker=/opt/so/state/setup-complete
+checkin_at_boot() {
+	local minion_config=/etc/salt/minion
 
-	info "Marking setup as complete"
-	mkdir -p "$(dirname "$marker")"
-	touch "$marker"
+	info "Enabling checkin at boot"
+	sed -i 's/#startup_states: highstate/startup_states: highstate/' "$minion_config"
 }
 
 check_requirements() {
@@ -980,8 +977,6 @@ docker_seed_registry() {
 		docker_seed_update_percent=25
 
 		update_docker_containers 'netinstall' '' 'docker_seed_update' '/dev/stdout' 2>&1 | tee -a "$setup_log"
-        # Use pipe exit status of 'update_docker_containers' for return code
-		return ${PIPESTATUS[0]}
 	fi
 }
 

setup/so-setup

@@ -223,8 +223,6 @@ if [ -n "$test_profile" ]; then
 	WEBPASSWD1=0n10nus3r
 	WEBPASSWD2=0n10nus3r
 	NODE_DESCRIPTION="${HOSTNAME} - ${install_type} - ${MSRVIP_OFFSET}"
-	# opt out of telemetry for automated testing
-	telemetry=1
 
 	update_sudoers_for_testing
 fi
@@ -769,10 +767,7 @@ if ! [[ -f $install_opt_file ]]; then
 		title "Applying the registry state"
 		logCmd "salt-call state.apply -l info registry"
 		title "Seeding the docker registry"
-		if ! docker_seed_registry; then
-			error "Failed to seed the docker registry"
-			fail_setup
-		fi
+		docker_seed_registry
 		title "Applying the manager state"
 		logCmd "salt-call state.apply -l info manager"
 		logCmd "salt-call state.apply influxdb -l info"
@@ -797,7 +792,7 @@ if ! [[ -f $install_opt_file ]]; then
 			error "Failed to run so-elastic-fleet-setup"
 			fail_setup
 		fi
-		mark_setup_complete
+		checkin_at_boot
 		set_initial_firewall_access
         initialize_elasticsearch_indices "so-case so-casehistory so-assistant-session so-assistant-chat"
 		# run a final highstate before enabling scheduled highstates.