From ebb93b4fa7b78cd0d21162e268d621f4e4af2e44 Mon Sep 17 00:00:00 2001
From: reyesj2 <94730068+reyesj2@users.noreply.github.com>
Date: Fri, 17 Apr 2026 14:43:07 -0500
Subject: [PATCH 01/16] add wait_for_so-elasticsearch state and split
 elasticsearch cluster configuration out of enabled.sls

---
 salt/elasticsearch/cluster.sls | 164 +++++++++++++++++++++++++++++++++
 salt/elasticsearch/enabled.sls | 160 +++-----------------------------
 2 files changed, 178 insertions(+), 146 deletions(-)
 create mode 100644 salt/elasticsearch/cluster.sls

diff --git a/salt/elasticsearch/cluster.sls b/salt/elasticsearch/cluster.sls
new file mode 100644
index 000000000..7a8a6675c
--- /dev/null
+++ b/salt/elasticsearch/cluster.sls
@@ -0,0 +1,164 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
+{%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %}
+{%   if GLOBALS.role != 'so-heavynode' %}
+{%     from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
+{%   endif %}
+
+escomponenttemplates:
+  file.recurse:
+    - name: /opt/so/conf/elasticsearch/templates/component
+    - source: salt://elasticsearch/templates/component
+    - user: 930
+    - group: 939
+    - clean: True
+    - onchanges_in:
+      - file: so-elasticsearch-templates-reload
+    - show_changes: False
+
+# Clean up legacy and non-SO managed templates from the elasticsearch/templates/index/ directory
+so_index_template_dir:
+  file.directory:
+    - name: /opt/so/conf/elasticsearch/templates/index
+    - clean: True
+    {%- if SO_MANAGED_INDICES %}
+    - require:
+      {%- for index in SO_MANAGED_INDICES %}
+      - file: so_index_template_{{index}}
+      {%- endfor %}
+    {%- endif %}
+
+# Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml)
+#   These index templates are for the core SO datasets and are always required
+{%  for index, settings in ES_INDEX_SETTINGS.items() %}
+{%    if settings.index_template is defined %}
+so_index_template_{{index}}:
+  file.managed:
+    - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json
+    - source: salt://elasticsearch/base-template.json.jinja
+    - defaults:
+        TEMPLATE_CONFIG: {{ settings.index_template }}
+    - template: jinja
+    - onchanges_in:
+      - file: so-elasticsearch-templates-reload
+{%    endif %}
+{%  endfor %}
+
+{%  if GLOBALS.role != "so-heavynode" %}
+# Auto-generate optional index templates for integration | input | content packages
+#   These index templates are not used by default (until user adds package to an agent policy).
+#   Pre-configured with standard defaults, and incorporated into SOC configuration for user customization.
+{%    for index,settings in ALL_ADDON_SETTINGS.items() %}
+{%      if settings.index_template is defined %}
+addon_index_template_{{index}}:
+  file.managed:
+    - name: /opt/so/conf/elasticsearch/templates/addon-index/{{ index }}-template.json
+    - source: salt://elasticsearch/base-template.json.jinja
+    - defaults:
+        TEMPLATE_CONFIG: {{ settings.index_template }}
+    - template: jinja
+    - show_changes: False
+    - onchanges_in:
+      - file: addon-elasticsearch-templates-reload
+{%       endif %}
+{%     endfor %}
+{%   endif %}
+
+{%  if GLOBALS.role in GLOBALS.manager_roles %}
+so-es-cluster-settings:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-cluster-settings
+    - cwd: /opt/so
+    - template: jinja
+    - require:
+      - docker_container: so-elasticsearch
+      - file: elasticsearch_sbin_jinja
+      - http: wait_for_so-elasticsearch
+{%  endif %}
+
+# heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)
+so-elasticsearch-ilm-policy-load:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-ilm-policy-load
+    - cwd: /opt/so
+    - require:
+      - docker_container: so-elasticsearch
+      - file: so-elasticsearch-ilm-policy-load-script
+    - onchanges:
+      - file: so-elasticsearch-ilm-policy-load-script
+
+so-elasticsearch-templates-reload:
+  file.absent:
+    - name: /opt/so/state/estemplates.txt
+
+addon-elasticsearch-templates-reload:
+  file.absent:
+    - name: /opt/so/state/addon_estemplates.txt
+
+# so-elasticsearch-templates-load will have its first successful run during the 'so-elastic-fleet-setup' script
+so-elasticsearch-templates:
+  cmd.run:
+{%-  if GLOBALS.role == "so-heavynode" %}
+    - name: /usr/sbin/so-elasticsearch-templates-load --heavynode
+{%-  else %}
+    - name: /usr/sbin/so-elasticsearch-templates-load
+{%-  endif %}
+    - cwd: /opt/so
+    - template: jinja
+    - require:
+      - docker_container: so-elasticsearch
+      - file: elasticsearch_sbin_jinja
+
+so-elasticsearch-pipelines:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }}
+    - require:
+      - docker_container: so-elasticsearch
+      - file: so-elasticsearch-pipelines-script
+
+so-elasticsearch-roles-load:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-roles-load
+    - cwd: /opt/so
+    - template: jinja
+    - require:
+      - docker_container: so-elasticsearch
+      - file: elasticsearch_sbin_jinja
+
+{%  if grains.role in ['so-managersearch', 'so-manager', 'so-managerhype'] %}
+{%    set ap = "absent" %}
+{%  endif %}
+{%  if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %}
+{%    if ELASTICSEARCHMERGED.index_clean %}
+{%      set ap = "present" %}
+{%    else %}
+{%      set ap = "absent" %}
+{%    endif %}
+{%  endif %}
+{%  if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
+so-elasticsearch-indices-delete:
+  cron.{{ap}}:
+    - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1
+    - identifier: so-elasticsearch-indices-delete
+    - user: root
+    - minute: '*/5'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+{%  endif %}
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}
diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls
index f4031ee5d..ab12b875e 100644
--- a/salt/elasticsearch/enabled.sls
+++ b/salt/elasticsearch/enabled.sls
@@ -10,10 +10,6 @@
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
-{%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %}
-{%   if GLOBALS.role != 'so-heavynode' %}
-{%     from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
-{%   endif %}
 
 include:
   - ca
@@ -21,6 +17,9 @@ include:
   - elasticsearch.ssl
   - elasticsearch.config
   - elasticsearch.sostatus
+{%- if GLOBALS.role != 'so-searchode' %}
+  - elasticsearch.cluster
+{%- endif%}
 
 so-elasticsearch:
   docker_container.running:
@@ -108,150 +107,19 @@ delete_so-elasticsearch_so-status.disabled:
     - name: /opt/so/conf/so-status/so-status.conf
     - regex: ^so-elasticsearch$
 
-{%   if GLOBALS.role != "so-searchnode" %}
-escomponenttemplates:
-  file.recurse:
-    - name: /opt/so/conf/elasticsearch/templates/component
-    - source: salt://elasticsearch/templates/component
-    - user: 930
-    - group: 939
-    - clean: True
-    - onchanges_in:
-      - file: so-elasticsearch-templates-reload
-    - show_changes: False
-
-# Clean up legacy and non-SO managed templates from the elasticsearch/templates/index/ directory
-so_index_template_dir:
-  file.directory:
-    - name: /opt/so/conf/elasticsearch/templates/index
-    - clean: True
-    {%- if SO_MANAGED_INDICES %}
-    - require:
-      {%- for index in SO_MANAGED_INDICES %}
-      - file: so_index_template_{{index}}
-      {%- endfor %}
-    {%- endif %}
-
-# Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml)
-#   These index templates are for the core SO datasets and are always required
-{%     for index, settings in ES_INDEX_SETTINGS.items() %}
-{%       if settings.index_template is defined %}
-so_index_template_{{index}}:
-  file.managed:
-    - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json
-    - source: salt://elasticsearch/base-template.json.jinja
-    - defaults:
-        TEMPLATE_CONFIG: {{ settings.index_template }}
-    - template: jinja
-    - onchanges_in:
-      - file: so-elasticsearch-templates-reload
-{%       endif %}
-{%     endfor %}
-
-{%     if GLOBALS.role != "so-heavynode" %}
-# Auto-generate optional index templates for integration | input | content packages
-#   These index templates are not used by default (until user adds package to an agent policy).
-#   Pre-configured with standard defaults, and incorporated into SOC configuration for user customization.
-{%       for index,settings in ALL_ADDON_SETTINGS.items() %}
-{%         if settings.index_template is defined %}
-addon_index_template_{{index}}:
-  file.managed:
-    - name: /opt/so/conf/elasticsearch/templates/addon-index/{{ index }}-template.json
-    - source: salt://elasticsearch/base-template.json.jinja
-    - defaults:
-        TEMPLATE_CONFIG: {{ settings.index_template }}
-    - template: jinja
-    - show_changes: False
-    - onchanges_in:
-      - file: addon-elasticsearch-templates-reload
-{%         endif %}
-{%       endfor %}
-{%     endif %}
-
-{%     if GLOBALS.role in GLOBALS.manager_roles %}
-so-es-cluster-settings:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-cluster-settings
-    - cwd: /opt/so
-    - template: jinja
+wait_for_so-elasticsearch:
+  http.wait_for_successful_query:
+    - name: "https://localhost:9200/"
+    - username: 'so_elastic'
+    - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}'
+    - ssl: True
+    - verify_ssl: False
+    - status: 200
+    - wait_for: 300
+    - request_interval: 15
+    - backend: requests
     - require:
       - docker_container: so-elasticsearch
-      - file: elasticsearch_sbin_jinja
-{%     endif %}
-
-# heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)
-so-elasticsearch-ilm-policy-load:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-ilm-policy-load
-    - cwd: /opt/so
-    - require:
-      - docker_container: so-elasticsearch
-      - file: so-elasticsearch-ilm-policy-load-script
-    - onchanges:
-      - file: so-elasticsearch-ilm-policy-load-script
-
-so-elasticsearch-templates-reload:
-  file.absent:
-    - name: /opt/so/state/estemplates.txt
-
-addon-elasticsearch-templates-reload:
-  file.absent:
-    - name: /opt/so/state/addon_estemplates.txt
-
-# so-elasticsearch-templates-load will have its first successful run during the 'so-elastic-fleet-setup' script
-so-elasticsearch-templates:
-  cmd.run:
-{%-    if GLOBALS.role == "so-heavynode" %}
-    - name: /usr/sbin/so-elasticsearch-templates-load --heavynode
-{%-    else %}
-    - name: /usr/sbin/so-elasticsearch-templates-load
-{%-     endif %}
-    - cwd: /opt/so
-    - template: jinja
-    - require:
-      - docker_container: so-elasticsearch
-      - file: elasticsearch_sbin_jinja
-
-so-elasticsearch-pipelines:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }}
-    - require:
-      - docker_container: so-elasticsearch
-      - file: so-elasticsearch-pipelines-script
-
-so-elasticsearch-roles-load:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-roles-load
-    - cwd: /opt/so
-    - template: jinja
-    - require:
-      - docker_container: so-elasticsearch
-      - file: elasticsearch_sbin_jinja
-
-{%     if grains.role in ['so-managersearch', 'so-manager', 'so-managerhype'] %}
-{%       set ap = "absent" %}
-{%     endif %}
-{%     if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %}
-{%       if ELASTICSEARCHMERGED.index_clean %}
-{%         set ap = "present" %}
-{%       else %}
-{%         set ap = "absent" %}
-{%       endif %}
-{%     endif %}  
-{%     if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
-so-elasticsearch-indices-delete:
-  cron.{{ap}}:
-    - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1
-    - identifier: so-elasticsearch-indices-delete
-    - user: root
-    - minute: '*/5'
-    - hour: '*'
-    - daymonth: '*'
-    - month: '*'
-    - dayweek: '*'
-{%     endif %}
-
-{%   endif %}
 
 {% else %}
 

From 1cb34b089cc6e678504e514a161c128d22dfbce3 Mon Sep 17 00:00:00 2001
From: Mike Reeves <reevesmk@gmail.com>
Date: Mon, 20 Apr 2026 14:38:55 -0400
Subject: [PATCH 02/16] Restore 3/dev soup and add postgres users to
 post_to_3.1.0
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

feature/postgres had rewritten the 3.1.0 upgrade block, dropping the
elastic upgrade work 3/dev landed for 9.0.8→9.3.3: elasticsearch_backup_index_templates,
the component template state cleanup, and the /usr/sbin/so-kibana-space-defaults
post-upgrade call. It also carried an older ES upgrade mapping
(8.18.8→9.0.8) that was superseded on 3/dev (9.0.8→9.3.3 for
3.0.0-20260331), and a handful of latent shell-quoting regressions in
verify_es_version_compatibility and the intermediate-upgrade helpers.

Adopt the 3/dev soup verbatim and only add the new Telegraf Postgres
provisioning to post_to_3.1.0 on top of so-kibana-space-defaults.
---
 salt/manager/tools/sbin/soup | 94 +++++++++++++++++++-----------------
 1 file changed, 49 insertions(+), 45 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 5ed66134f..c25358418 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -362,9 +362,8 @@ preupgrade_changes() {
     # This function is to add any new pillar items if needed.
     echo "Checking to see if changes are needed."
 
-    [[ "$INSTALLEDVERSION" =~ ^2\.4\.21[0-9]+$ ]] && up_to_3.0.0
-    [[ "$INSTALLEDVERSION" == 3.0.0 ]] && up_to_3.1.0   
-
+    [[ "$INSTALLEDVERSION" =~ ^2\.4\.21[0-9]+$ ]] && up_to_3.0.0   
+    [[ "$INSTALLEDVERSION" == "3.0.0" ]] && up_to_3.1.0
     true
 }
 
@@ -373,8 +372,7 @@ postupgrade_changes() {
     echo "Running post upgrade processes."
 
     [[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0
-    [[ "$POSTVERSION" =~ 3.0.0 ]] && post_to_3.1.0
-    
+    [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0
     true
 }
 
@@ -385,7 +383,7 @@ check_minimum_version() {
   fi
 }
 
-### 3.0.0 Start ###
+### 3.0.0 Scripts ###
 
 convert_suricata_yes_no() {
   echo "Starting suricata yes/no values to true/false conversion."
@@ -449,7 +447,6 @@ migrate_pcap_to_suricata() {
 }
 
 up_to_3.0.0() {
-  determine_elastic_agent_upgrade
   migrate_pcap_to_suricata
 
   INSTALLEDVERSION=3.0.0
@@ -473,13 +470,26 @@ post_to_3.0.0() {
 
 ### 3.0.0 End ###
 
-### 3.1.0 Start ###
+### 3.1.0 Scripts ###
+
+elasticsearch_backup_index_templates() {
+  echo "Backing up current elasticsearch index templates in /opt/so/conf/elasticsearch/templates/index/ to /nsm/backup/3.0.0_elasticsearch_index_templates.tar.gz"
+  tar -czf /nsm/backup/3.0.0_elasticsearch_index_templates.tar.gz -C /opt/so/conf/elasticsearch/templates/index/ .
+}
+
 up_to_3.1.0() {
-  echo "Nothing to do"
+  determine_elastic_agent_upgrade
+  elasticsearch_backup_index_templates
+  # Clear existing component template state file.
+  rm -f /opt/so/state/esfleet_component_templates.json
+
+
   INSTALLEDVERSION=3.1.0
 }
 
 post_to_3.1.0() {
+  /usr/sbin/so-kibana-space-defaults
+
   # Provision per-minion Telegraf Postgres users for every minion known to the
   # manager. postgres.auth iterates manage.up to generate any missing passwords;
   # postgres.telegraf_users reconciles the roles and schemas inside the so-postgres
@@ -493,6 +503,7 @@ post_to_3.1.0() {
 
 ### 3.1.0 End ###
 
+
 repo_sync() {
   echo "Sync the local repo."
   su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
@@ -752,12 +763,12 @@ verify_es_version_compatibility() {
     local is_active_intermediate_upgrade=1
     # supported upgrade paths for SO-ES versions
     declare -A es_upgrade_map=(
-	    ["8.18.8"]="9.0.8"
+        ["9.0.8"]="9.3.3"
     )
 
     # Elasticsearch MUST upgrade through these versions
     declare -A es_to_so_version=(
-	    ["8.18.8"]="2.4.190-20251024"
+        ["9.0.8"]="3.0.0-20260331"
     )
 
     # Get current Elasticsearch version
@@ -769,26 +780,17 @@ verify_es_version_compatibility() {
         exit 160
     fi
 
-    if ! target_es_version_raw=$(so-yaml.py get $UPDATE_DIR/salt/elasticsearch/defaults.yaml elasticsearch.version); then
-        # so-yaml.py failed to get the ES version from upgrade versions elasticsearch/defaults.yaml file. Likely they are upgrading to an SO version older than 2.4.110 prior to the ES version pinning and should be OKAY to continue with the upgrade.
+    if ! target_es_version=$(so-yaml.py get -r $UPDATE_DIR/salt/elasticsearch/defaults.yaml elasticsearch.version); then
+        echo "Couldn't determine the target Elasticsearch version (post soup version) to ensure compatibility with current Elasticsearch version. Exiting"
 
-        # if so-yaml.py failed to get the ES version AND the version we are upgrading to is newer than 2.4.110 then we should bail
-        if [[ $(cat $UPDATE_DIR/VERSION | cut -d'.' -f3) > 110 ]]; then
-            echo "Couldn't determine the target Elasticsearch version (post soup version) to ensure compatibility with current Elasticsearch version. Exiting"
-
-            exit 160
-        fi
-
-        # allow upgrade to version < 2.4.110 without checking ES version compatibility
-        return 0
-    else
-        target_es_version=$(sed -n '1p' <<< "$target_es_version_raw")
+        exit 160
     fi
 
     for statefile in "${es_required_version_statefile_base}"-*; do
         [[ -f $statefile ]] || continue
 
-        local es_required_version_statefile_value=$(cat "$statefile")
+        local es_required_version_statefile_value
+        es_required_version_statefile_value=$(cat "$statefile")
 
         if [[ "$es_required_version_statefile_value" == "$target_es_version" ]]; then
             echo "Intermediate upgrade to ES $target_es_version is in progress. Skipping Elasticsearch version compatibility check."
@@ -797,7 +799,7 @@ verify_es_version_compatibility() {
         fi
 
         # use sort to check if es_required_statefile_value is < the current es_version.
-        if [[ "$(printf '%s\n' $es_required_version_statefile_value $es_version | sort -V | head -n1)" == "$es_required_version_statefile_value" ]]; then
+        if [[ "$(printf '%s\n' "$es_required_version_statefile_value" "$es_version" | sort -V | head -n1)" == "$es_required_version_statefile_value" ]]; then
             rm -f "$statefile"
             continue
         fi
@@ -808,8 +810,7 @@ verify_es_version_compatibility() {
 
         echo -e "\n##############################################################################################################################\n"
         echo "A previously required intermediate Elasticsearch upgrade was detected. Verifying that all Searchnodes/Heavynodes have successfully upgraded Elasticsearch to $es_required_version_statefile_value before proceeding with soup to avoid potential data loss! This command can take up to an hour to complete."
-        timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$statefile"
-        if [[ $? -ne 0 ]]; then
+        if ! timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$statefile"; then
             echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
 
             echo "A previous required intermediate Elasticsearch upgrade to $es_required_version_statefile_value has yet to successfully complete across the grid. Please allow time for all Searchnodes/Heavynodes to have upgraded Elasticsearch to $es_required_version_statefile_value before running soup again to avoid potential data loss!"
@@ -826,6 +827,7 @@ verify_es_version_compatibility() {
         return 0
     fi
 
+    # shellcheck disable=SC2076 # Do not want a regex here eg usage " 8.18.8 9.0.8 " =~ " 9.0.8 "
     if [[ " ${es_upgrade_map[$es_version]} " =~ " $target_es_version " || "$es_version" == "$target_es_version" ]]; then
         # supported upgrade
         return 0
@@ -834,7 +836,7 @@ verify_es_version_compatibility() {
         if [[ -z "$compatible_versions" ]]; then
             # If current ES version is not explicitly defined in the upgrade map, we know they have an intermediate upgrade to do.
             # We default to the lowest ES version defined in es_to_so_version as $first_es_required_version
-            local first_es_required_version=$(printf '%s\n' "${!es_to_so_version[@]}" | sort -V | head -n1)
+            first_es_required_version=$(printf '%s\n' "${!es_to_so_version[@]}" | sort -V | head -n1)
             next_step_so_version=${es_to_so_version[$first_es_required_version]}
             required_es_upgrade_version="$first_es_required_version"
         else
@@ -853,7 +855,7 @@ verify_es_version_compatibility() {
         if [[ $is_airgap -eq 0 ]]; then
             run_airgap_intermediate_upgrade
         else
-            if [[ ! -z $ISOLOC ]]; then
+            if [[ -n $ISOLOC ]]; then
                 originally_requested_iso_location="$ISOLOC"
             fi
             # Make sure ISOLOC is not set. Network installs that used soup -f would have ISOLOC set.
@@ -885,7 +887,8 @@ wait_for_salt_minion_with_restart() {
 }
 
 run_airgap_intermediate_upgrade() {
-    local originally_requested_so_version=$(cat $UPDATE_DIR/VERSION)
+    local originally_requested_so_version
+    originally_requested_so_version=$(cat "$UPDATE_DIR/VERSION")
     # preserve ISOLOC value, so we can try to use it post intermediate upgrade
     local originally_requested_iso_location="$ISOLOC"
 
@@ -897,7 +900,8 @@ run_airgap_intermediate_upgrade() {
 
     while [[ -z "$next_iso_location" ]] || [[ ! -f "$next_iso_location" && ! -b "$next_iso_location" ]]; do
         # List removable devices if any are present
-        local removable_devices=$(lsblk -no PATH,SIZE,TYPE,MOUNTPOINTS,RM | awk '$NF==1')
+        local removable_devices
+        removable_devices=$(lsblk -no PATH,SIZE,TYPE,MOUNTPOINTS,RM | awk '$NF==1')
         if [[ -n "$removable_devices" ]]; then
             echo "PATH        SIZE    TYPE    MOUNTPOINTS    RM"
             echo "$removable_devices"
@@ -918,21 +922,21 @@ run_airgap_intermediate_upgrade() {
 
     echo "Using $next_iso_location for required intermediary upgrade."
     exec bash <<EOF
-        ISOLOC=$next_iso_location soup -y && \
-        ISOLOC=$next_iso_location soup -y && \
+        ISOLOC="$next_iso_location" soup -y && \
+        ISOLOC="$next_iso_location" soup -y && \
 
         echo -e "\n##############################################################################################################################\n" && \
         echo -e "Verifying Elasticsearch was successfully upgraded to $required_es_upgrade_version across the grid. This part can take a while as Searchnodes/Heavynodes sync up with the Manager! \n\nOnce verification completes the next soup will begin automatically. If verification takes longer than 1 hour it will stop waiting and your grid will remain at $next_step_so_version. Allowing for all Searchnodes/Heavynodes to upgrade Elasticsearch to the required version on their own time.\n" && \
 
-        timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh $required_es_upgrade_version $es_required_version_statefile && \
+        timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh "$required_es_upgrade_version" "$es_required_version_statefile" && \
 
         echo -e "\n##############################################################################################################################\n" && \
 
         # automatically start the next soup if the original ISO isn't using the same block device we just used
         if [[ -n "$originally_requested_iso_location" ]] && [[ "$originally_requested_iso_location" != "$next_iso_location" ]]; then
             umount /tmp/soagupdate
-            ISOLOC=$originally_requested_iso_location soup -y && \
-            ISOLOC=$originally_requested_iso_location soup -y
+            ISOLOC="$originally_requested_iso_location" soup -y && \
+            ISOLOC="$originally_requested_iso_location" soup -y
         else
             echo "Could not automatically start next soup to $originally_requested_so_version. Soup will now exit here at $(cat /etc/soversion)" && \
 
@@ -948,29 +952,29 @@ run_network_intermediate_upgrade() {
     if [[ -n "$BRANCH" ]]; then
         local originally_requested_so_branch="$BRANCH"
     else
-        local originally_requested_so_branch="2.4/main"
+        local originally_requested_so_branch="3/main"
     fi
 
     echo "Starting automated intermediate upgrade to $next_step_so_version."
     echo "After completion, the system will automatically attempt to upgrade to the latest version."
     echo -e "\n##############################################################################################################################\n"
     exec bash << EOF
-        BRANCH=$next_step_so_version soup -y && \
-        BRANCH=$next_step_so_version soup -y && \
+        BRANCH="$next_step_so_version" soup -y && \
+        BRANCH="$next_step_so_version" soup -y && \
 
         echo -e "\n##############################################################################################################################\n" && \
         echo -e "Verifying Elasticsearch was successfully upgraded to $required_es_upgrade_version across the grid. This part can take a while as Searchnodes/Heavynodes sync up with the Manager! \n\nOnce verification completes the next soup will begin automatically. If verification takes longer than 1 hour it will stop waiting and your grid will remain at $next_step_so_version. Allowing for all Searchnodes/Heavynodes to upgrade Elasticsearch to the required version on their own time.\n" && \
 
-        timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh $required_es_upgrade_version $es_required_version_statefile && \
+        timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh "$required_es_upgrade_version" "$es_required_version_statefile" && \
 
         echo -e "\n##############################################################################################################################\n" && \
         if [[ -n "$originally_requested_iso_location" ]]; then
             # nonairgap soup that used -f originally, runs intermediate upgrade using network + BRANCH, later coming back to the original ISO for the last soup
-            ISOLOC=$originally_requested_iso_location soup -y && \
-            ISOLOC=$originally_requested_iso_location soup -y
+            ISOLOC="$originally_requested_iso_location" soup -y && \
+            ISOLOC="$originally_requested_iso_location" soup -y
         else
-            BRANCH=$originally_requested_so_branch soup -y && \
-            BRANCH=$originally_requested_so_branch soup -y
+            BRANCH="$originally_requested_so_branch" soup -y && \
+            BRANCH="$originally_requested_so_branch" soup -y
         fi
         echo -e "\n##############################################################################################################################\n"
 EOF

From b6a3d1889c1d3077eea63935379dd7e3873bc3c7 Mon Sep 17 00:00:00 2001
From: Mike Reeves <reevesmk@gmail.com>
Date: Mon, 20 Apr 2026 14:40:32 -0400
Subject: [PATCH 03/16] Fix soup state.apply args for postgres provisioning

state.apply takes a single mods argument; space-separated names are not
a list, so `state.apply postgres.auth postgres.telegraf_users` was only
applying postgres.auth and silently dropping the telegraf_users state.

Use comma-separated mods and add queue=True to match the rest of soup.
---
 salt/manager/tools/sbin/soup | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index c25358418..0adffef86 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -496,7 +496,7 @@ post_to_3.1.0() {
   # container. Then push a telegraf state to every minion so their telegraf.conf
   # picks up the new credentials on the first apply after soup.
   echo "Provisioning Telegraf Postgres users for existing minions."
-  salt-call --local state.apply postgres.auth postgres.telegraf_users || true
+  salt-call --local state.apply postgres.auth,postgres.telegraf_users queue=True || true
 
   POSTVERSION=3.1.0
 }

From 3ecd19d085fb56f97b56cc7602e36ed56d49f2e4 Mon Sep 17 00:00:00 2001
From: Mike Reeves <reevesmk@gmail.com>
Date: Mon, 20 Apr 2026 16:03:02 -0400
Subject: [PATCH 04/16] Move telegraf_output from global pillar to telegraf
 pillar
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The Telegraf backend selector lived at global.telegraf_output but it is
a Telegraf-scoped setting, not a cross-cutting grid global. Move both
the value and the UI annotation under the telegraf pillar so it shows
up alongside the other Telegraf tuning knobs in the Configuration UI.

- salt/telegraf/defaults.yaml:    add telegraf.output: BOTH
- salt/telegraf/soc_telegraf.yaml: add telegraf.output annotation
- salt/global/defaults.yaml:      remove global.telegraf_output
- salt/global/soc_global.yaml:    remove global.telegraf_output annotation
- salt/vars/globals.map.jinja:    drop telegraf_output from GLOBALS
- salt/firewall/map.jinja:        read via pillar.get('telegraf:output')
- salt/postgres/telegraf_users.sls: read via pillar.get('telegraf:output')
- salt/telegraf/etc/telegraf.conf: read via TELEGRAFMERGED.output
- salt/postgres/tools/sbin/so-stats-show: update user-facing docs

No behavioral change — default stays BOTH.
---
 salt/firewall/map.jinja                | 2 +-
 salt/global/defaults.yaml              | 3 +--
 salt/global/soc_global.yaml            | 8 --------
 salt/postgres/telegraf_users.sls       | 2 +-
 salt/postgres/tools/sbin/so-stats-show | 4 ++--
 salt/telegraf/defaults.yaml            | 1 +
 salt/telegraf/etc/telegraf.conf        | 2 +-
 salt/telegraf/soc_telegraf.yaml        | 9 +++++++++
 salt/vars/globals.map.jinja            | 1 -
 9 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/salt/firewall/map.jinja b/salt/firewall/map.jinja
index 2821f62b4..b0c96de72 100644
--- a/salt/firewall/map.jinja
+++ b/salt/firewall/map.jinja
@@ -56,7 +56,7 @@
 {% endif %}
 
 {# Open Postgres (5432) to minion hostgroups when Telegraf is configured to write to Postgres #}
-{% set TG_OUT = (GLOBALS.telegraf_output | default('INFLUXDB')) | upper %}
+{% set TG_OUT = salt['pillar.get']('telegraf:output', 'BOTH') | upper %}
 {% if TG_OUT in ['POSTGRES', 'BOTH'] %}
 {%   if role.startswith('manager') or role == 'standalone' or role == 'eval' %}
 {%     for r in ['sensor', 'searchnode', 'heavynode', 'receiver', 'fleet', 'idh', 'desktop', 'import'] %}
diff --git a/salt/global/defaults.yaml b/salt/global/defaults.yaml
index d041306a7..92b9c1c1a 100644
--- a/salt/global/defaults.yaml
+++ b/salt/global/defaults.yaml
@@ -1,4 +1,3 @@
 global:
   pcapengine: SURICATA
-  pipeline: REDIS
-  telegraf_output: BOTH
\ No newline at end of file
+  pipeline: REDIS
\ No newline at end of file
diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml
index 61646168f..31d9f8d3b 100644
--- a/salt/global/soc_global.yaml
+++ b/salt/global/soc_global.yaml
@@ -59,13 +59,5 @@ global:
     description: Allows use of Endgame with Security Onion. This feature requires a license from Endgame.
     global: True
     advanced: True
-  telegraf_output:
-    description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation.
-    options:
-      - INFLUXDB
-      - POSTGRES
-      - BOTH
-    global: True
-    advanced: True
     helpLink: influxdb
 
diff --git a/salt/postgres/telegraf_users.sls b/salt/postgres/telegraf_users.sls
index cab65d8a8..6bcf0900c 100644
--- a/salt/postgres/telegraf_users.sls
+++ b/salt/postgres/telegraf_users.sls
@@ -7,7 +7,7 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 
-{% set TG_OUT = (GLOBALS.telegraf_output | default('INFLUXDB')) | upper %}
+{% set TG_OUT = salt['pillar.get']('telegraf:output', 'BOTH') | upper %}
 {% if TG_OUT in ['POSTGRES', 'BOTH'] %}
 
 # docker_container.running returns as soon as the container starts, but on
diff --git a/salt/postgres/tools/sbin/so-stats-show b/salt/postgres/tools/sbin/so-stats-show
index 102b51ccd..3cf7a05d8 100644
--- a/salt/postgres/tools/sbin/so-stats-show
+++ b/salt/postgres/tools/sbin/so-stats-show
@@ -24,7 +24,7 @@ Shows the most recent CPU, memory, disk, and load metrics for each host
 from the so_telegraf Postgres database. Without an argument, reports on
 every host that has data. With a host, limits output to that one.
 
-Requires: sudo, so-postgres running, global.telegraf_output set to
+Requires: sudo, so-postgres running, telegraf.output set to
 POSTGRES or BOTH.
 EOF
   exit 1
@@ -56,7 +56,7 @@ so_psql() {
 }
 
 if ! docker exec so-postgres psql -U postgres -lqt 2>/dev/null | cut -d\| -f1 | grep -qw so_telegraf; then
-  echo "Database so_telegraf not found. Is global.telegraf_output set to POSTGRES or BOTH?"
+  echo "Database so_telegraf not found. Is telegraf.output set to POSTGRES or BOTH?"
   exit 2
 fi
 
diff --git a/salt/telegraf/defaults.yaml b/salt/telegraf/defaults.yaml
index ef6c2bc77..ead122b0a 100644
--- a/salt/telegraf/defaults.yaml
+++ b/salt/telegraf/defaults.yaml
@@ -1,5 +1,6 @@
 telegraf:
   enabled: False
+  output: BOTH
   config:
     interval: '30s'
     metric_batch_size: 1000
diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf
index d28dc7f96..ee13e33d0 100644
--- a/salt/telegraf/etc/telegraf.conf
+++ b/salt/telegraf/etc/telegraf.conf
@@ -8,7 +8,7 @@
 {%- set ZEEK_ENABLED = salt['pillar.get']('zeek:enabled', True) %}
 {%- set MDENGINE = GLOBALS.md_engine %}
 {%- set LOGSTASH_ENABLED = LOGSTASH_MERGED.enabled %}
-{%- set TG_OUT = GLOBALS.telegraf_output | upper %}
+{%- set TG_OUT = TELEGRAFMERGED.output | upper %}
 {%- set PG_HOST = GLOBALS.manager_ip %}
 {%- set PG_SAFE = GLOBALS.minion_id | replace('.','_') | replace('-','_') | lower %}
 {%- set PG_USER = 'so_telegraf_' ~ PG_SAFE %}
diff --git a/salt/telegraf/soc_telegraf.yaml b/salt/telegraf/soc_telegraf.yaml
index 40ae7fed8..4b9a2e3d1 100644
--- a/salt/telegraf/soc_telegraf.yaml
+++ b/salt/telegraf/soc_telegraf.yaml
@@ -4,6 +4,15 @@ telegraf:
     forcedType: bool
     advanced: True
     helpLink: influxdb
+  output:
+    description: Selects the backend(s) Telegraf writes metrics to. INFLUXDB keeps the current behavior; POSTGRES writes to the grid's Postgres instance; BOTH dual-writes for migration validation.
+    options:
+      - INFLUXDB
+      - POSTGRES
+      - BOTH
+    global: True
+    advanced: True
+    helpLink: influxdb
   config:
     interval:
       description: Data collection interval.
diff --git a/salt/vars/globals.map.jinja b/salt/vars/globals.map.jinja
index 787691b13..385db02ae 100644
--- a/salt/vars/globals.map.jinja
+++ b/salt/vars/globals.map.jinja
@@ -24,7 +24,6 @@
     'md_engine': INIT.PILLAR.global.mdengine,
     'pcap_engine': GLOBALMERGED.pcapengine,
     'pipeline': GLOBALMERGED.pipeline,
-    'telegraf_output': GLOBALMERGED.telegraf_output,
     'so_version': INIT.PILLAR.global.soversion,
     'so_docker_gateway': DOCKERMERGED.gateway,
     'so_docker_range': DOCKERMERGED.range,

From b69e50542acd23e28d856136b5b0d632e718d863 Mon Sep 17 00:00:00 2001
From: Mike Reeves <reevesmk@gmail.com>
Date: Mon, 20 Apr 2026 16:06:01 -0400
Subject: [PATCH 05/16] Use TELEGRAFMERGED for telegraf.output and de-jinja
 pg_hba.conf
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- firewall/map.jinja and postgres/telegraf_users.sls now pull the
  telegraf output selector through TELEGRAFMERGED so the defaults.yaml
  value (BOTH) is the source of truth and pillar overrides merge in
  cleanly. pillar.get with a hardcoded fallback was brittle and would
  disagree with defaults.yaml if the two ever diverged.
- Rename salt/postgres/files/pg_hba.conf.jinja to pg_hba.conf and drop
  template: jinja from config.sls — the file has no jinja besides the
  comment header.
---
 salt/firewall/map.jinja                                | 3 ++-
 salt/postgres/config.sls                               | 3 +--
 salt/postgres/files/{pg_hba.conf.jinja => pg_hba.conf} | 9 +++++----
 salt/postgres/telegraf_users.sls                       | 3 ++-
 4 files changed, 10 insertions(+), 8 deletions(-)
 rename salt/postgres/files/{pg_hba.conf.jinja => pg_hba.conf} (67%)

diff --git a/salt/firewall/map.jinja b/salt/firewall/map.jinja
index b0c96de72..61f8215b8 100644
--- a/salt/firewall/map.jinja
+++ b/salt/firewall/map.jinja
@@ -1,5 +1,6 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{% from 'telegraf/map.jinja' import TELEGRAFMERGED %}
 {% import_yaml 'firewall/defaults.yaml' as FIREWALL_DEFAULT %}
 
 {# add our ip to self #}
@@ -56,7 +57,7 @@
 {% endif %}
 
 {# Open Postgres (5432) to minion hostgroups when Telegraf is configured to write to Postgres #}
-{% set TG_OUT = salt['pillar.get']('telegraf:output', 'BOTH') | upper %}
+{% set TG_OUT = TELEGRAFMERGED.output | upper %}
 {% if TG_OUT in ['POSTGRES', 'BOTH'] %}
 {%   if role.startswith('manager') or role == 'standalone' or role == 'eval' %}
 {%     for r in ['sensor', 'searchnode', 'heavynode', 'receiver', 'fleet', 'idh', 'desktop', 'import'] %}
diff --git a/salt/postgres/config.sls b/salt/postgres/config.sls
index 76a926d59..efa9dba93 100644
--- a/salt/postgres/config.sls
+++ b/salt/postgres/config.sls
@@ -65,11 +65,10 @@ postgresconf:
 postgreshba:
   file.managed:
     - name: /opt/so/conf/postgres/pg_hba.conf
-    - source: salt://postgres/files/pg_hba.conf.jinja
+    - source: salt://postgres/files/pg_hba.conf
     - user: 939
     - group: 939
     - mode: 640
-    - template: jinja
 
 postgres_super_secret:
   file.managed:
diff --git a/salt/postgres/files/pg_hba.conf.jinja b/salt/postgres/files/pg_hba.conf
similarity index 67%
rename from salt/postgres/files/pg_hba.conf.jinja
rename to salt/postgres/files/pg_hba.conf
index 1d6a22a04..e7d31c05f 100644
--- a/salt/postgres/files/pg_hba.conf.jinja
+++ b/salt/postgres/files/pg_hba.conf
@@ -1,7 +1,8 @@
-{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-   https://securityonion.net/license; you may not use this file except in compliance with the
-   Elastic License 2.0. #}
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+#
 # Managed by Salt — do not edit by hand.
 # Client authentication config: only local (Unix socket) connections and TLS-wrapped TCP
 # connections are accepted. Plain-text `host ...` lines are intentionally omitted so a
diff --git a/salt/postgres/telegraf_users.sls b/salt/postgres/telegraf_users.sls
index 6bcf0900c..8d718519c 100644
--- a/salt/postgres/telegraf_users.sls
+++ b/salt/postgres/telegraf_users.sls
@@ -6,8 +6,9 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'telegraf/map.jinja' import TELEGRAFMERGED %}
 
-{% set TG_OUT = salt['pillar.get']('telegraf:output', 'BOTH') | upper %}
+{% set TG_OUT = TELEGRAFMERGED.output | upper %}
 {% if TG_OUT in ['POSTGRES', 'BOTH'] %}
 
 # docker_container.running returns as soon as the container starts, but on

From 80bf07ffd807d74316bd10aefa0064cb121b4cc5 Mon Sep 17 00:00:00 2001
From: Mike Reeves <reevesmk@gmail.com>
Date: Mon, 20 Apr 2026 16:36:37 -0400
Subject: [PATCH 06/16] Flesh out soc_postgres.yaml annotations
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add Configuration-UI annotations for every postgres pillar key defined
in defaults.yaml, not just telegraf.retention_days:

- postgres.enabled          — readonly; admin-visible but toggled via state
- postgres.telegraf.retention_days — drop advanced so user-tunable knobs
  surface in the default view
- postgres.config.max_connections, shared_buffers, log_min_messages —
  user-tunable performance/verbosity knobs, not advanced
- postgres.config.listen_addresses, port, ssl, ssl_cert_file, ssl_key_file,
  ssl_ca_file, hba_file, log_destination, logging_collector,
  shared_preload_libraries, cron.database_name — infra/Salt-managed,
  marked advanced so they're visible but out of the way

No defaults.yaml change; value-side stays the same.
---
 salt/postgres/soc_postgres.yaml | 82 +++++++++++++++++++++++++++++++++
 1 file changed, 82 insertions(+)

diff --git a/salt/postgres/soc_postgres.yaml b/salt/postgres/soc_postgres.yaml
index 8b4e22921..d111e6923 100644
--- a/salt/postgres/soc_postgres.yaml
+++ b/salt/postgres/soc_postgres.yaml
@@ -1,7 +1,89 @@
 postgres:
+  enabled:
+    description: Whether the PostgreSQL database container is enabled on this grid. Backs the assistant store and the Telegraf metrics database.
+    forcedType: bool
+    readonly: True
+    helpLink: influxdb
   telegraf:
     retention_days:
       description: Number of days of Telegraf metrics to keep in the so_telegraf database. Older partitions are dropped hourly by pg_partman.
       forcedType: int
+      helpLink: influxdb
+  config:
+    max_connections:
+      description: Maximum number of concurrent PostgreSQL connections.
+      forcedType: int
+      global: True
+      helpLink: influxdb
+    shared_buffers:
+      description: Amount of memory PostgreSQL uses for shared buffers (e.g. 256MB, 1GB). Raising this improves read cache hit rate at the cost of system RAM.
+      global: True
+      helpLink: influxdb
+    log_min_messages:
+      description: Minimum severity of server messages written to the PostgreSQL log.
+      options:
+        - debug1
+        - info
+        - notice
+        - warning
+        - error
+        - log
+        - fatal
+      global: True
+      helpLink: influxdb
+    listen_addresses:
+      description: Interfaces PostgreSQL listens on. Must remain '*' so clients on the docker bridge network can connect.
+      global: True
+      advanced: True
+      helpLink: influxdb
+    port:
+      description: TCP port PostgreSQL listens on inside the container. Firewall rules and container port mapping assume 5432.
+      forcedType: int
+      global: True
+      advanced: True
+      helpLink: influxdb
+    ssl:
+      description: Whether PostgreSQL accepts TLS connections. Must remain 'on' — pg_hba.conf requires hostssl for TCP.
+      global: True
+      advanced: True
+      helpLink: influxdb
+    ssl_cert_file:
+      description: Path (inside the container) to the TLS server certificate. Salt-managed.
+      global: True
+      advanced: True
+      helpLink: influxdb
+    ssl_key_file:
+      description: Path (inside the container) to the TLS server private key. Salt-managed.
+      global: True
+      advanced: True
+      helpLink: influxdb
+    ssl_ca_file:
+      description: Path (inside the container) to the CA bundle PostgreSQL uses to verify client certificates. Salt-managed.
+      global: True
+      advanced: True
+      helpLink: influxdb
+    hba_file:
+      description: Path (inside the container) to the pg_hba.conf authentication file. Salt-managed — edit salt/postgres/files/pg_hba.conf.
+      global: True
+      advanced: True
+      helpLink: influxdb
+    log_destination:
+      description: Where PostgreSQL writes its server log. 'stderr' routes to the container log stream.
+      global: True
+      advanced: True
+      helpLink: influxdb
+    logging_collector:
+      description: Whether to run a separate logging collector process. Disabled because the docker log stream already captures stderr.
+      global: True
+      advanced: True
+      helpLink: influxdb
+    shared_preload_libraries:
+      description: Comma-separated list of extensions loaded at server start. Required for pg_cron which drives pg_partman maintenance — do not remove.
+      global: True
+      advanced: True
+      helpLink: influxdb
+    cron.database_name:
+      description: Database pg_cron schedules jobs in. Must be so_telegraf so partman maintenance runs in the right database context.
+      global: True
       advanced: True
       helpLink: influxdb

From ee89b78751b2cc4117ec9507ed82c5da14b0f0cd Mon Sep 17 00:00:00 2001
From: Mike Reeves <reevesmk@gmail.com>
Date: Mon, 20 Apr 2026 19:54:06 -0400
Subject: [PATCH 07/16] Fire telegraf user sync on salt/key accept, not
 salt/auth
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

salt/auth fires on every minion authentication — including every minion
restart and every master restart — so the reactor was re-running the
postgres.auth + postgres.telegraf_users + telegraf orchestration for
every already-accepted minion on every reconnect. The underlying states
are idempotent, so this was wasted work and log noise, not a correctness
issue.

Switch the subscription to salt/key, which fires only when the master
actually changes a key's state (accept / reject / delete). Match the
pattern used by salt/reactor/check_hypervisor.sls (registered in
salt/salt/cloud/reactor_config_hypervisor.sls) and add the result==True
guard so half-failed key operations don't trigger the orchestration.
---
 salt/reactor/telegraf_user_sync.sls | 4 ++--
 salt/salt/master.sls                | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/salt/reactor/telegraf_user_sync.sls b/salt/reactor/telegraf_user_sync.sls
index abf35d3b2..ec0aec336 100644
--- a/salt/reactor/telegraf_user_sync.sls
+++ b/salt/reactor/telegraf_user_sync.sls
@@ -3,8 +3,8 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-{# Fires on salt/auth. Only act on accepted keys — ignore pending/reject. #}
-{% if data.get('act') == 'accept' and data.get('id') %}
+{# Fires on salt/key. Only act on successful key acceptance — not reauth. #}
+{% if data.get('act') == 'accept' and data.get('result') == True and data.get('id') %}
 
 {{ data['id'] }}_telegraf_pg_sync:
   runner.state.orchestrate:
diff --git a/salt/salt/master.sls b/salt/salt/master.sls
index 7e3e48074..e61b09d21 100644
--- a/salt/salt/master.sls
+++ b/salt/salt/master.sls
@@ -67,7 +67,7 @@ reactor_config_telegraf:
     - name: /etc/salt/master.d/reactor_telegraf.conf
     - contents: |
         reactor:
-          - 'salt/auth':
+          - 'salt/key':
             - /opt/so/saltstack/default/salt/reactor/telegraf_user_sync.sls
     - user: root
     - group: root

From 72105f1f2f29d25039a2f157b4685d73402ce201 Mon Sep 17 00:00:00 2001
From: Mike Reeves <reevesmk@gmail.com>
Date: Tue, 21 Apr 2026 09:31:45 -0400
Subject: [PATCH 08/16] Drop telegraf push from new-minion orch; highstate
 covers it

New minions run highstate as part of onboarding, which already applies
the telegraf state with the fresh pillar entry we just wrote. Pushing
telegraf a second time from the reactor is redundant.

- Remove the MINION-scoped salt.state block from the orch; keep only
  the manager-side postgres.auth + postgres.telegraf_users provisioning.
- Stop passing minion_id as pillar in the reactor; the orch doesn't
  reference it anymore.
---
 salt/orch/telegraf_postgres_sync.sls | 17 +++++------------
 salt/reactor/telegraf_user_sync.sls  |  2 --
 2 files changed, 5 insertions(+), 14 deletions(-)

diff --git a/salt/orch/telegraf_postgres_sync.sls b/salt/orch/telegraf_postgres_sync.sls
index 90c42fc07..f2a3d950f 100644
--- a/salt/orch/telegraf_postgres_sync.sls
+++ b/salt/orch/telegraf_postgres_sync.sls
@@ -3,9 +3,13 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-{% set MINION = salt['pillar.get']('minion_id') %}
 {% set MANAGER = salt['pillar.get']('setup:manager') or salt['grains.get']('master') %}
 
+# Fired by salt/reactor/telegraf_user_sync.sls when salt-key accepts a new
+# minion. Only provisions the per-minion pillar entry and DB role on the
+# manager; the minion itself will pick up its telegraf config on its first
+# highstate during onboarding, so there's no need to push the telegraf state
+# from here.
 manager_sync_telegraf_pg_users:
   salt.state:
     - tgt: {{ MANAGER }}
@@ -13,14 +17,3 @@ manager_sync_telegraf_pg_users:
       - postgres.auth
       - postgres.telegraf_users
     - queue: True
-
-{% if MINION and MINION != MANAGER %}
-{{ MINION }}_apply_telegraf:
-  salt.state:
-    - tgt: {{ MINION }}
-    - sls:
-      - telegraf
-    - queue: True
-    - require:
-      - salt: manager_sync_telegraf_pg_users
-{% endif %}
diff --git a/salt/reactor/telegraf_user_sync.sls b/salt/reactor/telegraf_user_sync.sls
index ec0aec336..4830dbc53 100644
--- a/salt/reactor/telegraf_user_sync.sls
+++ b/salt/reactor/telegraf_user_sync.sls
@@ -10,8 +10,6 @@
   runner.state.orchestrate:
     - args:
       - mods: orch.telegraf_postgres_sync
-      - pillar:
-          minion_id: {{ data['id'] }}
 
 {% do salt.log.info('telegraf_user_sync reactor: syncing telegraf PG user for minion %s' % data['id']) %}
 

From 37e925769884563364f19b3eada30bfa13a23b92 Mon Sep 17 00:00:00 2001
From: Mike Reeves <reevesmk@gmail.com>
Date: Tue, 21 Apr 2026 09:33:47 -0400
Subject: [PATCH 09/16] Change so-postgres final_octet to 47

---
 salt/docker/defaults.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml
index 900d2cf53..81ff07190 100644
--- a/salt/docker/defaults.yaml
+++ b/salt/docker/defaults.yaml
@@ -238,7 +238,7 @@ docker:
       extra_env: []
       ulimits: []
     'so-postgres':
-      final_octet: 89
+      final_octet: 47
       port_bindings:
         - 0.0.0.0:5432:5432
       custom_bind_mounts: []

From f72c30abd0797c266be63ae86c857ecae92ab203 Mon Sep 17 00:00:00 2001
From: Mike Reeves <reevesmk@gmail.com>
Date: Tue, 21 Apr 2026 09:35:59 -0400
Subject: [PATCH 10/16] Have postgres.telegraf_users include postgres.enabled
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

postgres_wait_ready requires docker_container: so-postgres, which is
declared in postgres.enabled. Running postgres.telegraf_users on its own
— as the reactor orch and the soup post-upgrade step both do — errored
because Salt couldn't resolve the require.

Include postgres.enabled from postgres.telegraf_users so the container
state is always in the render. postgres.enabled already includes
telegraf_users; Salt de-duplicates the circular include and the included
states are all idempotent, so repeated application is a no-op.
---
 salt/postgres/telegraf_users.sls | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/salt/postgres/telegraf_users.sls b/salt/postgres/telegraf_users.sls
index 8d718519c..dbbc0f03e 100644
--- a/salt/postgres/telegraf_users.sls
+++ b/salt/postgres/telegraf_users.sls
@@ -8,6 +8,13 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'telegraf/map.jinja' import TELEGRAFMERGED %}
 
+{# postgres_wait_ready below requires `docker_container: so-postgres`, which is
+   declared in postgres.enabled. Include it here so state.apply postgres.telegraf_users
+   on its own (from the reactor orch or from soup) still has that ID in scope. Salt
+   de-duplicates the circular include. #}
+include:
+  - postgres.enabled
+
 {% set TG_OUT = TELEGRAFMERGED.output | upper %}
 {% if TG_OUT in ['POSTGRES', 'BOTH'] %}
 

From a902f667ba6c8ea3a4db477793c42535fa2cd357 Mon Sep 17 00:00:00 2001
From: Mike Reeves <reevesmk@gmail.com>
Date: Tue, 21 Apr 2026 09:37:35 -0400
Subject: [PATCH 11/16] Target manager by role grain in telegraf_postgres_sync
 orch

The previous MANAGER resolution used pillar.get('setup:manager') with a
fallback to grains.get('master'). Neither works from the reactor:
setup:manager is only populated by the setup workflow (not by reactor
runs), and grains.master returns the minion's master-hostname setting,
not a targetable minion id.

Match the pattern used by orch/delete_hypervisor.sls: compound-target
whichever minion is the manager via role grain.
---
 salt/orch/telegraf_postgres_sync.sls | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/salt/orch/telegraf_postgres_sync.sls b/salt/orch/telegraf_postgres_sync.sls
index f2a3d950f..94be77137 100644
--- a/salt/orch/telegraf_postgres_sync.sls
+++ b/salt/orch/telegraf_postgres_sync.sls
@@ -3,16 +3,19 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-{% set MANAGER = salt['pillar.get']('setup:manager') or salt['grains.get']('master') %}
-
 # Fired by salt/reactor/telegraf_user_sync.sls when salt-key accepts a new
 # minion. Only provisions the per-minion pillar entry and DB role on the
 # manager; the minion itself will pick up its telegraf config on its first
 # highstate during onboarding, so there's no need to push the telegraf state
 # from here.
+#
+# Target the manager via role grains — same pattern as orch/delete_hypervisor.sls.
+# The reactor doesn't know the manager's minion id, and grains.master on the
+# runner is a hostname, not a targetable id.
 manager_sync_telegraf_pg_users:
   salt.state:
-    - tgt: {{ MANAGER }}
+    - tgt: 'G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone or G@role:so-eval'
+    - tgt_type: compound
     - sls:
       - postgres.auth
       - postgres.telegraf_users

From 89a6e7c0dd4745ad9baf8c83ae35a619d42097c4 Mon Sep 17 00:00:00 2001
From: Mike Reeves <reevesmk@gmail.com>
Date: Tue, 21 Apr 2026 09:39:58 -0400
Subject: [PATCH 12/16] Tidy config.sls makedirs and postgres helpLinks
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- config.sls: postgresconfdir creates /opt/so/conf/postgres, so the
  two subdirectories under it (postgressecretsdir, postgresinitdir)
  don't need their own makedirs — require the parent instead.
- soc_postgres.yaml: helpLink for every annotated key now points to
  'postgres' instead of the carried-over 'influxdb' slug.
---
 salt/postgres/config.sls        |  6 ++++--
 salt/postgres/soc_postgres.yaml | 30 +++++++++++++++---------------
 2 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/salt/postgres/config.sls b/salt/postgres/config.sls
index efa9dba93..11ca52649 100644
--- a/salt/postgres/config.sls
+++ b/salt/postgres/config.sls
@@ -21,7 +21,8 @@ postgressecretsdir:
     - user: 939
     - group: 939
     - mode: 700
-    - makedirs: True
+    - require:
+      - file: postgresconfdir
 
 postgresdatadir:
   file.directory:
@@ -42,7 +43,8 @@ postgresinitdir:
     - name: /opt/so/conf/postgres/init
     - user: 939
     - group: 939
-    - makedirs: True
+    - require:
+      - file: postgresconfdir
 
 postgresinitusers:
   file.managed:
diff --git a/salt/postgres/soc_postgres.yaml b/salt/postgres/soc_postgres.yaml
index d111e6923..4b25cd4f5 100644
--- a/salt/postgres/soc_postgres.yaml
+++ b/salt/postgres/soc_postgres.yaml
@@ -8,17 +8,17 @@ postgres:
     retention_days:
       description: Number of days of Telegraf metrics to keep in the so_telegraf database. Older partitions are dropped hourly by pg_partman.
       forcedType: int
-      helpLink: influxdb
+      helpLink: postgres
   config:
     max_connections:
       description: Maximum number of concurrent PostgreSQL connections.
       forcedType: int
       global: True
-      helpLink: influxdb
+      helpLink: postgres
     shared_buffers:
       description: Amount of memory PostgreSQL uses for shared buffers (e.g. 256MB, 1GB). Raising this improves read cache hit rate at the cost of system RAM.
       global: True
-      helpLink: influxdb
+      helpLink: postgres
     log_min_messages:
       description: Minimum severity of server messages written to the PostgreSQL log.
       options:
@@ -30,60 +30,60 @@ postgres:
         - log
         - fatal
       global: True
-      helpLink: influxdb
+      helpLink: postgres
     listen_addresses:
       description: Interfaces PostgreSQL listens on. Must remain '*' so clients on the docker bridge network can connect.
       global: True
       advanced: True
-      helpLink: influxdb
+      helpLink: postgres
     port:
       description: TCP port PostgreSQL listens on inside the container. Firewall rules and container port mapping assume 5432.
       forcedType: int
       global: True
       advanced: True
-      helpLink: influxdb
+      helpLink: postgres
     ssl:
       description: Whether PostgreSQL accepts TLS connections. Must remain 'on' — pg_hba.conf requires hostssl for TCP.
       global: True
       advanced: True
-      helpLink: influxdb
+      helpLink: postgres
     ssl_cert_file:
       description: Path (inside the container) to the TLS server certificate. Salt-managed.
       global: True
       advanced: True
-      helpLink: influxdb
+      helpLink: postgres
     ssl_key_file:
       description: Path (inside the container) to the TLS server private key. Salt-managed.
       global: True
       advanced: True
-      helpLink: influxdb
+      helpLink: postgres
     ssl_ca_file:
       description: Path (inside the container) to the CA bundle PostgreSQL uses to verify client certificates. Salt-managed.
       global: True
       advanced: True
-      helpLink: influxdb
+      helpLink: postgres
     hba_file:
       description: Path (inside the container) to the pg_hba.conf authentication file. Salt-managed — edit salt/postgres/files/pg_hba.conf.
       global: True
       advanced: True
-      helpLink: influxdb
+      helpLink: postgres
     log_destination:
       description: Where PostgreSQL writes its server log. 'stderr' routes to the container log stream.
       global: True
       advanced: True
-      helpLink: influxdb
+      helpLink: postgres
     logging_collector:
       description: Whether to run a separate logging collector process. Disabled because the docker log stream already captures stderr.
       global: True
       advanced: True
-      helpLink: influxdb
+      helpLink: postgres
     shared_preload_libraries:
       description: Comma-separated list of extensions loaded at server start. Required for pg_cron which drives pg_partman maintenance — do not remove.
       global: True
       advanced: True
-      helpLink: influxdb
+      helpLink: postgres
     cron.database_name:
       description: Database pg_cron schedules jobs in. Must be so_telegraf so partman maintenance runs in the right database context.
       global: True
       advanced: True
-      helpLink: influxdb
+      helpLink: postgres

From 84197fb33bba5d128391f344545209c4c3c3ae71 Mon Sep 17 00:00:00 2001
From: Mike Reeves <reevesmk@gmail.com>
Date: Tue, 21 Apr 2026 09:42:41 -0400
Subject: [PATCH 13/16] Move postgres backup script and cron to the postgres
 states
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The so-postgres-backup script and its cron were living under
salt/backup/config_backup.sls, which meant the backup script and cron
were deployed independently of whether postgres was enabled/disabled.

- Relocate salt/backup/tools/sbin/so-postgres-backup to
  salt/postgres/tools/sbin/so-postgres-backup so the existing
  postgres_sbin file.recurse in postgres/config.sls picks it up with
  everything else — no separate file.managed needed.
- Remove postgres_backup_script and so_postgres_backup from
  salt/backup/config_backup.sls.
- Add cron.present for so_postgres_backup to salt/postgres/enabled.sls
  and the matching cron.absent to salt/postgres/disabled.sls so the
  cron follows the container's lifecycle.
---
 salt/backup/config_backup.sls                 | 19 -------------------
 salt/postgres/disabled.sls                    |  6 ++++++
 salt/postgres/enabled.sls                     | 11 +++++++++++
 .../tools/sbin/so-postgres-backup             |  0
 4 files changed, 17 insertions(+), 19 deletions(-)
 rename salt/{backup => postgres}/tools/sbin/so-postgres-backup (100%)

diff --git a/salt/backup/config_backup.sls b/salt/backup/config_backup.sls
index c8e342463..a4297444b 100644
--- a/salt/backup/config_backup.sls
+++ b/salt/backup/config_backup.sls
@@ -33,22 +33,3 @@ so_config_backup:
     - month: '*'
     - dayweek: '*'
 
-postgres_backup_script:
-  file.managed:
-    - name: /usr/sbin/so-postgres-backup
-    - user: root
-    - group: root
-    - mode: 755
-    - source: salt://backup/tools/sbin/so-postgres-backup
-
-# Add postgres database backup
-so_postgres_backup:
-  cron.present:
-    - name: /usr/sbin/so-postgres-backup > /dev/null 2>&1
-    - identifier: so_postgres_backup
-    - user: root
-    - minute: '5'
-    - hour: '0'
-    - daymonth: '*'
-    - month: '*'
-    - dayweek: '*'
diff --git a/salt/postgres/disabled.sls b/salt/postgres/disabled.sls
index 56dc451b7..4b5b62328 100644
--- a/salt/postgres/disabled.sls
+++ b/salt/postgres/disabled.sls
@@ -18,6 +18,12 @@ so-postgres_so-status.disabled:
     - name: /opt/so/conf/so-status/so-status.conf
     - regex: ^so-postgres$
 
+so_postgres_backup:
+  cron.absent:
+    - name: /usr/sbin/so-postgres-backup > /dev/null 2>&1
+    - identifier: so_postgres_backup
+    - user: root
+
 {% else %}
 
 {{sls}}_state_not_allowed:
diff --git a/salt/postgres/enabled.sls b/salt/postgres/enabled.sls
index 4c5838466..b3abb621e 100644
--- a/salt/postgres/enabled.sls
+++ b/salt/postgres/enabled.sls
@@ -89,6 +89,17 @@ delete_so-postgres_so-status.disabled:
     - name: /opt/so/conf/so-status/so-status.conf
     - regex: ^so-postgres$
 
+so_postgres_backup:
+  cron.present:
+    - name: /usr/sbin/so-postgres-backup > /dev/null 2>&1
+    - identifier: so_postgres_backup
+    - user: root
+    - minute: '5'
+    - hour: '0'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+
 {% else %}
 
 {{sls}}_state_not_allowed:
diff --git a/salt/backup/tools/sbin/so-postgres-backup b/salt/postgres/tools/sbin/so-postgres-backup
similarity index 100%
rename from salt/backup/tools/sbin/so-postgres-backup
rename to salt/postgres/tools/sbin/so-postgres-backup

From bb71e44614f25b299041dbdb46eb908dd908ac4c Mon Sep 17 00:00:00 2001
From: Mike Reeves <reevesmk@gmail.com>
Date: Tue, 21 Apr 2026 09:57:35 -0400
Subject: [PATCH 14/16] Write per-minion telegraf creds to each minion's own
 pillar file

pillar/top.sls only distributes postgres.auth to manager-class roles,
so sensors / heavynodes / searchnodes / receivers / fleet / idh /
hypervisor / desktop minions never received the postgres telegraf
password they need to write metrics. Broadcasting the aggregate
postgres.auth pillar to every role would leak the so_postgres admin
password and every other minion's cred.

Fan out per-minion credentials into each minion's own pillar file at
/opt/so/saltstack/local/pillar/minions/<id>.sls. That file is already
distributed by pillar/top.sls exclusively to the matching minion via
`- minions.{{ grains.id }}`, so each minion sees only its own
postgres.telegraf.{user,pass} and nothing else.

- salt/postgres/auth.sls: after writing the manager-scoped aggregate
  pillar, fan the per-minion creds out via so-yaml.py replace for every
  up-minion. Creates the minion pillar file if missing. Requires
  postgres_auth_pillar so the manager pillar lands first.
- salt/telegraf/etc/telegraf.conf: consume postgres:telegraf:user and
  postgres:telegraf:pass directly from the minion's own pillar instead
  of walking postgres:auth:users which isn't visible off the manager.
---
 salt/postgres/auth.sls          | 29 +++++++++++++++++++++++++++++
 salt/telegraf/etc/telegraf.conf |  9 ++++++---
 2 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/salt/postgres/auth.sls b/salt/postgres/auth.sls
index 3da1bcde0..0b94ece99 100644
--- a/salt/postgres/auth.sls
+++ b/salt/postgres/auth.sls
@@ -49,6 +49,35 @@ postgres_auth_pillar:
                 pass: "{{ entry.pass }}"
               {% endfor %}
     - show_changes: False
+
+  {# Fan each minion's telegraf cred out to its own pillar file. The minions/
+     <id>.sls file is only served to that specific minion via pillar/top.sls
+     (`- minions.{{ grains.id }}`), so sensors, heavynodes, etc. see their own
+     credential without the admin password or anyone else's. Run per up-minion
+     so we have the original minion id (not just the safe-normalized version). #}
+  {% for mid in up_minions %}
+    {%- set safe = mid | replace('.','_') | replace('-','_') | lower %}
+    {%- set key = 'telegraf_' ~ safe %}
+    {%- set entry = telegraf_users.get(key) %}
+    {%- if entry %}
+
+postgres_telegraf_minion_pillar_{{ safe }}:
+  cmd.run:
+    - name: |
+        set -e
+        PILLAR_FILE=/opt/so/saltstack/local/pillar/minions/{{ mid }}.sls
+        if [ ! -f "$PILLAR_FILE" ]; then
+          echo '{}' > "$PILLAR_FILE"
+          chown socore:socore "$PILLAR_FILE" 2>/dev/null || true
+          chmod 640 "$PILLAR_FILE"
+        fi
+        /usr/sbin/so-yaml.py replace "$PILLAR_FILE" postgres.telegraf.user '{{ entry.user }}'
+        /usr/sbin/so-yaml.py replace "$PILLAR_FILE" postgres.telegraf.pass '{{ entry.pass }}'
+    - require:
+      - file: postgres_auth_pillar
+
+    {%- endif %}
+  {% endfor %}
 {% else %}
 
 {{sls}}_state_not_allowed:
diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf
index ee13e33d0..001a61d93 100644
--- a/salt/telegraf/etc/telegraf.conf
+++ b/salt/telegraf/etc/telegraf.conf
@@ -10,9 +10,12 @@
 {%- set LOGSTASH_ENABLED = LOGSTASH_MERGED.enabled %}
 {%- set TG_OUT = TELEGRAFMERGED.output | upper %}
 {%- set PG_HOST = GLOBALS.manager_ip %}
-{%- set PG_SAFE = GLOBALS.minion_id | replace('.','_') | replace('-','_') | lower %}
-{%- set PG_USER = 'so_telegraf_' ~ PG_SAFE %}
-{%- set PG_PASS = salt['pillar.get']('postgres:auth:users:telegraf_' ~ PG_SAFE ~ ':pass', '') %}
+{#- Per-minion telegraf creds are written into the minion's own pillar file
+    (/opt/so/saltstack/local/pillar/minions/<id>.sls) by postgres.auth on the
+    manager. Each minion only sees its own password — the aggregate map in
+    postgres:auth:users is manager-scoped. #}
+{%- set PG_USER = salt['pillar.get']('postgres:telegraf:user', '') %}
+{%- set PG_PASS = salt['pillar.get']('postgres:telegraf:pass', '') %}
 # Global tags can be specified here in key="value" format.
 [global_tags]
   role = "{{ GLOBALS.role.split('-') | last }}"

From a149ea7e8f39a5da7ca1db2c569ab4072bfad527 Mon Sep 17 00:00:00 2001
From: Mike Reeves <reevesmk@gmail.com>
Date: Tue, 21 Apr 2026 09:59:46 -0400
Subject: [PATCH 15/16] Skip per-minion pillar fan-out when cred is already in
 place

Every postgres.auth run was rewriting every minion pillar file via
two so-yaml.py replace calls, even when nothing had changed. Passwords
are only generated on first encounter (see the `if key not in
telegraf_users` guard) and never rotate, so re-writing the same values
on every apply is wasted work and noisy state output.

Add an `unless:` check that compares the already-written
postgres.telegraf.user to the one we'd set. If they match, skip the
fan-out entirely. On first apply for a new minion the key isn't there,
so the replace runs; on subsequent applies it's a no-op.
---
 salt/postgres/auth.sls | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/salt/postgres/auth.sls b/salt/postgres/auth.sls
index 0b94ece99..44c89c581 100644
--- a/salt/postgres/auth.sls
+++ b/salt/postgres/auth.sls
@@ -73,6 +73,13 @@ postgres_telegraf_minion_pillar_{{ safe }}:
         fi
         /usr/sbin/so-yaml.py replace "$PILLAR_FILE" postgres.telegraf.user '{{ entry.user }}'
         /usr/sbin/so-yaml.py replace "$PILLAR_FILE" postgres.telegraf.pass '{{ entry.pass }}'
+    {#- Skip if this minion's pillar file already carries a matching user.
+        Passwords are generated once per minion (see the `if key not in telegraf_users`
+        guard above) and never rotate, so once a cred is fanned out the file
+        doesn't need to be rewritten on subsequent auth runs. If we ever add
+        rotation, we'd need to delete postgres.telegraf to force a re-fan. #}
+    - unless: |
+        [ "$(/usr/sbin/so-yaml.py get -r /opt/so/saltstack/local/pillar/minions/{{ mid }}.sls postgres.telegraf.user 2>/dev/null)" = '{{ entry.user }}' ]
     - require:
       - file: postgres_auth_pillar
 

From 05f6503d61b9faae6d9f8587e1c11725f3407ca4 Mon Sep 17 00:00:00 2001
From: Mike Reeves <reevesmk@gmail.com>
Date: Tue, 21 Apr 2026 10:05:08 -0400
Subject: [PATCH 16/16] Gate postgres telegraf fan-out on reactor-provided
 minion id
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

postgres.auth was running an `unless` shell check per up-minion on every
manager highstate, even when nothing had changed — N fork+python starts
of so-yaml.py add up on large grids. The work is only needed when a
specific minion's key is accepted.

- salt/postgres/auth.sls: fan out only when postgres_fanout_minion
  pillar is set (targets that single minion). Manager highstates with
  no pillar take a zero-N code path.
- salt/reactor/telegraf_user_sync.sls: re-pass the accepted minion id
  as postgres_fanout_minion to the orch.
- salt/orch/telegraf_postgres_sync.sls: forward the pillar to the
  salt.state invocation so the state render sees it.
- salt/manager/tools/sbin/soup: for the one-time 3.1.0 backfill, drop
  the per-minion state.apply and do an in-shell loop over the minion
  pillar files using so-yaml.py directly. Skips minions that already
  have postgres.telegraf.user set.
---
 salt/manager/tools/sbin/soup         | 28 +++++++++++++++++++++++-----
 salt/orch/telegraf_postgres_sync.sls |  6 ++++++
 salt/postgres/auth.sls               | 26 +++++++++++---------------
 salt/reactor/telegraf_user_sync.sls  |  2 ++
 4 files changed, 42 insertions(+), 20 deletions(-)

diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup
index 0adffef86..c19fe487e 100755
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -490,14 +490,32 @@ up_to_3.1.0() {
 post_to_3.1.0() {
   /usr/sbin/so-kibana-space-defaults
 
-  # Provision per-minion Telegraf Postgres users for every minion known to the
-  # manager. postgres.auth iterates manage.up to generate any missing passwords;
-  # postgres.telegraf_users reconciles the roles and schemas inside the so-postgres
-  # container. Then push a telegraf state to every minion so their telegraf.conf
-  # picks up the new credentials on the first apply after soup.
+  # One-time backfill for minions that existed before the postgres Telegraf
+  # feature shipped. Generate the aggregate pillar on the manager and create
+  # the per-minion DB roles, then fan each minion's cred into its own pillar
+  # file. Going forward the reactor handles each new salt-key accept with a
+  # targeted fan-out, so a manager highstate no longer needs to iterate.
   echo "Provisioning Telegraf Postgres users for existing minions."
   salt-call --local state.apply postgres.auth,postgres.telegraf_users queue=True || true
 
+  AGGREGATE_PILLAR=/opt/so/saltstack/local/pillar/postgres/auth.sls
+  MINIONS_DIR=/opt/so/saltstack/local/pillar/minions
+  if [[ -f "$AGGREGATE_PILLAR" && -d "$MINIONS_DIR" ]]; then
+    for pillar_file in "$MINIONS_DIR"/*.sls; do
+      [[ -f "$pillar_file" ]] || continue
+      mid=$(basename "$pillar_file" .sls)
+      [[ "$mid" == adv_* ]] && continue
+      safe=$(echo "$mid" | tr '.-' '__' | tr '[:upper:]' '[:lower:]')
+      existing_user=$(so-yaml.py get -r "$pillar_file" postgres.telegraf.user 2>/dev/null || true)
+      [[ "$existing_user" == "so_telegraf_${safe}" ]] && continue
+      user=$(so-yaml.py get -r "$AGGREGATE_PILLAR" "postgres.auth.users.telegraf_${safe}.user" 2>/dev/null || true)
+      pass=$(so-yaml.py get -r "$AGGREGATE_PILLAR" "postgres.auth.users.telegraf_${safe}.pass" 2>/dev/null || true)
+      [[ -z "$user" || -z "$pass" ]] && continue
+      so-yaml.py replace "$pillar_file" postgres.telegraf.user "$user" >/dev/null
+      so-yaml.py replace "$pillar_file" postgres.telegraf.pass "$pass" >/dev/null
+    done
+  fi
+
   POSTVERSION=3.1.0
 }
 
diff --git a/salt/orch/telegraf_postgres_sync.sls b/salt/orch/telegraf_postgres_sync.sls
index 94be77137..5b11d1619 100644
--- a/salt/orch/telegraf_postgres_sync.sls
+++ b/salt/orch/telegraf_postgres_sync.sls
@@ -12,6 +12,8 @@
 # Target the manager via role grains — same pattern as orch/delete_hypervisor.sls.
 # The reactor doesn't know the manager's minion id, and grains.master on the
 # runner is a hostname, not a targetable id.
+{% set FANOUT_MINION = salt['pillar.get']('postgres_fanout_minion', '') %}
+
 manager_sync_telegraf_pg_users:
   salt.state:
     - tgt: 'G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone or G@role:so-eval'
@@ -20,3 +22,7 @@ manager_sync_telegraf_pg_users:
       - postgres.auth
       - postgres.telegraf_users
     - queue: True
+    {% if FANOUT_MINION %}
+    - pillar:
+        postgres_fanout_minion: {{ FANOUT_MINION }}
+    {% endif %}
diff --git a/salt/postgres/auth.sls b/salt/postgres/auth.sls
index 44c89c581..e0397beba 100644
--- a/salt/postgres/auth.sls
+++ b/salt/postgres/auth.sls
@@ -50,13 +50,14 @@ postgres_auth_pillar:
               {% endfor %}
     - show_changes: False
 
-  {# Fan each minion's telegraf cred out to its own pillar file. The minions/
-     <id>.sls file is only served to that specific minion via pillar/top.sls
-     (`- minions.{{ grains.id }}`), so sensors, heavynodes, etc. see their own
-     credential without the admin password or anyone else's. Run per up-minion
-     so we have the original minion id (not just the safe-normalized version). #}
-  {% for mid in up_minions %}
-    {%- set safe = mid | replace('.','_') | replace('-','_') | lower %}
+  {# Fan a specific minion's telegraf cred out to its own pillar file. Only
+     runs when postgres_fanout_minion pillar is provided — otherwise this state
+     is a no-op. That keeps manager highstates from doing N so-yaml.py forks
+     when nothing changed. The reactor passes postgres_fanout_minion through
+     the orch on salt-key accept; soup handles bulk backfill separately. #}
+  {% set fanout_mid = salt['pillar.get']('postgres_fanout_minion') %}
+  {% if fanout_mid %}
+    {%- set safe = fanout_mid | replace('.','_') | replace('-','_') | lower %}
     {%- set key = 'telegraf_' ~ safe %}
     {%- set entry = telegraf_users.get(key) %}
     {%- if entry %}
@@ -65,7 +66,7 @@ postgres_telegraf_minion_pillar_{{ safe }}:
   cmd.run:
     - name: |
         set -e
-        PILLAR_FILE=/opt/so/saltstack/local/pillar/minions/{{ mid }}.sls
+        PILLAR_FILE=/opt/so/saltstack/local/pillar/minions/{{ fanout_mid }}.sls
         if [ ! -f "$PILLAR_FILE" ]; then
           echo '{}' > "$PILLAR_FILE"
           chown socore:socore "$PILLAR_FILE" 2>/dev/null || true
@@ -73,18 +74,13 @@ postgres_telegraf_minion_pillar_{{ safe }}:
         fi
         /usr/sbin/so-yaml.py replace "$PILLAR_FILE" postgres.telegraf.user '{{ entry.user }}'
         /usr/sbin/so-yaml.py replace "$PILLAR_FILE" postgres.telegraf.pass '{{ entry.pass }}'
-    {#- Skip if this minion's pillar file already carries a matching user.
-        Passwords are generated once per minion (see the `if key not in telegraf_users`
-        guard above) and never rotate, so once a cred is fanned out the file
-        doesn't need to be rewritten on subsequent auth runs. If we ever add
-        rotation, we'd need to delete postgres.telegraf to force a re-fan. #}
     - unless: |
-        [ "$(/usr/sbin/so-yaml.py get -r /opt/so/saltstack/local/pillar/minions/{{ mid }}.sls postgres.telegraf.user 2>/dev/null)" = '{{ entry.user }}' ]
+        [ "$(/usr/sbin/so-yaml.py get -r /opt/so/saltstack/local/pillar/minions/{{ fanout_mid }}.sls postgres.telegraf.user 2>/dev/null)" = '{{ entry.user }}' ]
     - require:
       - file: postgres_auth_pillar
 
     {%- endif %}
-  {% endfor %}
+  {% endif %}
 {% else %}
 
 {{sls}}_state_not_allowed:
diff --git a/salt/reactor/telegraf_user_sync.sls b/salt/reactor/telegraf_user_sync.sls
index 4830dbc53..075dbf62e 100644
--- a/salt/reactor/telegraf_user_sync.sls
+++ b/salt/reactor/telegraf_user_sync.sls
@@ -10,6 +10,8 @@
   runner.state.orchestrate:
     - args:
       - mods: orch.telegraf_postgres_sync
+      - pillar:
+          postgres_fanout_minion: {{ data['id'] }}
 
 {% do salt.log.info('telegraf_user_sync reactor: syncing telegraf PG user for minion %s' % data['id']) %}