From d5dc28e52689d08983530ef8da04ded2f8c7166f Mon Sep 17 00:00:00 2001
From: Mike Reeves <reevesmk@gmail.com>
Date: Tue, 21 Apr 2026 14:40:19 -0400
Subject: [PATCH] Fan postgres telegraf cred for manager on every auth run
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The empty-pillar case produced a telegraf.conf with `user= password=`
which libpq misparses ("password=" gets consumed as the user value),
yielding `password authentication failed for user "password="` on
every manager without a prior fan-out (fresh install, not the salt-key
path the reactor handles).

Two fixes:

- salt/postgres/auth.sls: always fan for grains.id in addition to any
  postgres_fanout_minion from the reactor, so the manager's own pillar
  is populated on every postgres.auth run. The existing `unless` guard
  keeps re-runs idempotent.
- salt/telegraf/etc/telegraf.conf: gate the [[outputs.postgresql]]
  block on PG_USER and PG_PASS being non-empty. If a minion hasn't
  received its pillar yet the output block simply isn't rendered — the
  next highstate picks up the creds once the fan-out completes, and in
  the meantime telegraf keeps running the other outputs instead of
  erroring with a malformed connection string.
---
 salt/postgres/auth.sls          | 33 +++++++++++++++++++++++----------
 salt/telegraf/etc/telegraf.conf |  2 +-
 2 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/salt/postgres/auth.sls b/salt/postgres/auth.sls
index e0397beba..ec6f3ec7e 100644
--- a/salt/postgres/auth.sls
+++ b/salt/postgres/auth.sls
@@ -50,14 +50,27 @@ postgres_auth_pillar:
               {% endfor %}
     - show_changes: False
 
-  {# Fan a specific minion's telegraf cred out to its own pillar file. Only
-     runs when postgres_fanout_minion pillar is provided — otherwise this state
-     is a no-op. That keeps manager highstates from doing N so-yaml.py forks
-     when nothing changed. The reactor passes postgres_fanout_minion through
-     the orch on salt-key accept; soup handles bulk backfill separately. #}
+  {# Fan a specific minion's telegraf cred out to its own pillar file.
+     Two triggers populate the target list:
+       - grains.id (always) so the manager's own pillar is populated on every
+         postgres.auth run — otherwise the manager's telegraf has no cred on
+         a fresh install and can't write to its own postgres.
+       - pillar postgres_fanout_minion (when the reactor fires on a new
+         minion's salt-key accept).
+     The `unless` guard keeps re-runs idempotent, so this is one so-yaml.py
+     check per target, not per minion in the grid. Bulk backfill for
+     already-accepted minions lives in soup. #}
+  {% set fanout_targets = [] %}
+  {% if grains.id %}
+  {%-   do fanout_targets.append(grains.id) %}
+  {% endif %}
   {% set fanout_mid = salt['pillar.get']('postgres_fanout_minion') %}
-  {% if fanout_mid %}
-    {%- set safe = fanout_mid | replace('.','_') | replace('-','_') | lower %}
+  {% if fanout_mid and fanout_mid not in fanout_targets %}
+  {%-   do fanout_targets.append(fanout_mid) %}
+  {% endif %}
+
+  {% for mid in fanout_targets %}
+    {%- set safe = mid | replace('.','_') | replace('-','_') | lower %}
     {%- set key = 'telegraf_' ~ safe %}
     {%- set entry = telegraf_users.get(key) %}
     {%- if entry %}
@@ -66,7 +79,7 @@ postgres_telegraf_minion_pillar_{{ safe }}:
   cmd.run:
     - name: |
         set -e
-        PILLAR_FILE=/opt/so/saltstack/local/pillar/minions/{{ fanout_mid }}.sls
+        PILLAR_FILE=/opt/so/saltstack/local/pillar/minions/{{ mid }}.sls
         if [ ! -f "$PILLAR_FILE" ]; then
           echo '{}' > "$PILLAR_FILE"
           chown socore:socore "$PILLAR_FILE" 2>/dev/null || true
@@ -75,12 +88,12 @@ postgres_telegraf_minion_pillar_{{ safe }}:
         /usr/sbin/so-yaml.py replace "$PILLAR_FILE" postgres.telegraf.user '{{ entry.user }}'
         /usr/sbin/so-yaml.py replace "$PILLAR_FILE" postgres.telegraf.pass '{{ entry.pass }}'
     - unless: |
-        [ "$(/usr/sbin/so-yaml.py get -r /opt/so/saltstack/local/pillar/minions/{{ fanout_mid }}.sls postgres.telegraf.user 2>/dev/null)" = '{{ entry.user }}' ]
+        [ "$(/usr/sbin/so-yaml.py get -r /opt/so/saltstack/local/pillar/minions/{{ mid }}.sls postgres.telegraf.user 2>/dev/null)" = '{{ entry.user }}' ]
     - require:
       - file: postgres_auth_pillar
 
     {%- endif %}
-  {% endif %}
+  {% endfor %}
 {% else %}
 
 {{sls}}_state_not_allowed:
diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf
index 001a61d93..53b96e4ab 100644
--- a/salt/telegraf/etc/telegraf.conf
+++ b/salt/telegraf/etc/telegraf.conf
@@ -96,7 +96,7 @@
   # insecure_skip_verify = false
 {%- endif %}
 
-{%- if TG_OUT in ['POSTGRES', 'BOTH'] %}
+{%- if TG_OUT in ['POSTGRES', 'BOTH'] and PG_USER and PG_PASS %}
 # Configuration for sending metrics to PostgreSQL.
 # options='-c role=so_telegraf' makes every connection SET ROLE to the shared
 # group role so tables created on first write are owned by so_telegraf, and