Merge pull request #15808 from Security-Onion-Solutions/feature/postgres

Feature/postgres

salt/manager/tools/sbin/so-minion

@@ -273,7 +273,7 @@ function deleteMinionFiles () {
 		log "ERROR" "Failed to delete $PILLARFILE"
 		return 1
 	fi
-	
+
 	rm -f $ADVPILLARFILE
 	if [ $? -ne 0 ]; then
 		log "ERROR" "Failed to delete $ADVPILLARFILE"
@@ -281,6 +281,43 @@ function deleteMinionFiles () {
 	fi
 }
 
+# Remove this minion's postgres Telegraf credential from both the aggregate
+# pillar and the postgres database. Paired with add_telegraf_to_minion:
+# add/delete cycle both here and in the DB. Always returns 0 so a dead or
+# unreachable so-postgres doesn't block minion deletion — in that case we
+# log a warning and leave the role behind for manual cleanup.
+function remove_postgres_telegraf_from_minion() {
+	local MINION_SAFE
+	MINION_SAFE=$(echo "$MINION_ID" | tr '.-' '__' | tr '[:upper:]' '[:lower:]')
+	local PG_USER="so_telegraf_${MINION_SAFE}"
+	local AGGREGATE=/opt/so/saltstack/local/pillar/postgres/auth.sls
+
+	log "INFO" "Removing postgres telegraf cred for $MINION_ID"
+
+	if [[ -f "$AGGREGATE" ]]; then
+		so-yaml.py remove "$AGGREGATE" "postgres.auth.users.telegraf_${MINION_SAFE}" >/dev/null 2>&1 || true
+	fi
+
+	if docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^so-postgres$'; then
+		if ! docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf >/dev/null 2>&1 <<EOSQL
+DO \$\$
+BEGIN
+    IF EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '$PG_USER') THEN
+        EXECUTE format('REASSIGN OWNED BY %I TO so_telegraf', '$PG_USER');
+        EXECUTE format('DROP OWNED BY %I', '$PG_USER');
+        EXECUTE format('DROP ROLE %I', '$PG_USER');
+    END IF;
+END
+\$\$;
+EOSQL
+		then
+			log "WARN" "Failed to drop postgres role $PG_USER; aggregate pillar entry was removed — drop manually if the role persists"
+		fi
+	else
+		log "WARN" "so-postgres container is not running; skipping DB role cleanup for $PG_USER"
+	fi
+}
+
 # Create the minion file
 function ensure_socore_ownership() {
 	log "INFO" "Setting socore ownership on minion files"
@@ -542,6 +579,37 @@ function add_telegraf_to_minion() {
         log "ERROR" "Failed to add telegraf configuration to $PILLARFILE"
         return 1
     fi
+
+    # Provision the per-minion postgres Telegraf credential so telegraf.conf
+    # renders correctly on the minion's first highstate and postgres.telegraf_users
+    # picks up the matching aggregate entry on the next manager apply.
+    #
+    # Writes:
+    #   - postgres.telegraf.{user,pass} into the minion's own pillar file
+    #     (distributed to only this minion via pillar/top.sls).
+    #   - postgres.auth.users.telegraf_<safe>.{user,pass} into the aggregate
+    #     pillar so postgres.telegraf_users CREATE ROLE finds it.
+    #
+    # An existing password is reused if the aggregate already has one (re-add),
+    # so rerunning so-minion for the same minion keeps the cred stable.
+    local MINION_SAFE
+    MINION_SAFE=$(echo "$MINION_ID" | tr '.-' '__' | tr '[:upper:]' '[:lower:]')
+    local PG_USER="so_telegraf_${MINION_SAFE}"
+    local AGGREGATE=/opt/so/saltstack/local/pillar/postgres/auth.sls
+    local PG_PASS=""
+    if [[ -f "$AGGREGATE" ]]; then
+        PG_PASS=$(so-yaml.py get -r "$AGGREGATE" "postgres.auth.users.telegraf_${MINION_SAFE}.pass" 2>/dev/null || true)
+    fi
+    if [[ -z "$PG_PASS" ]]; then
+        PG_PASS=$(tr -dc 'A-Za-z0-9~!@#^&*()_=+[]|;:,.<>?-' < /dev/urandom | head -c 72)
+    fi
+
+    so-yaml.py replace "$PILLARFILE" postgres.telegraf.user "$PG_USER" >/dev/null
+    so-yaml.py replace "$PILLARFILE" postgres.telegraf.pass "$PG_PASS" >/dev/null
+    if [[ -f "$AGGREGATE" ]]; then
+        so-yaml.py replace "$AGGREGATE" "postgres.auth.users.telegraf_${MINION_SAFE}.user" "$PG_USER" >/dev/null
+        so-yaml.py replace "$AGGREGATE" "postgres.auth.users.telegraf_${MINION_SAFE}.pass" "$PG_PASS" >/dev/null
+    fi
 }
 
 function add_influxdb_to_minion() {
@@ -1069,6 +1137,7 @@ case "$OPERATION" in
 
 	"delete")
 		log "INFO" "Removing minion $MINION_ID"
+		remove_postgres_telegraf_from_minion
 		deleteMinionFiles || {
 			log "ERROR" "Failed to delete minion files for $MINION_ID"
 			exit 1

salt/manager/tools/sbin/so-yaml.py

@@ -285,7 +285,8 @@ def add(args):
 def removeKey(content, key):
     pieces = key.split(".", 1)
     if len(pieces) > 1:
-        removeKey(content[pieces[0]], pieces[1])
+        if pieces[0] in content:
+            removeKey(content[pieces[0]], pieces[1])
     else:
         content.pop(key, None)
 

salt/manager/tools/sbin/soup

@@ -491,10 +491,11 @@ post_to_3.1.0() {
   /usr/sbin/so-kibana-space-defaults
 
   # One-time backfill for minions that existed before the postgres Telegraf
-  # feature shipped. Generate the aggregate pillar on the manager and create
-  # the per-minion DB roles, then fan each minion's cred into its own pillar
-  # file. Going forward the reactor handles each new salt-key accept with a
-  # targeted fan-out, so a manager highstate no longer needs to iterate.
+  # feature shipped. postgres.auth's up_minions fallback loop generates any
+  # missing aggregate pillar entries; postgres.telegraf_users CREATEs the
+  # matching DB roles; then the bash loop below copies each minion's cred
+  # into its own pillar file. Going forward, so-minion owns add/delete for
+  # every new minion, so this backfill is only needed on the upgrade boundary.
   echo "Provisioning Telegraf Postgres users for existing minions."
   salt-call --local state.apply postgres.auth,postgres.telegraf_users queue=True || true
 

salt/orch/telegraf_postgres_sync.sls

@@ -1,28 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-# Fired by salt/reactor/telegraf_user_sync.sls when salt-key accepts a new
-# minion. Only provisions the per-minion pillar entry and DB role on the
-# manager; the minion itself will pick up its telegraf config on its first
-# highstate during onboarding, so there's no need to push the telegraf state
-# from here.
-#
-# Target the manager via role grains — same pattern as orch/delete_hypervisor.sls.
-# The reactor doesn't know the manager's minion id, and grains.master on the
-# runner is a hostname, not a targetable id.
-{% set FANOUT_MINION = salt['pillar.get']('postgres_fanout_minion', '') %}
-
-manager_sync_telegraf_pg_users:
-  salt.state:
-    - tgt: 'G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone or G@role:so-eval'
-    - tgt_type: compound
-    - sls:
-      - postgres.auth
-      - postgres.telegraf_users
-    - queue: True
-    {% if FANOUT_MINION %}
-    - pillar:
-        postgres_fanout_minion: {{ FANOUT_MINION }}
-    {% endif %}

salt/postgres/auth.sls

@@ -49,38 +49,6 @@ postgres_auth_pillar:
                 pass: "{{ entry.pass }}"
               {% endfor %}
     - show_changes: False
-
-  {# Fan a specific minion's telegraf cred out to its own pillar file. Only
-     runs when postgres_fanout_minion pillar is provided — otherwise this state
-     is a no-op. That keeps manager highstates from doing N so-yaml.py forks
-     when nothing changed. The reactor passes postgres_fanout_minion through
-     the orch on salt-key accept; soup handles bulk backfill separately. #}
-  {% set fanout_mid = salt['pillar.get']('postgres_fanout_minion') %}
-  {% if fanout_mid %}
-    {%- set safe = fanout_mid | replace('.','_') | replace('-','_') | lower %}
-    {%- set key = 'telegraf_' ~ safe %}
-    {%- set entry = telegraf_users.get(key) %}
-    {%- if entry %}
-
-postgres_telegraf_minion_pillar_{{ safe }}:
-  cmd.run:
-    - name: |
-        set -e
-        PILLAR_FILE=/opt/so/saltstack/local/pillar/minions/{{ fanout_mid }}.sls
-        if [ ! -f "$PILLAR_FILE" ]; then
-          echo '{}' > "$PILLAR_FILE"
-          chown socore:socore "$PILLAR_FILE" 2>/dev/null || true
-          chmod 640 "$PILLAR_FILE"
-        fi
-        /usr/sbin/so-yaml.py replace "$PILLAR_FILE" postgres.telegraf.user '{{ entry.user }}'
-        /usr/sbin/so-yaml.py replace "$PILLAR_FILE" postgres.telegraf.pass '{{ entry.pass }}'
-    - unless: |
-        [ "$(/usr/sbin/so-yaml.py get -r /opt/so/saltstack/local/pillar/minions/{{ fanout_mid }}.sls postgres.telegraf.user 2>/dev/null)" = '{{ entry.user }}' ]
-    - require:
-      - file: postgres_auth_pillar
-
-    {%- endif %}
-  {% endif %}
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/reactor/telegraf_user_sync.sls

@@ -1,18 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{# Fires on salt/key. Only act on successful key acceptance — not reauth. #}
-{% if data.get('act') == 'accept' and data.get('result') == True and data.get('id') %}
-
-{{ data['id'] }}_telegraf_pg_sync:
-  runner.state.orchestrate:
-    - args:
-      - mods: orch.telegraf_postgres_sync
-      - pillar:
-          postgres_fanout_minion: {{ data['id'] }}
-
-{% do salt.log.info('telegraf_user_sync reactor: syncing telegraf PG user for minion %s' % data['id']) %}
-
-{% endif %}

salt/salt/master.sls

@@ -62,19 +62,6 @@ engines_config:
     - name: /etc/salt/master.d/engines.conf
     - source: salt://salt/files/engines.conf
 
-reactor_config_telegraf:
-  file.managed:
-    - name: /etc/salt/master.d/reactor_telegraf.conf
-    - contents: |
-        reactor:
-          - 'salt/key':
-            - /opt/so/saltstack/default/salt/reactor/telegraf_user_sync.sls
-    - user: root
-    - group: root
-    - mode: 644
-    - watch_in:
-      - service: salt_master_service
-
 # update the bootstrap script when used for salt-cloud
 salt_bootstrap_cloud:
   file.managed:

salt/telegraf/etc/telegraf.conf

@@ -96,7 +96,7 @@
   # insecure_skip_verify = false
 {%- endif %}
 
-{%- if TG_OUT in ['POSTGRES', 'BOTH'] %}
+{%- if TG_OUT in ['POSTGRES', 'BOTH'] and PG_USER and PG_PASS %}
 # Configuration for sending metrics to PostgreSQL.
 # options='-c role=so_telegraf' makes every connection SET ROLE to the shared
 # group role so tables created on first write are owned by so_telegraf, and