Prevent PostgreSQL from leaking plaintext passwords to postgres.log

PostgreSQL's log_min_error_statement defaults to 'error', so whenever a
CREATE/ALTER ROLE ... PASSWORD statement errored, the full statement text --
including the plaintext password -- was written to /opt/so/log/postgres/postgres.log.
The role-provisioning paths (init-db.sh for so_postgres, so-telegraf-postgres
for per-minion telegraf roles) both dispatch such DDL, the latter on every
state.apply.

- init-db.sh / so-telegraf-postgres: SET log_min_error_statement = panic before
  the password-bearing DDL so an error no longer emits the STATEMENT line. The
  ERROR message itself (no password) still logs, preserving debuggability.
- logrotate: add a postgres stanza (daily, keep 14, copytruncate, compress) so
  postgres.log is rotated like every other service and leaked content can't
  persist indefinitely. copytruncate is required because the container holds the
  log open via redirected stderr.
- soup: scrub any already-logged PASSWORD lines from postgres.log during
  post_to_3.2.0, rewriting in place to preserve the inode postgres is writing to.

salt/logrotate/defaults.yaml

@@ -150,6 +150,16 @@ logrotate:
       - extension .log
       - dateext
       - dateyesterday
+    /opt/so/log/postgres/*_x_log:
+      - daily
+      - rotate 14
+      - missingok
+      - copytruncate
+      - compress
+      - create
+      - extension .log
+      - dateext
+      - dateyesterday
     /opt/so/log/telegraf/*_x_log:
       - daily
       - rotate 14

salt/logrotate/soc_logrotate.yaml

@@ -91,6 +91,13 @@ logrotate:
       multiline: True
       global: True
       forcedType: "[]string"
+    "/opt/so/log/postgres/*_x_log":
+      description: List of logrotate options for this file.
+      title: /opt/so/log/postgres/*.log
+      advanced: True
+      multiline: True
+      global: True
+      forcedType: "[]string"
     "/opt/so/log/telegraf/*_x_log":
       description: List of logrotate options for this file.
       title: /opt/so/log/telegraf/*.log

salt/manager/tools/sbin/soup

@@ -905,6 +905,30 @@ bootstrap_so_soc_database() {
   docker restart so-soc
 }
 
+scrub_postgres_log_passwords() {
+  # Before 3.2 PostgreSQL could log the full CREATE/ALTER ROLE ... PASSWORD
+  # statement -- with the plaintext password -- to postgres.log whenever such a
+  # DDL errored, because log_min_error_statement defaulted to 'error'. The leak
+  # is fixed going forward (the provisioning scripts now SET log_min_error_statement
+  # = panic), but any already-logged secret persists in the on-disk log. Purge it
+  # here. Only the manager runs so-postgres, so this is a local scrub.
+  local log=/opt/so/log/postgres/postgres.log
+  [[ -f "$log" ]] || return 0
+  if ! grep -qai "PASSWORD '" "$log" 2>/dev/null; then
+    echo "No leaked passwords found in $log."
+    return 0
+  fi
+  echo "Removing leaked password statements from $log."
+  local tmp
+  tmp=$(mktemp)
+  # Drop every line carrying a PASSWORD literal, then rewrite in place with
+  # `cat >` so the file keeps its inode -- postgres holds it open via redirected
+  # stderr and must keep logging to the same file.
+  grep -avi "PASSWORD '" "$log" > "$tmp" 2>/dev/null || true
+  cat "$tmp" > "$log"
+  rm -f "$tmp"
+}
+
 # Existing grids should keep ILM unless an admin explicitly opts in to DLM.
 pin_elasticsearch_data_retention_method() {
   local elasticsearch_file=/opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls
@@ -991,6 +1015,9 @@ post_to_3.2.0() {
 
   bootstrap_so_soc_database
 
+  # Purge any plaintext passwords a pre-3.2 postgres logged on DDL errors.
+  scrub_postgres_log_passwords
+
   # Generate 9.3.7 elastic agent installers
   echo "Regenerating Elastic Agent Installers"
   /sbin/so-elastic-agent-gen-installers

salt/postgres/files/init-db.sh

@@ -8,6 +8,10 @@ if [ -z "${SO_POSTGRES_PASS:-}" ] && [ -n "${SO_POSTGRES_PASS_FILE:-}" ] && [ -r
     SO_POSTGRES_PASS="$(< "$SO_POSTGRES_PASS_FILE")"
 fi
 psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    -- Shield the plaintext password below from postgres.log if this DDL errors:
+    -- log_min_error_statement defaults to 'error', which would append a STATEMENT line
+    -- containing the full CREATE/ALTER ROLE ... PASSWORD text. panic suppresses that.
+    SET log_min_error_statement = panic;
     DO \$\$
     BEGIN
         IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${SO_POSTGRES_USER}') THEN

salt/postgres/tools/sbin/so-telegraf-postgres

@@ -71,6 +71,10 @@ EOSQL
       -v role_user="$ROLE_USER" \
       -v role_pass="$ROLE_PASS" \
       -U postgres -d so_telegraf <<'EOSQL'
+-- Shield the plaintext password below from postgres.log if this DDL errors:
+-- log_min_error_statement defaults to 'error', which would append a STATEMENT line
+-- containing the full CREATE/ALTER ROLE ... PASSWORD text. panic suppresses that.
+SET log_min_error_statement = panic;
 SELECT format(
   CASE WHEN EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'role_user')
        THEN 'ALTER ROLE %I WITH LOGIN PASSWORD %L'