From ad78e84ccd7c114fa1719d15de267e31295570ac Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 20 Jul 2026 12:41:14 -0400 Subject: [PATCH] Prevent PostgreSQL from leaking plaintext passwords to postgres.log PostgreSQL's log_min_error_statement defaults to 'error', so whenever a CREATE/ALTER ROLE ... PASSWORD statement errored, the full statement text -- including the plaintext password -- was written to /opt/so/log/postgres/postgres.log. The role-provisioning paths (init-db.sh for so_postgres, so-telegraf-postgres for per-minion telegraf roles) both dispatch such DDL, the latter on every state.apply. - init-db.sh / so-telegraf-postgres: SET log_min_error_statement = panic before the password-bearing DDL so an error no longer emits the STATEMENT line. The ERROR message itself (no password) still logs, preserving debuggability. - logrotate: add a postgres stanza (daily, keep 14, copytruncate, compress) so postgres.log is rotated like every other service and leaked content can't persist indefinitely. copytruncate is required because the container holds the log open via redirected stderr. - soup: scrub any already-logged PASSWORD lines from postgres.log during post_to_3.2.0, rewriting in place to preserve the inode postgres is writing to. --- salt/logrotate/defaults.yaml | 10 +++++++ salt/logrotate/soc_logrotate.yaml | 7 +++++ salt/manager/tools/sbin/soup | 27 +++++++++++++++++++ salt/postgres/files/init-db.sh | 4 +++ salt/postgres/tools/sbin/so-telegraf-postgres | 4 +++ 5 files changed, 52 insertions(+) diff --git a/salt/logrotate/defaults.yaml b/salt/logrotate/defaults.yaml index 2261bb4f7..e37193d0d 100644 --- a/salt/logrotate/defaults.yaml +++ b/salt/logrotate/defaults.yaml @@ -150,6 +150,16 @@ logrotate: - extension .log - dateext - dateyesterday + /opt/so/log/postgres/*_x_log: + - daily + - rotate 14 + - missingok + - copytruncate + - compress + - create + - extension .log + - dateext + - dateyesterday /opt/so/log/telegraf/*_x_log: - daily - rotate 14 diff --git a/salt/logrotate/soc_logrotate.yaml b/salt/logrotate/soc_logrotate.yaml index f407ab48d..f329ed623 100644 --- a/salt/logrotate/soc_logrotate.yaml +++ b/salt/logrotate/soc_logrotate.yaml @@ -91,6 +91,13 @@ logrotate: multiline: True global: True forcedType: "[]string" + "/opt/so/log/postgres/*_x_log": + description: List of logrotate options for this file. + title: /opt/so/log/postgres/*.log + advanced: True + multiline: True + global: True + forcedType: "[]string" "/opt/so/log/telegraf/*_x_log": description: List of logrotate options for this file. title: /opt/so/log/telegraf/*.log diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index f5b40310e..8def509e1 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -905,6 +905,30 @@ bootstrap_so_soc_database() { docker restart so-soc } +scrub_postgres_log_passwords() { + # Before 3.2 PostgreSQL could log the full CREATE/ALTER ROLE ... PASSWORD + # statement -- with the plaintext password -- to postgres.log whenever such a + # DDL errored, because log_min_error_statement defaulted to 'error'. The leak + # is fixed going forward (the provisioning scripts now SET log_min_error_statement + # = panic), but any already-logged secret persists in the on-disk log. Purge it + # here. Only the manager runs so-postgres, so this is a local scrub. + local log=/opt/so/log/postgres/postgres.log + [[ -f "$log" ]] || return 0 + if ! grep -qai "PASSWORD '" "$log" 2>/dev/null; then + echo "No leaked passwords found in $log." + return 0 + fi + echo "Removing leaked password statements from $log." + local tmp + tmp=$(mktemp) + # Drop every line carrying a PASSWORD literal, then rewrite in place with + # `cat >` so the file keeps its inode -- postgres holds it open via redirected + # stderr and must keep logging to the same file. + grep -avi "PASSWORD '" "$log" > "$tmp" 2>/dev/null || true + cat "$tmp" > "$log" + rm -f "$tmp" +} + # Existing grids should keep ILM unless an admin explicitly opts in to DLM. pin_elasticsearch_data_retention_method() { local elasticsearch_file=/opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls @@ -991,6 +1015,9 @@ post_to_3.2.0() { bootstrap_so_soc_database + # Purge any plaintext passwords a pre-3.2 postgres logged on DDL errors. + scrub_postgres_log_passwords + # Generate 9.3.7 elastic agent installers echo "Regenerating Elastic Agent Installers" /sbin/so-elastic-agent-gen-installers diff --git a/salt/postgres/files/init-db.sh b/salt/postgres/files/init-db.sh index d12bc4c9b..d1f1e375f 100644 --- a/salt/postgres/files/init-db.sh +++ b/salt/postgres/files/init-db.sh @@ -8,6 +8,10 @@ if [ -z "${SO_POSTGRES_PASS:-}" ] && [ -n "${SO_POSTGRES_PASS_FILE:-}" ] && [ -r SO_POSTGRES_PASS="$(< "$SO_POSTGRES_PASS_FILE")" fi psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + -- Shield the plaintext password below from postgres.log if this DDL errors: + -- log_min_error_statement defaults to 'error', which would append a STATEMENT line + -- containing the full CREATE/ALTER ROLE ... PASSWORD text. panic suppresses that. + SET log_min_error_statement = panic; DO \$\$ BEGIN IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${SO_POSTGRES_USER}') THEN diff --git a/salt/postgres/tools/sbin/so-telegraf-postgres b/salt/postgres/tools/sbin/so-telegraf-postgres index ef7c3f9e6..e6cc69fa6 100644 --- a/salt/postgres/tools/sbin/so-telegraf-postgres +++ b/salt/postgres/tools/sbin/so-telegraf-postgres @@ -71,6 +71,10 @@ EOSQL -v role_user="$ROLE_USER" \ -v role_pass="$ROLE_PASS" \ -U postgres -d so_telegraf <<'EOSQL' +-- Shield the plaintext password below from postgres.log if this DDL errors: +-- log_min_error_statement defaults to 'error', which would append a STATEMENT line +-- containing the full CREATE/ALTER ROLE ... PASSWORD text. panic suppresses that. +SET log_min_error_statement = panic; SELECT format( CASE WHEN EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'role_user') THEN 'ALTER ROLE %I WITH LOGIN PASSWORD %L'