Merge pull request #16084 from Security-Onion-Solutions/postgres-log-password-scrub

PostgreSQL: prevent password log leak + self-heal SOC database bootstrap

salt/logrotate/defaults.yaml

@@ -150,6 +150,16 @@ logrotate:
       - extension .log
       - dateext
       - dateyesterday
+    /opt/so/log/postgres/*_x_log:
+      - daily
+      - rotate 14
+      - missingok
+      - copytruncate
+      - compress
+      - create
+      - extension .log
+      - dateext
+      - dateyesterday
     /opt/so/log/telegraf/*_x_log:
       - daily
       - rotate 14

salt/logrotate/soc_logrotate.yaml

@@ -91,6 +91,13 @@ logrotate:
       multiline: True
       global: True
       forcedType: "[]string"
+    "/opt/so/log/postgres/*_x_log":
+      description: List of logrotate options for this file.
+      title: /opt/so/log/postgres/*.log
+      advanced: True
+      multiline: True
+      global: True
+      forcedType: "[]string"
     "/opt/so/log/telegraf/*_x_log":
       description: List of logrotate options for this file.
       title: /opt/so/log/telegraf/*.log

salt/manager/tools/sbin/soup

@@ -880,29 +880,21 @@ recollate_postgres() {
   echo ""
 }
 
-bootstrap_so_soc_database() {
-  # init-db.sh is mounted into so-postgres at /docker-entrypoint-initdb.d/init-db.sh
-  # and runs automatically only on a fresh data directory. Hosts upgrading from
-  # 3.1.0 already have /nsm/postgres populated, so the so_soc bootstrap block
-  # added in 3.2 never fires. Re-run the script explicitly; it's idempotent.
-  echo "Bootstrapping database via init-db.sh."
-  # The postgres image has no USER directive, so `docker exec` defaults to
-  # root, and the container env intentionally omits POSTGRES_USER (the upstream
-  # entrypoint defaults it transiently during first-init only). Recreate both
-  # so psql inside init-db.sh resolves the connect user correctly.
-  local exec_cmd="docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh"
-  if ! /usr/sbin/so-postgres-wait; then
-    FINAL_MESSAGE_QUEUE+=("WARNING: so-postgres was not ready during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd")
+scrub_postgres_log_passwords() {
+  # Purge plaintext passwords a pre-3.2 postgres could log on DDL errors.
+  local log=/opt/so/log/postgres/postgres.log
+  [[ -f "$log" ]] || return 0
+  if ! grep -qai "PASSWORD '" "$log" 2>/dev/null; then
+    echo "No leaked passwords found in $log."
     return 0
   fi
-  if ! $exec_cmd; then
-    FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the database may not have been bootstrapped. Re-run manually: $exec_cmd")
-    return 0
-  fi
-  echo "Database bootstrap complete."
-
-  echo "Restarting so-soc container to pick up database changes"
-  docker restart so-soc
+  echo "Removing leaked password statements from $log."
+  local tmp
+  tmp=$(mktemp)
+  # Rewrite in place (cat >) to keep the inode postgres is writing to.
+  grep -avi "PASSWORD '" "$log" > "$tmp" 2>/dev/null || true
+  cat "$tmp" > "$log"
+  rm -f "$tmp"
 }
 
 # Existing grids should keep ILM unless an admin explicitly opts in to DLM.
@@ -989,7 +981,8 @@ post_to_3.2.0() {
   # Recollate due to image OS rebase
   recollate_postgres
 
-  bootstrap_so_soc_database
+  # SOC database bootstrap is handled by the postgres.enabled highstate.
+  scrub_postgres_log_passwords
 
   # Generate 9.3.7 elastic agent installers
   echo "Regenerating Elastic Agent Installers"

salt/postgres/enabled.sls

@@ -85,6 +85,23 @@ so-postgres:
       - x509: postgres_crt
       - x509: postgres_key
 
+postgres_wait_ready:
+  cmd.run:
+    - name: /usr/sbin/so-postgres-wait
+    - require:
+      - docker_container: so-postgres
+      - file: postgres_sbin
+
+# Reconcile the SOC database (role, grants, so_telegraf) every highstate so a
+# partially-initialized cluster self-heals. POSTGRES_USER is injected because
+# the container env omits it.
+postgres_bootstrap_soc_db:
+  cmd.run:
+    - name: docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh
+    - require:
+      - cmd: postgres_wait_ready
+      - file: postgresinitdb
+
 delete_so-postgres_so-status.disabled:
   file.uncomment:
     - name: /opt/so/conf/so-status/so-status.conf

salt/postgres/files/init-db.sh

@@ -8,13 +8,17 @@ if [ -z "${SO_POSTGRES_PASS:-}" ] && [ -n "${SO_POSTGRES_PASS_FILE:-}" ] && [ -r
     SO_POSTGRES_PASS="$(< "$SO_POSTGRES_PASS_FILE")"
 fi
 psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    -- Keep the password out of postgres.log if this DDL errors.
+    SET log_min_error_statement = panic;
+    -- Idempotent, race-safe upsert: CREATE, falling back to ALTER if the role
+    -- already exists or is created concurrently.
     DO \$\$
     BEGIN
-        IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${SO_POSTGRES_USER}') THEN
+        BEGIN
             EXECUTE format('CREATE ROLE %I WITH LOGIN PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}');
-        ELSE
-            EXECUTE format('ALTER ROLE %I WITH PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}');
-        END IF;
+        EXCEPTION WHEN duplicate_object OR unique_violation THEN
+            EXECUTE format('ALTER ROLE %I WITH LOGIN PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}');
+        END;
     END
     \$\$;
     GRANT ALL ON SCHEMA public TO "$SO_POSTGRES_USER";

salt/postgres/telegraf_users.sls

@@ -8,23 +8,14 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'telegraf/map.jinja' import TELEGRAFMERGED %}
 
-{# postgres_wait_ready below requires `docker_container: so-postgres`, which is
-   declared in postgres.enabled. Include it here so state.apply postgres.telegraf_users
-   on its own (e.g. from orch.deploy_newnode) still has that ID in scope. Salt
-   de-duplicates the circular include. #}
+{# postgres.enabled declares the so-postgres container and postgres_wait_ready
+   that the requires below reference. Salt de-duplicates the circular include. #}
 include:
   - postgres.enabled
 
 {% set TG_OUT = TELEGRAFMERGED.output | upper %}
 {% if TG_OUT in ['POSTGRES', 'BOTH'] %}
 
-postgres_wait_ready:
-  cmd.run:
-    - name: /usr/sbin/so-postgres-wait
-    - require:
-      - docker_container: so-postgres
-      - file: postgres_sbin
-
 # Ensure the shared Telegraf database exists. init-db.sh only runs on a
 # fresh data dir, so hosts upgraded onto an existing /nsm/postgres volume
 # would otherwise never get so_telegraf.

salt/postgres/tools/sbin/so-telegraf-postgres

@@ -71,6 +71,8 @@ EOSQL
       -v role_user="$ROLE_USER" \
       -v role_pass="$ROLE_PASS" \
       -U postgres -d so_telegraf <<'EOSQL'
+-- Keep the password out of postgres.log if this DDL errors.
+SET log_min_error_statement = panic;
 SELECT format(
   CASE WHEN EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'role_user')
        THEN 'ALTER ROLE %I WITH LOGIN PASSWORD %L'