diff --git a/salt/logrotate/defaults.yaml b/salt/logrotate/defaults.yaml index 2261bb4f7..e37193d0d 100644 --- a/salt/logrotate/defaults.yaml +++ b/salt/logrotate/defaults.yaml @@ -150,6 +150,16 @@ logrotate: - extension .log - dateext - dateyesterday + /opt/so/log/postgres/*_x_log: + - daily + - rotate 14 + - missingok + - copytruncate + - compress + - create + - extension .log + - dateext + - dateyesterday /opt/so/log/telegraf/*_x_log: - daily - rotate 14 diff --git a/salt/logrotate/soc_logrotate.yaml b/salt/logrotate/soc_logrotate.yaml index f407ab48d..f329ed623 100644 --- a/salt/logrotate/soc_logrotate.yaml +++ b/salt/logrotate/soc_logrotate.yaml @@ -91,6 +91,13 @@ logrotate: multiline: True global: True forcedType: "[]string" + "/opt/so/log/postgres/*_x_log": + description: List of logrotate options for this file. + title: /opt/so/log/postgres/*.log + advanced: True + multiline: True + global: True + forcedType: "[]string" "/opt/so/log/telegraf/*_x_log": description: List of logrotate options for this file. title: /opt/so/log/telegraf/*.log diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index f5b40310e..9f02eb32a 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -880,29 +880,21 @@ recollate_postgres() { echo "" } -bootstrap_so_soc_database() { - # init-db.sh is mounted into so-postgres at /docker-entrypoint-initdb.d/init-db.sh - # and runs automatically only on a fresh data directory. Hosts upgrading from - # 3.1.0 already have /nsm/postgres populated, so the so_soc bootstrap block - # added in 3.2 never fires. Re-run the script explicitly; it's idempotent. - echo "Bootstrapping database via init-db.sh." - # The postgres image has no USER directive, so `docker exec` defaults to - # root, and the container env intentionally omits POSTGRES_USER (the upstream - # entrypoint defaults it transiently during first-init only). Recreate both - # so psql inside init-db.sh resolves the connect user correctly. - local exec_cmd="docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh" - if ! /usr/sbin/so-postgres-wait; then - FINAL_MESSAGE_QUEUE+=("WARNING: so-postgres was not ready during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd") +scrub_postgres_log_passwords() { + # Purge plaintext passwords a pre-3.2 postgres could log on DDL errors. + local log=/opt/so/log/postgres/postgres.log + [[ -f "$log" ]] || return 0 + if ! grep -qai "PASSWORD '" "$log" 2>/dev/null; then + echo "No leaked passwords found in $log." return 0 fi - if ! $exec_cmd; then - FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the database may not have been bootstrapped. Re-run manually: $exec_cmd") - return 0 - fi - echo "Database bootstrap complete." - - echo "Restarting so-soc container to pick up database changes" - docker restart so-soc + echo "Removing leaked password statements from $log." + local tmp + tmp=$(mktemp) + # Rewrite in place (cat >) to keep the inode postgres is writing to. + grep -avi "PASSWORD '" "$log" > "$tmp" 2>/dev/null || true + cat "$tmp" > "$log" + rm -f "$tmp" } # Existing grids should keep ILM unless an admin explicitly opts in to DLM. @@ -989,7 +981,8 @@ post_to_3.2.0() { # Recollate due to image OS rebase recollate_postgres - bootstrap_so_soc_database + # SOC database bootstrap is handled by the postgres.enabled highstate. + scrub_postgres_log_passwords # Generate 9.3.7 elastic agent installers echo "Regenerating Elastic Agent Installers" diff --git a/salt/postgres/enabled.sls b/salt/postgres/enabled.sls index b0eadd205..fa91d146a 100644 --- a/salt/postgres/enabled.sls +++ b/salt/postgres/enabled.sls @@ -85,6 +85,23 @@ so-postgres: - x509: postgres_crt - x509: postgres_key +postgres_wait_ready: + cmd.run: + - name: /usr/sbin/so-postgres-wait + - require: + - docker_container: so-postgres + - file: postgres_sbin + +# Reconcile the SOC database (role, grants, so_telegraf) every highstate so a +# partially-initialized cluster self-heals. POSTGRES_USER is injected because +# the container env omits it. +postgres_bootstrap_soc_db: + cmd.run: + - name: docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh + - require: + - cmd: postgres_wait_ready + - file: postgresinitdb + delete_so-postgres_so-status.disabled: file.uncomment: - name: /opt/so/conf/so-status/so-status.conf diff --git a/salt/postgres/files/init-db.sh b/salt/postgres/files/init-db.sh index d12bc4c9b..4d65b0c97 100644 --- a/salt/postgres/files/init-db.sh +++ b/salt/postgres/files/init-db.sh @@ -8,13 +8,17 @@ if [ -z "${SO_POSTGRES_PASS:-}" ] && [ -n "${SO_POSTGRES_PASS_FILE:-}" ] && [ -r SO_POSTGRES_PASS="$(< "$SO_POSTGRES_PASS_FILE")" fi psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + -- Keep the password out of postgres.log if this DDL errors. + SET log_min_error_statement = panic; + -- Idempotent, race-safe upsert: CREATE, falling back to ALTER if the role + -- already exists or is created concurrently. DO \$\$ BEGIN - IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${SO_POSTGRES_USER}') THEN + BEGIN EXECUTE format('CREATE ROLE %I WITH LOGIN PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}'); - ELSE - EXECUTE format('ALTER ROLE %I WITH PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}'); - END IF; + EXCEPTION WHEN duplicate_object OR unique_violation THEN + EXECUTE format('ALTER ROLE %I WITH LOGIN PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}'); + END; END \$\$; GRANT ALL ON SCHEMA public TO "$SO_POSTGRES_USER"; diff --git a/salt/postgres/telegraf_users.sls b/salt/postgres/telegraf_users.sls index 5e3566a95..4c63d40b0 100644 --- a/salt/postgres/telegraf_users.sls +++ b/salt/postgres/telegraf_users.sls @@ -8,23 +8,14 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'telegraf/map.jinja' import TELEGRAFMERGED %} -{# postgres_wait_ready below requires `docker_container: so-postgres`, which is - declared in postgres.enabled. Include it here so state.apply postgres.telegraf_users - on its own (e.g. from orch.deploy_newnode) still has that ID in scope. Salt - de-duplicates the circular include. #} +{# postgres.enabled declares the so-postgres container and postgres_wait_ready + that the requires below reference. Salt de-duplicates the circular include. #} include: - postgres.enabled {% set TG_OUT = TELEGRAFMERGED.output | upper %} {% if TG_OUT in ['POSTGRES', 'BOTH'] %} -postgres_wait_ready: - cmd.run: - - name: /usr/sbin/so-postgres-wait - - require: - - docker_container: so-postgres - - file: postgres_sbin - # Ensure the shared Telegraf database exists. init-db.sh only runs on a # fresh data dir, so hosts upgraded onto an existing /nsm/postgres volume # would otherwise never get so_telegraf. diff --git a/salt/postgres/tools/sbin/so-telegraf-postgres b/salt/postgres/tools/sbin/so-telegraf-postgres index ef7c3f9e6..17e4da744 100644 --- a/salt/postgres/tools/sbin/so-telegraf-postgres +++ b/salt/postgres/tools/sbin/so-telegraf-postgres @@ -71,6 +71,8 @@ EOSQL -v role_user="$ROLE_USER" \ -v role_pass="$ROLE_PASS" \ -U postgres -d so_telegraf <<'EOSQL' +-- Keep the password out of postgres.log if this DDL errors. +SET log_min_error_statement = panic; SELECT format( CASE WHEN EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'role_user') THEN 'ALTER ROLE %I WITH LOGIN PASSWORD %L'