diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 8def509e1..eb63a1d60 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -880,31 +880,6 @@ recollate_postgres() { echo "" } -bootstrap_so_soc_database() { - # init-db.sh is mounted into so-postgres at /docker-entrypoint-initdb.d/init-db.sh - # and runs automatically only on a fresh data directory. Hosts upgrading from - # 3.1.0 already have /nsm/postgres populated, so the so_soc bootstrap block - # added in 3.2 never fires. Re-run the script explicitly; it's idempotent. - echo "Bootstrapping database via init-db.sh." - # The postgres image has no USER directive, so `docker exec` defaults to - # root, and the container env intentionally omits POSTGRES_USER (the upstream - # entrypoint defaults it transiently during first-init only). Recreate both - # so psql inside init-db.sh resolves the connect user correctly. - local exec_cmd="docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh" - if ! /usr/sbin/so-postgres-wait; then - FINAL_MESSAGE_QUEUE+=("WARNING: so-postgres was not ready during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd") - return 0 - fi - if ! $exec_cmd; then - FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the database may not have been bootstrapped. Re-run manually: $exec_cmd") - return 0 - fi - echo "Database bootstrap complete." - - echo "Restarting so-soc container to pick up database changes" - docker restart so-soc -} - scrub_postgres_log_passwords() { # Before 3.2 PostgreSQL could log the full CREATE/ALTER ROLE ... PASSWORD # statement -- with the plaintext password -- to postgres.log whenever such a @@ -1013,7 +988,9 @@ post_to_3.2.0() { # Recollate due to image OS rebase recollate_postgres - bootstrap_so_soc_database + # The SOC database (so_postgres role, schema/db grants, so_telegraf) is + # bootstrapped idempotently by the postgres.enabled highstate that runs + # before this postupgrade step, so no explicit bootstrap call is needed here. # Purge any plaintext passwords a pre-3.2 postgres logged on DDL errors. scrub_postgres_log_passwords diff --git a/salt/postgres/enabled.sls b/salt/postgres/enabled.sls index b0eadd205..ad6ff7819 100644 --- a/salt/postgres/enabled.sls +++ b/salt/postgres/enabled.sls @@ -85,6 +85,26 @@ so-postgres: - x509: postgres_crt - x509: postgres_key +postgres_wait_ready: + cmd.run: + - name: /usr/sbin/so-postgres-wait + - require: + - docker_container: so-postgres + - file: postgres_sbin + +# Re-run the SOC database bootstrap on every highstate so a cluster left +# partially initialized -- e.g. the container was restarted mid first-init by a +# watch trigger, so docker-entrypoint-initdb.d never completed -- self-heals: +# the so_postgres role, its schema/database grants, and the so_telegraf database +# are all reconciled idempotently. init-db.sh is idempotent and race-safe. +# POSTGRES_USER is injected because the container env intentionally omits it. +postgres_bootstrap_soc_db: + cmd.run: + - name: docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh + - require: + - cmd: postgres_wait_ready + - file: postgresinitdb + delete_so-postgres_so-status.disabled: file.uncomment: - name: /opt/so/conf/so-status/so-status.conf diff --git a/salt/postgres/files/init-db.sh b/salt/postgres/files/init-db.sh index d1f1e375f..540980469 100644 --- a/salt/postgres/files/init-db.sh +++ b/salt/postgres/files/init-db.sh @@ -12,13 +12,19 @@ psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-E -- log_min_error_statement defaults to 'error', which would append a STATEMENT line -- containing the full CREATE/ALTER ROLE ... PASSWORD text. panic suppresses that. SET log_min_error_statement = panic; + -- Race-safe upsert: try CREATE, and if the role was created concurrently + -- (another session re-entering init) fall back to ALTER. Catching the + -- exception -- rather than an IF NOT EXISTS check -- avoids a TOCTOU window + -- where CREATE ROLE would abort the whole script with a duplicate-key error + -- and skip the grants below. Both SQLSTATEs are covered: duplicate_object + -- (role already exists) and unique_violation (racy pg_authid index insert). DO \$\$ BEGIN - IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${SO_POSTGRES_USER}') THEN + BEGIN EXECUTE format('CREATE ROLE %I WITH LOGIN PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}'); - ELSE - EXECUTE format('ALTER ROLE %I WITH PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}'); - END IF; + EXCEPTION WHEN duplicate_object OR unique_violation THEN + EXECUTE format('ALTER ROLE %I WITH LOGIN PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}'); + END; END \$\$; GRANT ALL ON SCHEMA public TO "$SO_POSTGRES_USER"; diff --git a/salt/postgres/telegraf_users.sls b/salt/postgres/telegraf_users.sls index 5e3566a95..62c33b892 100644 --- a/salt/postgres/telegraf_users.sls +++ b/salt/postgres/telegraf_users.sls @@ -8,23 +8,16 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'telegraf/map.jinja' import TELEGRAFMERGED %} -{# postgres_wait_ready below requires `docker_container: so-postgres`, which is - declared in postgres.enabled. Include it here so state.apply postgres.telegraf_users - on its own (e.g. from orch.deploy_newnode) still has that ID in scope. Salt - de-duplicates the circular include. #} +{# postgres_wait_ready and the so-postgres container are declared in + postgres.enabled; include it so the require references below resolve and so + state.apply postgres.telegraf_users on its own (e.g. from orch.deploy_newnode) + still has those IDs in scope. Salt de-duplicates the circular include. #} include: - postgres.enabled {% set TG_OUT = TELEGRAFMERGED.output | upper %} {% if TG_OUT in ['POSTGRES', 'BOTH'] %} -postgres_wait_ready: - cmd.run: - - name: /usr/sbin/so-postgres-wait - - require: - - docker_container: so-postgres - - file: postgres_sbin - # Ensure the shared Telegraf database exists. init-db.sh only runs on a # fresh data dir, so hosts upgraded onto an existing /nsm/postgres volume # would otherwise never get so_telegraf.