CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2026-07-21 16:27:07 +02:00
Code Issues Packages Projects Releases Wiki Activity

Self-heal PostgreSQL SOC database bootstrap

Browse Source 
init-db.sh only runs on a fresh PGDATA (docker-entrypoint-initdb.d), so a
cluster left partially initialized -- e.g. the container restarted mid first-init
by a watch trigger -- never recovered: the so_postgres role existed but its
schema grants and the so_telegraf database were missing, breaking SOC with
"permission denied for schema public".

- init-db.sh: make the role upsert race-safe. Try CREATE ROLE and fall back to
  ALTER on duplicate_object/unique_violation instead of IF NOT EXISTS, so a
  concurrent creator no longer aborts the script (set -e + ON_ERROR_STOP=1)
  before the grants and so_telegraf creation run. The whole script is now
  idempotent and safe to re-run.
- postgres/enabled.sls: move postgres_wait_ready here (was telegraf-gated in
  telegraf_users.sls) and add postgres_bootstrap_soc_db, which re-runs init-db.sh
  every highstate so a partially-initialized cluster self-heals.
- telegraf_users.sls: drop its now-duplicate postgres_wait_ready definition; it
  resolves from postgres.enabled via the existing include.
- soup: remove bootstrap_so_soc_database and its post_to_3.2.0 call. The
  highstates soup runs before postupgrade now reconcile the SOC DB, making the
  one-shot bootstrap redundant.
This commit is contained in:
Mike Reeves
2026-07-20 15:04:58 -04:00
parent ad78e84ccd
commit 011749ad09
4 changed files with 37 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
salt/manager/tools/sbin/soup
+3 -26
View File
@@ -880,31 +880,6 @@ recollate_postgres() {
echo ""
}
bootstrap_so_soc_database() {
# init-db.sh is mounted into so-postgres at /docker-entrypoint-initdb.d/init-db.sh
# and runs automatically only on a fresh data directory. Hosts upgrading from
# 3.1.0 already have /nsm/postgres populated, so the so_soc bootstrap block
# added in 3.2 never fires. Re-run the script explicitly; it's idempotent.
echo "Bootstrapping database via init-db.sh."
# The postgres image has no USER directive, so `docker exec` defaults to
# root, and the container env intentionally omits POSTGRES_USER (the upstream
# entrypoint defaults it transiently during first-init only). Recreate both
# so psql inside init-db.sh resolves the connect user correctly.
local exec_cmd="docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh"
if ! /usr/sbin/so-postgres-wait; then
FINAL_MESSAGE_QUEUE+=("WARNING: so-postgres was not ready during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd")
return 0
fi
if ! $exec_cmd; then
FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the database may not have been bootstrapped. Re-run manually: $exec_cmd")
return 0
fi
echo "Database bootstrap complete."
echo "Restarting so-soc container to pick up database changes"
docker restart so-soc
}
scrub_postgres_log_passwords() {
# Before 3.2 PostgreSQL could log the full CREATE/ALTER ROLE ... PASSWORD
# statement -- with the plaintext password -- to postgres.log whenever such a
@@ -1013,7 +988,9 @@ post_to_3.2.0() {
# Recollate due to image OS rebase
recollate_postgres
bootstrap_so_soc_database
# The SOC database (so_postgres role, schema/db grants, so_telegraf) is
# bootstrapped idempotently by the postgres.enabled highstate that runs
# before this postupgrade step, so no explicit bootstrap call is needed here.
# Purge any plaintext passwords a pre-3.2 postgres logged on DDL errors.
scrub_postgres_log_passwords
salt/postgres/enabled.sls
+20
View File
@@ -85,6 +85,26 @@ so-postgres:
- x509: postgres_crt
- x509: postgres_key
postgres_wait_ready:
cmd.run:
- name: /usr/sbin/so-postgres-wait
- require:
- docker_container: so-postgres
- file: postgres_sbin
# Re-run the SOC database bootstrap on every highstate so a cluster left
# partially initialized -- e.g. the container was restarted mid first-init by a
# watch trigger, so docker-entrypoint-initdb.d never completed -- self-heals:
# the so_postgres role, its schema/database grants, and the so_telegraf database
# are all reconciled idempotently. init-db.sh is idempotent and race-safe.
# POSTGRES_USER is injected because the container env intentionally omits it.
postgres_bootstrap_soc_db:
cmd.run:
- name: docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh
- require:
- cmd: postgres_wait_ready
- file: postgresinitdb
delete_so-postgres_so-status.disabled:
file.uncomment:
- name: /opt/so/conf/so-status/so-status.conf
salt/postgres/files/init-db.sh
+10 -4
View File
@@ -12,13 +12,19 @@ psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-E
-- log_min_error_statement defaults to 'error', which would append a STATEMENT line
-- containing the full CREATE/ALTER ROLE ... PASSWORD text. panic suppresses that.
SET log_min_error_statement = panic;
-- Race-safe upsert: try CREATE, and if the role was created concurrently
-- (another session re-entering init) fall back to ALTER. Catching the
-- exception -- rather than an IF NOT EXISTS check -- avoids a TOCTOU window
-- where CREATE ROLE would abort the whole script with a duplicate-key error
-- and skip the grants below. Both SQLSTATEs are covered: duplicate_object
-- (role already exists) and unique_violation (racy pg_authid index insert).
DO \$\$
BEGIN
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${SO_POSTGRES_USER}') THEN
BEGIN
EXECUTE format('CREATE ROLE %I WITH LOGIN PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}');
ELSE
EXECUTE format('ALTER ROLE %I WITH PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}');
END IF;
EXCEPTION WHEN duplicate_object OR unique_violation THEN
EXECUTE format('ALTER ROLE %I WITH LOGIN PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}');
END;
END
\$\$;
GRANT ALL ON SCHEMA public TO "$SO_POSTGRES_USER";
salt/postgres/telegraf_users.sls
+4 -11
View File
@@ -8,23 +8,16 @@
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'telegraf/map.jinja' import TELEGRAFMERGED %}
{# postgres_wait_ready below requires `docker_container: so-postgres`, which is
declared in postgres.enabled. Include it here so state.apply postgres.telegraf_users
on its own (e.g. from orch.deploy_newnode) still has that ID in scope. Salt
de-duplicates the circular include. #}
{# postgres_wait_ready and the so-postgres container are declared in
postgres.enabled; include it so the require references below resolve and so
state.apply postgres.telegraf_users on its own (e.g. from orch.deploy_newnode)
still has those IDs in scope. Salt de-duplicates the circular include. #}
include:
- postgres.enabled
{% set TG_OUT = TELEGRAFMERGED.output | upper %}
{% if TG_OUT in ['POSTGRES', 'BOTH'] %}
postgres_wait_ready:
cmd.run:
- name: /usr/sbin/so-postgres-wait
- require:
- docker_container: so-postgres
- file: postgres_sbin
# Ensure the shared Telegraf database exists. init-db.sh only runs on a
# fresh data dir, so hosts upgraded onto an existing /nsm/postgres volume
# would otherwise never get so_telegraf.