From 92a7bb305370e49f52568a1bcc2bdbef9cadf1fa Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 4 May 2026 19:47:38 -0400 Subject: [PATCH] fix: get postsalt's PG-canonical pillar actually working end-to-end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five blockers turned up the first time the so_pillar schema was applied against a fresh standalone install. Fixing them in order: 1. 006_rls.sql ordering bug 006 GRANTed on so_pillar.change_queue and its sequence, but the table isn't created until 008_change_notify.sql. 006 errored mid-file with "relation so_pillar.change_queue does not exist", short-circuiting the rest of the pillar staging chain. Moved the three change_queue grants into 008 alongside the table creation so each file is self-contained. 2. so_pillar_* roles unable to log in 006 created the roles as NOLOGIN and set no password. Salt-master's ext_pillar (postgres) and the pg_notify_pillar engine both connect as so_pillar_master via TCP, so both came up with "password authentication failed for user so_pillar_master". Added a templated cmd.run step in schema_pillar.sls (so_pillar_role_login_passwords) that ALTERs all three roles WITH LOGIN PASSWORD pulling from secrets:pillar_master_pass — the same password ext_pillar_postgres.conf.jinja and the engines.conf pg_notify_pillar block render with. 3. Missing GRANT CONNECT ON DATABASE securityonion USAGE on the schema is granted in 006 but CONNECT on the database isn't. Engine + ext_pillar succeeded auth then died with "permission denied for database securityonion". Added the explicit GRANT CONNECT in 006. 4. psycopg2 missing from salt's bundled python /opt/saltstack/salt/bin/python3 doesn't ship psycopg by default, so when salt-master tries to load the pg_notify_pillar engine its `import psycopg2` fails inside salt's loader and the engine silently doesn't start (no error in the salt log — you only notice when nothing ever drains so_pillar.change_queue). Added a pip.installed state in schema_pillar.sls bound to that interpreter via bin_env. 5. engines.conf vs pg_notify_pillar_engine.conf list-replace Salt's master.d/*.conf merge replaces top-level lists rather than concatenating them. The engine config used to live in its own master.d/pg_notify_pillar_engine.conf with `engines: [pg_notify_pillar]` alongside the legacy `engines.conf` carrying `engines: [checkmine, pillarWatch]`. Whichever loaded last won, so the engine never showed up in the loaded set even when the file existed. Fold the pg_notify_pillar declaration into engines.conf (now jinja-rendered, gated on postgres:so_pillar:enabled), drop the standalone state from pg_notify_pillar_engine.sls, and delete the now-orphaned conf jinja. End state validated against a live standalone-net install on the dev rig: salt-master ext_pillar reads from so_pillar.* with no errors, the pg_notify_pillar engine LISTENs on so_pillar_change and drains the change_queue (134-row backlog → 0 within seconds), and a so-yaml replace on a pillar key flows disk → PG → ext_pillar → salt pillar.get with the new value visible after a saltutil.refresh_pillar. --- salt/postgres/files/schema/pillar/006_rls.sql | 19 +++++------ .../files/schema/pillar/008_change_notify.sql | 12 +++++++ salt/postgres/schema_pillar.sls | 32 ++++++++++++++++++- salt/salt/files/engines.conf | 20 ++++++++++++ salt/salt/master.sls | 1 + .../files/pg_notify_pillar_engine.conf.jinja | 20 ------------ salt/salt/master/pg_notify_pillar_engine.sls | 23 +++++++------ 7 files changed, 85 insertions(+), 42 deletions(-) delete mode 100644 salt/salt/master/files/pg_notify_pillar_engine.conf.jinja diff --git a/salt/postgres/files/schema/pillar/006_rls.sql b/salt/postgres/files/schema/pillar/006_rls.sql index 3b0da95ae..d18bd8ff4 100644 --- a/salt/postgres/files/schema/pillar/006_rls.sql +++ b/salt/postgres/files/schema/pillar/006_rls.sql @@ -28,6 +28,14 @@ BEGIN END $$; +-- USAGE on the schema is the bare minimum needed to reference its tables. +-- CONNECT on the database is needed before the role can establish a session +-- at all (default privileges on a new DB grant CONNECT to PUBLIC, but if the +-- securityonion database is restricted that grant has to be explicit). +-- Password + LOGIN privileges are set later in schema_pillar.sls because +-- the password lives in pillar (secrets:pillar_master_pass) and plain SQL +-- can't substitute pillar values. +GRANT CONNECT ON DATABASE securityonion TO so_pillar_master, so_pillar_writer, so_pillar_secret_owner; GRANT USAGE ON SCHEMA so_pillar TO so_pillar_master, so_pillar_writer, so_pillar_secret_owner; -- Read access for ext_pillar through the views only. @@ -37,15 +45,8 @@ GRANT SELECT ON so_pillar.v_pillar_global, TO so_pillar_master; GRANT EXECUTE ON FUNCTION so_pillar.fn_pillar_secrets(text) TO so_pillar_master; --- Engine reads + drains the change queue from the salt-master process. It --- needs SELECT to find unprocessed rows and UPDATE to mark them processed. --- The queue contains only locator metadata (no pillar data), so the master --- role's existing privilege footprint is unchanged in practice. -GRANT SELECT, UPDATE ON so_pillar.change_queue TO so_pillar_master; -GRANT USAGE ON SEQUENCE so_pillar.change_queue_id_seq TO so_pillar_master; --- Writer needs INSERT (the trigger runs as table owner, so this is just for --- direct testing / manual replays from psql). -GRANT INSERT ON so_pillar.change_queue TO so_pillar_writer; +-- (change_queue grants live in 008_change_notify.sql alongside the table itself, +-- since the table doesn't exist until 008 runs.) -- Writer needs CRUD on pillar_entry/minion/role_member plus access to seed tables. GRANT SELECT, INSERT, UPDATE, DELETE diff --git a/salt/postgres/files/schema/pillar/008_change_notify.sql b/salt/postgres/files/schema/pillar/008_change_notify.sql index 7fe22ad18..595e78b74 100644 --- a/salt/postgres/files/schema/pillar/008_change_notify.sql +++ b/salt/postgres/files/schema/pillar/008_change_notify.sql @@ -75,3 +75,15 @@ CREATE TRIGGER tg_pillar_entry_notify ON so_pillar.pillar_entry FOR EACH ROW EXECUTE FUNCTION so_pillar.fn_pillar_entry_notify(); + +-- Role grants on the change_queue table. Lived in 006_rls.sql historically but +-- moved here so the GRANT references resolve — 006 runs before this file does. +-- Engine reads + drains the change queue from the salt-master process. It +-- needs SELECT to find unprocessed rows and UPDATE to mark them processed. +-- The queue contains only locator metadata (no pillar data), so the master +-- role's existing privilege footprint is unchanged in practice. +GRANT SELECT, UPDATE ON so_pillar.change_queue TO so_pillar_master; +GRANT USAGE ON SEQUENCE so_pillar.change_queue_id_seq TO so_pillar_master; +-- Writer needs INSERT (the trigger runs as table owner, so this is just for +-- direct testing / manual replays from psql). +GRANT INSERT ON so_pillar.change_queue TO so_pillar_writer; diff --git a/salt/postgres/schema_pillar.sls b/salt/postgres/schema_pillar.sls index 6bd846c63..5d8c793af 100644 --- a/salt/postgres/schema_pillar.sls +++ b/salt/postgres/schema_pillar.sls @@ -93,13 +93,43 @@ so_pillar_master_key_configure: - require: - cmd: so_pillar_apply_{{ sql_files[-1] | replace('.', '_') }} +# Set login passwords on the so_pillar_* roles. 006_rls.sql creates the roles +# as NOLOGIN with no password (plain SQL can't substitute pillar values), so +# the salt-master ext_pillar and the pg_notify_pillar engine — both of which +# connect as so_pillar_master via TCP — would fail with "password +# authentication failed" without this step. The password lives in pillar +# under secrets:pillar_master_pass (generated by setup/so-functions::secrets_pillar) +# and is the same one rendered into ext_pillar_postgres.conf.jinja and the +# engines.conf pg_notify_pillar block, so all three sides agree. +so_pillar_role_login_passwords: + cmd.run: + - name: | + docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d securityonion <