Merge pull request #15787 from Security-Onion-Solutions/feature/postgres

Fix Telegraf postgres template syntax, partman privileges, and idempotency

salt/postgres/files/init-users.sh

@@ -20,7 +20,6 @@ EOSQL
 # Bootstrap the Telegraf metrics database. Per-minion roles + schemas are
 # reconciled on every state.apply by postgres/telegraf_users.sls; this block
 # only ensures the shared database exists on first initialization.
-psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
-    SELECT 'CREATE DATABASE so_telegraf'
-    WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'so_telegraf')\gexec
-EOSQL
+if ! psql -U "$POSTGRES_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1; then
+    psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE DATABASE so_telegraf"
+fi

salt/postgres/telegraf_users.sls

@@ -34,10 +34,9 @@ postgres_wait_ready:
 postgres_create_telegraf_db:
   cmd.run:
     - name: |
-        docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d postgres <<'EOSQL'
-        SELECT 'CREATE DATABASE so_telegraf'
-        WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'so_telegraf')\gexec
-        EOSQL
+        if ! docker exec so-postgres psql -U postgres -tAc "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1; then
+          docker exec so-postgres psql -v ON_ERROR_STOP=1 -U postgres -c "CREATE DATABASE so_telegraf"
+        fi
     - require:
       - cmd: postgres_wait_ready
 
@@ -62,6 +61,20 @@ postgres_telegraf_group_role:
         CREATE SCHEMA IF NOT EXISTS partman;
         CREATE EXTENSION IF NOT EXISTS pg_partman SCHEMA partman;
         CREATE EXTENSION IF NOT EXISTS pg_cron;
+        -- Telegraf (running as so_telegraf) calls partman.create_parent()
+        -- on first write of each metric, which needs USAGE on the partman
+        -- schema, EXECUTE on its functions/procedures, and write access to
+        -- partman.part_config so it can register new partitioned parents.
+        GRANT USAGE, CREATE ON SCHEMA partman TO so_telegraf;
+        GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA partman TO so_telegraf;
+        GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA partman TO so_telegraf;
+        GRANT EXECUTE ON ALL PROCEDURES IN SCHEMA partman TO so_telegraf;
+        -- partman creates per-parent template tables (partman.template_*) at
+        -- runtime; default privileges extend DML/sequence access to them.
+        ALTER DEFAULT PRIVILEGES IN SCHEMA partman
+            GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO so_telegraf;
+        ALTER DEFAULT PRIVILEGES IN SCHEMA partman
+            GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO so_telegraf;
         -- Hourly partman maintenance. cron.schedule is idempotent by jobname.
         SELECT cron.schedule(
           'telegraf-partman-maintenance',

salt/telegraf/etc/telegraf.conf

@@ -109,10 +109,22 @@
   fields_as_jsonb = true
   # Every metric table is a daily time-range partitioned parent managed by
   # pg_partman. Retention drops old partitions instead of row-by-row DELETEs.
+  {% raw %}
+  # pg_partman 5.x requires the control column (time) to be NOT NULL, so
+  # ALTER it before create_parent(). And create_parent() splits
+  # p_parent_table on '.' to look up raw identifiers, so the literal must
+  # be 'schema.name' (not '"schema"."name"' as .table|quoteLiteral emits).
+  # IF NOT EXISTS keeps the three templates idempotent so a Telegraf
+  # restart after any DB-side surgery re-runs them safely.
   create_templates = [
-    '''CREATE TABLE {TABLE} ({COLUMNS}) PARTITION BY RANGE ("time")''',
-    '''SELECT partman.create_parent(p_parent_table := {TABLELITERAL}, p_control := 'time', p_type := 'range', p_interval := '1 day', p_premake := 3)'''
+    '''CREATE TABLE IF NOT EXISTS {{ .table }} ({{ .columns }}) PARTITION BY RANGE ("time")''',
+    '''ALTER TABLE {{ .table }} ALTER COLUMN "time" SET NOT NULL''',
+    '''SELECT partman.create_parent(p_parent_table := {{ printf "%s.%s" .table.Schema .table.Name | quoteLiteral }}, p_control := 'time', p_type := 'range', p_interval := '1 day', p_premake := 3) WHERE NOT EXISTS (SELECT 1 FROM partman.part_config WHERE parent_table = {{ printf "%s.%s" .table.Schema .table.Name | quoteLiteral }})'''
   ]
+  tag_table_create_templates = [
+    '''CREATE TABLE IF NOT EXISTS {{ .table }} ({{ .columns }}, PRIMARY KEY (tag_id))'''
+  ]
+  {% endraw %}
 {%- endif %}
 
 ###############################################################################