From 5d36d00dec839dbbeb2fdcb4d2e0c5871197ceb9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 5 Aug 2026 17:28:28 -0400 Subject: [PATCH 1/7] Fix Telegraf metrics falling into pg_partman default partitions pg_cron's launcher connects to cron.database_name at postmaster start and is registered BGW_NEVER_RESTART. On a host upgraded onto an existing /nsm/postgres volume, init-db.sh never runs, so so_telegraf does not exist when PostgreSQL starts -- the launcher dies and never retries. Salt then creates the database, the extension, and the schedule, all of which succeed, but no worker is left to fire the job. partman.run_maintenance_proc() therefore never runs: partitions stop being premade after create_parent's initial window and every metric lands in _default. Retention never fires either. That state is self-perpetuating. Once the default partition holds rows for a day with no child, PostgreSQL cannot create that child at all -- attaching it would violate the default partition's constraint -- so maintenance aborts on the first parent it reaches. Fixing the scheduler alone does not recover a stalled grid. Point cron.database_name at the always-present postgres database and register the job with cron.schedule_in_database targeting so_telegraf, so the launcher no longer depends on database creation order. group_role drops any registration left behind in so_telegraf, and both halves are guarded on the live GUC so applying postgres.telegraf_users before the postgresql.conf change has restarted the container skips instead of failing. Maintenance now runs so_admin.telegraf_maintenance(), which drains stranded rows out of any default partition before calling partman: expired rows are deleted, the rest are repartitioned. It runs from the state on every highstate as well as hourly from pg_cron, so a grid whose worker is dead still recovers on its own. The routines live in a postgres-owned schema so_telegraf has no rights on, since pg_cron executes them as postgres. Existing grids are recovered by a marker-guarded repair state that truncates the non-empty defaults once per host. The backlog is mostly past retention already and moving tens of GB just to delete most of it is not worth the WAL. Also raise premake from 3 to 7, reconciled onto existing parents in the retention subcommand, so an outage has a week of headroom before anything reaches a default partition, and add a check subcommand reporting partition age, default occupancy and last job status. --- salt/postgres/defaults.yaml | 2 +- salt/postgres/soc_postgres.yaml | 2 +- salt/postgres/telegraf_users.sls | 19 ++ salt/postgres/tools/sbin/so-telegraf-postgres | 235 +++++++++++++++++- salt/telegraf/etc/telegraf.conf | 2 +- 5 files changed, 250 insertions(+), 10 deletions(-) diff --git a/salt/postgres/defaults.yaml b/salt/postgres/defaults.yaml index 7ad82f453..754a0d44c 100644 --- a/salt/postgres/defaults.yaml +++ b/salt/postgres/defaults.yaml @@ -16,4 +16,4 @@ postgres: logging_collector: 'off' log_min_messages: 'warning' shared_preload_libraries: pg_cron - cron.database_name: so_telegraf + cron.database_name: postgres diff --git a/salt/postgres/soc_postgres.yaml b/salt/postgres/soc_postgres.yaml index 4b25cd4f5..4ab1e4dac 100644 --- a/salt/postgres/soc_postgres.yaml +++ b/salt/postgres/soc_postgres.yaml @@ -83,7 +83,7 @@ postgres: advanced: True helpLink: postgres cron.database_name: - description: Database pg_cron schedules jobs in. Must be so_telegraf so partman maintenance runs in the right database context. + description: Database pg_cron keeps its job metadata in. Must already exist when PostgreSQL starts, because pg_cron's launcher connects to it at startup and never retries if it is missing. The maintenance job itself targets so_telegraf. global: True advanced: True helpLink: postgres diff --git a/salt/postgres/telegraf_users.sls b/salt/postgres/telegraf_users.sls index 4c63d40b0..3544755a5 100644 --- a/salt/postgres/telegraf_users.sls +++ b/salt/postgres/telegraf_users.sls @@ -70,6 +70,25 @@ postgres_telegraf_retention_reconcile: - cmd: postgres_telegraf_group_role - file: postgres_sbin +# One-time recovery for grids whose pg_cron job never ran. Truncating the +# defaults is destructive, so the watermark keeps it to one pass per host. +postgres_telegraf_partman_default_repair: + cmd.run: + - name: mkdir -p /opt/so/state && /usr/sbin/so-telegraf-postgres repair && touch /opt/so/state/telegraf_partman_default_repair + - creates: /opt/so/state/telegraf_partman_default_repair + - require: + - cmd: postgres_telegraf_retention_reconcile + - file: postgres_sbin + +# Also run from the state, not just hourly from pg_cron, so a grid whose +# pg_cron worker is dead still recovers on its own. +postgres_telegraf_partman_maintenance: + cmd.run: + - name: /usr/sbin/so-telegraf-postgres maintenance + - require: + - cmd: postgres_telegraf_partman_default_repair + - file: postgres_sbin + {% endif %} {% else %} diff --git a/salt/postgres/tools/sbin/so-telegraf-postgres b/salt/postgres/tools/sbin/so-telegraf-postgres index 17e4da744..fe31fb949 100644 --- a/salt/postgres/tools/sbin/so-telegraf-postgres +++ b/salt/postgres/tools/sbin/so-telegraf-postgres @@ -5,14 +5,35 @@ set -e # Usage: so-telegraf-postgres # create_db Ensure the so_telegraf database exists. # group_role Provision the so_telegraf group role, telegraf/partman schemas, -# pg_partman, pg_cron, and the hourly partman maintenance job. +# pg_partman, the so_admin maintenance routines, and the hourly +# pg_cron maintenance job. # user Create or update a per-minion login role granted to so_telegraf. # Env: ROLE_USER, ROLE_PASS. -# retention Reconcile partman retention on telegraf parents. +# retention Reconcile partman retention and premake on telegraf parents. # Env: RETENTION_DAYS. +# maintenance Drain default partitions and run partman maintenance. +# repair One-time recovery: TRUNCATE non-empty defaults, then maintain. +# check Report partition health. Non-zero if any parent is unhealthy. +# +# Once a default partition holds rows for a day with no child, Postgres cannot +# create that child at all -- attaching it would violate the default's +# constraint -- so maintenance drains defaults before calling partman. cmd="${1:?subcommand required}" +# A function rather than re-invoking $0, which assumes the script is executable. +run_maintenance() { + docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL' +SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL + THEN 'true' ELSE 'false' END AS has_proc \gset +\if :has_proc +CALL so_admin.telegraf_maintenance(); +\else +\echo 'so_admin.telegraf_maintenance() is missing; run so-telegraf-postgres group_role first.' +\endif +EOSQL +} + case "$cmd" in create_db) if ! docker exec so-postgres psql -U postgres -tAc \ @@ -36,7 +57,6 @@ CREATE SCHEMA IF NOT EXISTS telegraf AUTHORIZATION so_telegraf; GRANT USAGE, CREATE ON SCHEMA telegraf TO so_telegraf; CREATE SCHEMA IF NOT EXISTS partman; CREATE EXTENSION IF NOT EXISTS pg_partman SCHEMA partman; -CREATE EXTENSION IF NOT EXISTS pg_cron; -- Telegraf (running as so_telegraf) calls partman.create_parent() -- on first write of each metric, which needs USAGE on the partman -- schema, EXECUTE on its functions/procedures, and write access to @@ -51,12 +71,141 @@ ALTER DEFAULT PRIVILEGES IN SCHEMA partman GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO so_telegraf; ALTER DEFAULT PRIVILEGES IN SCHEMA partman GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO so_telegraf; --- Hourly partman maintenance. cron.schedule is idempotent by jobname. -SELECT cron.schedule( + +-- pg_cron runs these as postgres, so they must not sit in a schema any +-- Telegraf role can create objects in. +CREATE SCHEMA IF NOT EXISTS so_admin AUTHORIZATION postgres; +REVOKE ALL ON SCHEMA so_admin FROM PUBLIC; + +CREATE OR REPLACE PROCEDURE so_admin.telegraf_maintenance() +LANGUAGE plpgsql +AS $proc$ +DECLARE + r record; + v_default text; + v_rows bigint; +BEGIN + -- No per-parent EXCEPTION handler: partition_data_proc commits internally, + -- and COMMIT is illegal while a subtransaction is active. A failing parent + -- aborts the run and the next pass retries. + FOR r IN + SELECT parent_table, retention + FROM partman.part_config + WHERE parent_table LIKE 'telegraf.%' + ORDER BY parent_table + LOOP + v_default := format('%I.%I', + split_part(r.parent_table, '.', 1), + split_part(r.parent_table, '.', 2) || '_default'); + + CONTINUE WHEN to_regclass(v_default) IS NULL; + + EXECUTE format('SELECT count(*) FROM %s', v_default) INTO v_rows; + CONTINUE WHEN v_rows = 0; + + RAISE WARNING 'so_admin.telegraf_maintenance: % rows stranded in %, draining', + v_rows, v_default; + + -- Cheaper to delete expired rows than to repartition and then drop them. + IF r.retention IS NOT NULL THEN + EXECUTE format('DELETE FROM %s WHERE "time" < now() - %L::interval', + v_default, r.retention); + COMMIT; + END IF; + + -- Bounded so a large backlog drains across several runs. + CALL partman.partition_data_proc( + p_parent_table := r.parent_table, + p_loop_count := 200, + p_source_table := v_default + ); + COMMIT; + END LOOP; + + CALL partman.run_maintenance_proc(); +END; +$proc$; + +CREATE OR REPLACE FUNCTION so_admin.telegraf_partition_status() +RETURNS TABLE ( + parent_table text, + oldest_child date, + newest_child date, + days_ahead int, + retention text, + default_rows bigint, + default_size text +) +LANGUAGE plpgsql +AS $func$ +DECLARE + r record; + v_default regclass; +BEGIN + FOR r IN + SELECT pc.parent_table AS pt, pc.retention AS ret + FROM partman.part_config pc + WHERE pc.parent_table LIKE 'telegraf.%' + ORDER BY pc.parent_table + LOOP + parent_table := r.pt; + retention := r.ret; + + SELECT min(d), max(d) INTO oldest_child, newest_child + FROM ( + SELECT to_date(substring(c.relname FROM '_p(\d{8})$'), 'YYYYMMDD') AS d + FROM pg_inherits i + JOIN pg_class c ON c.oid = i.inhrelid + WHERE i.inhparent = r.pt::regclass + AND pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT' + ) s; + + days_ahead := newest_child - current_date; + + v_default := to_regclass(format('%I.%I', + split_part(r.pt, '.', 1), + split_part(r.pt, '.', 2) || '_default')); + IF v_default IS NULL THEN + default_rows := NULL; + default_size := NULL; + ELSE + EXECUTE format('SELECT count(*) FROM %s', v_default::text) INTO default_rows; + default_size := pg_size_pretty(pg_total_relation_size(v_default)); + END IF; + + RETURN NEXT; + END LOOP; +END; +$func$; + +-- Drop the registration older releases left in so_telegraf. +SELECT CASE + WHEN current_setting('cron.database_name', true) IS DISTINCT FROM current_database() + AND EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_cron') + THEN 'true' ELSE 'false' + END AS drop_stale_cron \gset +\if :drop_stale_cron +DROP EXTENSION pg_cron CASCADE; +\endif +EOSQL + + # Guarded on the live GUC so applying this before the postgresql.conf change + # has restarted the container skips rather than failing. + docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d postgres <<'EOSQL' +SELECT CASE WHEN current_setting('cron.database_name', true) = current_database() + THEN 'true' ELSE 'false' END AS cron_here \gset +\if :cron_here +CREATE EXTENSION IF NOT EXISTS pg_cron; +-- cron.schedule_in_database is idempotent by jobname. +SELECT cron.schedule_in_database( 'telegraf-partman-maintenance', '17 * * * *', - 'CALL partman.run_maintenance_proc()' + 'CALL so_admin.telegraf_maintenance()', + 'so_telegraf' ); +\else +\echo 'pg_cron metadata database is not `postgres` yet; skipping job registration.' +\endif EOSQL ;; @@ -90,6 +239,8 @@ EOSQL : "${RETENTION_DAYS:?RETENTION_DAYS is required}" # \gset + \if guards against a missing pg_partman without using a DO # block (psql :var substitution doesn't reach into dollar-quoted code). + # premake is reconciled here because telegraf.conf only applies it to + # parents created from now on. docker exec -i so-postgres psql \ -v ON_ERROR_STOP=1 \ -v retention_days="$RETENTION_DAYS" \ @@ -99,12 +250,82 @@ SELECT CASE WHEN EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = ' \if :has_partman UPDATE partman.part_config SET retention = :'retention_days' || ' days', - retention_keep_table = false + retention_keep_table = false, + premake = 7 WHERE parent_table LIKE 'telegraf.%'; \endif EOSQL ;; + maintenance) + run_maintenance + ;; + + repair) + # Stranded rows are discarded, not repartitioned: most are already past + # retention, and moving tens of GB just to drop them isn't worth the WAL. + docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL' +SELECT CASE WHEN EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_partman') + THEN 'true' ELSE 'false' END AS has_partman \gset +\if :has_partman +\echo 'Default partitions holding stranded Telegraf metrics:' +SELECT format('%I.%I', n.nspname, c.relname) AS default_partition, + pg_size_pretty(pg_total_relation_size(c.oid)) AS size +FROM partman.part_config pc +JOIN pg_class p ON p.oid = pc.parent_table::regclass +JOIN pg_inherits i ON i.inhparent = p.oid +JOIN pg_class c ON c.oid = i.inhrelid +JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE pc.parent_table LIKE 'telegraf.%' + AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT' + AND pg_total_relation_size(c.oid) > 0 +ORDER BY pg_total_relation_size(c.oid) DESC; + +SELECT format('TRUNCATE TABLE %I.%I', n.nspname, c.relname) +FROM partman.part_config pc +JOIN pg_class p ON p.oid = pc.parent_table::regclass +JOIN pg_inherits i ON i.inhparent = p.oid +JOIN pg_class c ON c.oid = i.inhrelid +JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE pc.parent_table LIKE 'telegraf.%' + AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT' +\gexec +\endif +EOSQL + # Premake the window so the next flush has somewhere to land. + run_maintenance + ;; + + check) + docker exec -i so-postgres psql -U postgres -d so_telegraf <<'EOSQL' +\pset border 2 +SELECT * FROM so_admin.telegraf_partition_status(); +EOSQL + docker exec -i so-postgres psql -U postgres -d postgres <<'EOSQL' +\pset border 2 +SELECT CASE WHEN to_regclass('cron.job_run_details') IS NOT NULL + THEN 'true' ELSE 'false' END AS has_cron \gset +\if :has_cron +SELECT d.status, d.return_message, d.start_time +FROM cron.job_run_details d +JOIN cron.job j ON j.jobid = d.jobid +WHERE j.jobname = 'telegraf-partman-maintenance' +ORDER BY d.start_time DESC +LIMIT 5; +\else +\echo 'pg_cron is not installed in this database.' +\endif +EOSQL + unhealthy=$(docker exec so-postgres psql -U postgres -d so_telegraf -tAc \ + "SELECT count(*) FROM so_admin.telegraf_partition_status() + WHERE coalesce(default_rows, 0) > 0 OR coalesce(days_ahead, -1) < 1") + if [ "${unhealthy:-1}" != "0" ]; then + echo "so-telegraf-postgres check: $unhealthy telegraf parent(s) unhealthy" >&2 + exit 1 + fi + echo "so-telegraf-postgres check: all telegraf parents healthy" + ;; + *) echo "Unknown subcommand: $cmd" >&2 exit 1 diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index d7cc15a38..93d0e71b3 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -122,7 +122,7 @@ create_templates = [ '''CREATE TABLE IF NOT EXISTS {{ .table }} ({{ .columns }}) PARTITION BY RANGE ("time")''', '''ALTER TABLE {{ .table }} ALTER COLUMN "time" SET NOT NULL''', - '''SELECT partman.create_parent(p_parent_table := {{ printf "%s.%s" .table.Schema .table.Name | quoteLiteral }}, p_control := 'time', p_type := 'range', p_interval := '1 day', p_premake := 3) WHERE NOT EXISTS (SELECT 1 FROM partman.part_config WHERE parent_table = {{ printf "%s.%s" .table.Schema .table.Name | quoteLiteral }})''' + '''SELECT partman.create_parent(p_parent_table := {{ printf "%s.%s" .table.Schema .table.Name | quoteLiteral }}, p_control := 'time', p_type := 'range', p_interval := '1 day', p_premake := 7) WHERE NOT EXISTS (SELECT 1 FROM partman.part_config WHERE parent_table = {{ printf "%s.%s" .table.Schema .table.Name | quoteLiteral }})''' ] tag_table_create_templates = [ '''CREATE TABLE IF NOT EXISTS {{ .table }} ({{ .columns }}, PRIMARY KEY (tag_id))''' From a2a4d9314d47cdda234fcbdc3661a50dd306b722 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 6 Aug 2026 12:28:40 -0400 Subject: [PATCH 2/7] Repair stalled Telegraf partitions from soup instead of every highstate Truncating default partitions is destructive and premaking them is pg_cron's job, so neither belongs in a state that runs on every checkin. Replace the two telegraf_users states with so-telegraf-partition-repair, a standalone tool that reports partition health and clears the backlog, and call it once from soup. The script depends only on pg_partman, so it also runs against a grid that has not yet picked up the new postgres state. It no-ops when nothing is stranded, refuses to discard rows non-interactively without --yes, and reports when the pg_cron job has never fired, which is the underlying cause rather than a symptom the truncate addresses. Hourly self-healing stays with so_admin.telegraf_maintenance() via pg_cron, so a grid that never soups still recovers, just gradually and without discarding in-retention metrics. --- salt/manager/tools/sbin/soup | 15 ++ salt/postgres/telegraf_users.sls | 19 --- .../tools/sbin/so-telegraf-partition-repair | 153 ++++++++++++++++++ salt/postgres/tools/sbin/so-telegraf-postgres | 55 +------ 4 files changed, 176 insertions(+), 66 deletions(-) create mode 100644 salt/postgres/tools/sbin/so-telegraf-partition-repair diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index b155d1bec..b5366facf 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1013,9 +1013,24 @@ up_to_3.3.0() { INSTALLEDVERSION=3.3.0 } +telegraf_partition_repair() { + # Grids upgraded onto an existing /nsm/postgres never ran init-db.sh, so + # so_telegraf did not exist when PostgreSQL started and pg_cron's launcher + # died without retrying. partman maintenance never ran and every metric piled + # up in _default, which then blocks partition creation outright. + # Fresh installs are unaffected. The script no-ops when nothing is stranded. + [[ -x /usr/sbin/so-telegraf-partition-repair ]] || return 0 + docker ps --format '{{.Names}}' | grep -qx so-postgres || return 0 + echo "Checking Telegraf metric partitions." + /usr/sbin/so-telegraf-partition-repair --yes \ + || echo " warning: so-telegraf-partition-repair failed; run it manually" >&2 +} + post_to_3.3.0() { # Recollate again since some internal DBs were excluded during 3.2.0 soup recollate_postgres + + telegraf_partition_repair } ### 3.3.0 End ### diff --git a/salt/postgres/telegraf_users.sls b/salt/postgres/telegraf_users.sls index 3544755a5..4c63d40b0 100644 --- a/salt/postgres/telegraf_users.sls +++ b/salt/postgres/telegraf_users.sls @@ -70,25 +70,6 @@ postgres_telegraf_retention_reconcile: - cmd: postgres_telegraf_group_role - file: postgres_sbin -# One-time recovery for grids whose pg_cron job never ran. Truncating the -# defaults is destructive, so the watermark keeps it to one pass per host. -postgres_telegraf_partman_default_repair: - cmd.run: - - name: mkdir -p /opt/so/state && /usr/sbin/so-telegraf-postgres repair && touch /opt/so/state/telegraf_partman_default_repair - - creates: /opt/so/state/telegraf_partman_default_repair - - require: - - cmd: postgres_telegraf_retention_reconcile - - file: postgres_sbin - -# Also run from the state, not just hourly from pg_cron, so a grid whose -# pg_cron worker is dead still recovers on its own. -postgres_telegraf_partman_maintenance: - cmd.run: - - name: /usr/sbin/so-telegraf-postgres maintenance - - require: - - cmd: postgres_telegraf_partman_default_repair - - file: postgres_sbin - {% endif %} {% else %} diff --git a/salt/postgres/tools/sbin/so-telegraf-partition-repair b/salt/postgres/tools/sbin/so-telegraf-partition-repair new file mode 100644 index 000000000..e6c83a434 --- /dev/null +++ b/salt/postgres/tools/sbin/so-telegraf-partition-repair @@ -0,0 +1,153 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Repair Telegraf metric partitions on a grid where pg_partman maintenance +# stalled and metrics piled up in _default. +# +# Once a default partition holds rows for a day with no child partition, +# Postgres cannot create that child at all -- attaching it would violate the +# default's constraint -- so maintenance stays broken until the default is +# emptied. This discards the stranded rows rather than repartitioning them: +# most are already past retention, and moving tens of GB just to drop them is +# not worth the WAL. +# +# Self-contained on purpose: it depends only on pg_partman, so it runs against +# a grid that has not yet picked up the current postgres state. +# +# Usage: so-telegraf-partition-repair [--dry-run] [--yes] +# --dry-run Report only; change nothing. +# --yes Skip the confirmation prompt (for soup and other automation). + +set -e + +DRY_RUN=false +ASSUME_YES=false + +usage() { + sed -n '8,24p' "$0" | sed 's/^# \?//' +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) DRY_RUN=true ;; + --yes|-y) ASSUME_YES=true ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage >&2; exit 1 ;; + esac + shift +done + +fail() { echo "ERROR: $*" >&2; exit 1; } + +psql_tg() { docker exec -i so-postgres psql -U postgres -d so_telegraf "$@"; } + +# Row counts have to come from dynamic SQL; query_to_xml keeps that in a plain +# query so this needs no helper functions installed in the database. +REPORT=" +WITH parents AS ( + SELECT pc.parent_table, + split_part(pc.parent_table, '.', 1) AS sch, + split_part(pc.parent_table, '.', 2) AS tbl + FROM partman.part_config pc + WHERE pc.parent_table LIKE 'telegraf.%' +), children AS ( + SELECT p.parent_table, + max(to_date(substring(c.relname FROM '_p(\d{8})\$'), 'YYYYMMDD')) AS newest_child + FROM parents p + JOIN pg_class pt ON pt.oid = p.parent_table::regclass + JOIN pg_inherits i ON i.inhparent = pt.oid + JOIN pg_class c ON c.oid = i.inhrelid + WHERE pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT' + GROUP BY p.parent_table +), defaults AS ( + SELECT p.parent_table, + format('%I.%I', p.sch, p.tbl || '_default') AS default_table, + to_regclass(format('%I.%I', p.sch, p.tbl || '_default')) AS default_oid + FROM parents p +) +SELECT d.parent_table, + c.newest_child, + (c.newest_child - current_date) AS days_ahead, + CASE WHEN d.default_oid IS NULL THEN NULL ELSE + (xpath('/row/cnt/text()', + query_to_xml(format('SELECT count(*) AS cnt FROM %s', d.default_table), + false, true, '')))[1]::text::bigint + END AS default_rows, + CASE WHEN d.default_oid IS NULL THEN NULL + ELSE pg_size_pretty(pg_total_relation_size(d.default_oid)) END AS default_size +FROM defaults d +LEFT JOIN children c ON c.parent_table = d.parent_table +ORDER BY 1 +" + +docker ps --format '{{.Names}}' | grep -qx so-postgres \ + || fail "so-postgres is not running." +docker exec so-postgres psql -U postgres -tAc \ + "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1 \ + || fail "The so_telegraf database does not exist; Telegraf is not writing to Postgres." +psql_tg -tAc "SELECT 1 FROM pg_extension WHERE extname='pg_partman'" | grep -q 1 \ + || fail "pg_partman is not installed in so_telegraf." + +echo "Telegraf partition status:" +psql_tg -c "$REPORT" + +stranded=$(psql_tg -tAc "SELECT coalesce(sum(default_rows), 0) FROM ( $REPORT ) t") +echo "Rows stranded in default partitions: $stranded" + +# Report the scheduler too. A grid that needed this script almost always has a +# pg_cron job that has never fired, and repairing partitions without fixing that +# just delays the next occurrence. +cron_db=$(docker exec so-postgres psql -U postgres -tAc \ + "SELECT current_setting('cron.database_name', true)" | tr -d '[:space:]') +if [[ -n "$cron_db" ]]; then + runs=$(docker exec so-postgres psql -U postgres -d "$cron_db" -tAc \ + "SELECT count(*) FROM cron.job_run_details d JOIN cron.job j USING (jobid) + WHERE j.jobname = 'telegraf-partman-maintenance'" 2>/dev/null | tr -d '[:space:]' || true) + if [[ "$runs" == "0" ]]; then + echo + echo "WARNING: the telegraf-partman-maintenance job has never run. Apply the" + echo " postgres state so pg_cron is pointed at a database that exists" + echo " at server start, or this will recur." + fi +fi + +if [[ "$stranded" == "0" ]]; then + echo "Nothing stranded. Running maintenance to premake the current window." + $DRY_RUN && { echo "(dry run: skipping maintenance)"; exit 0; } + psql_tg -v ON_ERROR_STOP=1 -c "CALL partman.run_maintenance_proc()" + exit 0 +fi + +if $DRY_RUN; then + echo "(dry run: would TRUNCATE the default partitions listed above)" + exit 0 +fi + +if ! $ASSUME_YES; then + echo + echo "This will permanently discard the $stranded stranded row(s) above." + [[ -t 0 ]] || fail "Not a terminal; re-run with --yes to confirm." + read -r -p "Continue? [y/N] " answer + [[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 1; } +fi + +psql_tg -v ON_ERROR_STOP=1 <<'EOSQL' +SELECT format('TRUNCATE TABLE %I.%I', n.nspname, c.relname) +FROM partman.part_config pc +JOIN pg_class p ON p.oid = pc.parent_table::regclass +JOIN pg_inherits i ON i.inhparent = p.oid +JOIN pg_class c ON c.oid = i.inhrelid +JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE pc.parent_table LIKE 'telegraf.%' + AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT' +\gexec +CALL partman.run_maintenance_proc(); +EOSQL + +echo +echo "Telegraf partition status after repair:" +psql_tg -c "$REPORT" diff --git a/salt/postgres/tools/sbin/so-telegraf-postgres b/salt/postgres/tools/sbin/so-telegraf-postgres index fe31fb949..eb096eb65 100644 --- a/salt/postgres/tools/sbin/so-telegraf-postgres +++ b/salt/postgres/tools/sbin/so-telegraf-postgres @@ -12,28 +12,16 @@ set -e # retention Reconcile partman retention and premake on telegraf parents. # Env: RETENTION_DAYS. # maintenance Drain default partitions and run partman maintenance. -# repair One-time recovery: TRUNCATE non-empty defaults, then maintain. # check Report partition health. Non-zero if any parent is unhealthy. # # Once a default partition holds rows for a day with no child, Postgres cannot # create that child at all -- attaching it would violate the default's # constraint -- so maintenance drains defaults before calling partman. +# To recover a grid that is already in that state, use +# so-telegraf-partition-repair, which discards the backlog instead of moving it. cmd="${1:?subcommand required}" -# A function rather than re-invoking $0, which assumes the script is executable. -run_maintenance() { - docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL' -SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL - THEN 'true' ELSE 'false' END AS has_proc \gset -\if :has_proc -CALL so_admin.telegraf_maintenance(); -\else -\echo 'so_admin.telegraf_maintenance() is missing; run so-telegraf-postgres group_role first.' -\endif -EOSQL -} - case "$cmd" in create_db) if ! docker exec so-postgres psql -U postgres -tAc \ @@ -258,42 +246,15 @@ EOSQL ;; maintenance) - run_maintenance - ;; - - repair) - # Stranded rows are discarded, not repartitioned: most are already past - # retention, and moving tens of GB just to drop them isn't worth the WAL. docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL' -SELECT CASE WHEN EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_partman') - THEN 'true' ELSE 'false' END AS has_partman \gset -\if :has_partman -\echo 'Default partitions holding stranded Telegraf metrics:' -SELECT format('%I.%I', n.nspname, c.relname) AS default_partition, - pg_size_pretty(pg_total_relation_size(c.oid)) AS size -FROM partman.part_config pc -JOIN pg_class p ON p.oid = pc.parent_table::regclass -JOIN pg_inherits i ON i.inhparent = p.oid -JOIN pg_class c ON c.oid = i.inhrelid -JOIN pg_namespace n ON n.oid = c.relnamespace -WHERE pc.parent_table LIKE 'telegraf.%' - AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT' - AND pg_total_relation_size(c.oid) > 0 -ORDER BY pg_total_relation_size(c.oid) DESC; - -SELECT format('TRUNCATE TABLE %I.%I', n.nspname, c.relname) -FROM partman.part_config pc -JOIN pg_class p ON p.oid = pc.parent_table::regclass -JOIN pg_inherits i ON i.inhparent = p.oid -JOIN pg_class c ON c.oid = i.inhrelid -JOIN pg_namespace n ON n.oid = c.relnamespace -WHERE pc.parent_table LIKE 'telegraf.%' - AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT' -\gexec +SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL + THEN 'true' ELSE 'false' END AS has_proc \gset +\if :has_proc +CALL so_admin.telegraf_maintenance(); +\else +\echo 'so_admin.telegraf_maintenance() is missing; run so-telegraf-postgres group_role first.' \endif EOSQL - # Premake the window so the next flush has somewhere to land. - run_maintenance ;; check) From 668ab447a27706b99fd1587e408d817b8607ae73 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 7 Aug 2026 15:02:55 -0400 Subject: [PATCH 3/7] Exclude so_telegraf from the nightly Postgres backup pg_dumpall dumped every database, and so_telegraf dominated the result: it is the only database that grows with grid size and metric volume, while everything else in the cluster is small and mostly static. Nothing is lost by skipping it. The data is transient metrics on a 14-day retention window, roles are globals so the per-minion telegraf logins are still dumped, and the database itself is rebuilt after a restore without operator action -- init-db.sh recreates it (run on every highstate by postgres_bootstrap_soc_db, not just on a fresh volume), telegraf_users.sls re-provisions the roles and schema, and Telegraf recreates its tables on first write. --- salt/postgres/tools/sbin/so-postgres-backup | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/postgres/tools/sbin/so-postgres-backup b/salt/postgres/tools/sbin/so-postgres-backup index 08a73e3a4..4de5adf62 100644 --- a/salt/postgres/tools/sbin/so-postgres-backup +++ b/salt/postgres/tools/sbin/so-postgres-backup @@ -47,7 +47,12 @@ trap 'rm -f "$TMPFILE"' EXIT # Dump all databases and roles, compress. Write to a temp file so the final # filename only ever appears for a complete, verified backup. -if ! docker exec so-postgres pg_dumpall -U postgres | gzip > "$TMPFILE"; then +# +# so_telegraf is excluded: it is transient metrics on a short retention window, +# it dominates the dump size, and it is rebuilt automatically after a restore -- +# init-db.sh recreates the database and Telegraf recreates its tables on first +# write. Roles are globals, so the per-minion telegraf logins are still dumped. +if ! docker exec so-postgres pg_dumpall -U postgres --exclude-database=so_telegraf | gzip > "$TMPFILE"; then log "ERROR: pg_dumpall/gzip failed; backup aborted" exit 1 fi From fe4f7ad2f71bff53a88ec57844caefa49c539557 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2026 10:05:16 -0400 Subject: [PATCH 4/7] Repair a stalled grid in one step, and only when it needs it so-telegraf-partition-repair cleared the backlog but left the cause in place: pg_cron's launcher is still dead, so the grid re-stalls as soon as it walks off the premade window. Operators on the preview release need something they can run once, before they soup, that leaves Telegraf collecting again. Replace it with so-telegraf-repair, which fixes both halves. The running release already creates the pg_cron extension and registers telegraf-partman-maintenance in so_telegraf; only the launcher is missing, because so_telegraf did not exist when the postmaster started. Restarting so-postgres is therefore enough to get the existing job firing, so this touches no configuration and duplicates none of the postgres state's SQL -- group_role still migrates the job to the postgres database on the next soup. It also reconciles premake to 7 and prefers so_admin.telegraf_maintenance() when that state has already landed. Exit status separates healthy (0) from needs-repair (1) from does-not-apply (2), which is what soup now gates on. postupgrade_changes runs after the highstate, so the database is already converted by then and the backlog is the only thing left to detect. Truncating is destructive and most grids were never affected -- fresh installs in particular, since they have no Telegraf history at all -- so soup asks first and skips silently rather than clearing defaults on every host. --- salt/manager/tools/sbin/soup | 26 +- .../tools/sbin/so-telegraf-partition-repair | 153 ---------- salt/postgres/tools/sbin/so-telegraf-postgres | 5 +- salt/postgres/tools/sbin/so-telegraf-repair | 261 ++++++++++++++++++ 4 files changed, 284 insertions(+), 161 deletions(-) delete mode 100644 salt/postgres/tools/sbin/so-telegraf-partition-repair create mode 100644 salt/postgres/tools/sbin/so-telegraf-repair diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index b5366facf..ade67ab54 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1013,24 +1013,38 @@ up_to_3.3.0() { INSTALLEDVERSION=3.3.0 } -telegraf_partition_repair() { +telegraf_repair() { # Grids upgraded onto an existing /nsm/postgres never ran init-db.sh, so # so_telegraf did not exist when PostgreSQL started and pg_cron's launcher # died without retrying. partman maintenance never ran and every metric piled # up in _default, which then blocks partition creation outright. - # Fresh installs are unaffected. The script no-ops when nothing is stranded. - [[ -x /usr/sbin/so-telegraf-partition-repair ]] || return 0 + # + # The highstate above has already converted the database to the current state, + # but it deliberately does not discard the backlog those grids accumulated. + # --check reports whether this host is one of them; it exits 1 only when there + # is something to repair, so fresh installs and grids that were never affected + # are left alone rather than truncated. + local repair=/usr/sbin/so-telegraf-repair + [[ -x "$repair" ]] || return 0 docker ps --format '{{.Names}}' | grep -qx so-postgres || return 0 + echo "Checking Telegraf metric partitions." - /usr/sbin/so-telegraf-partition-repair --yes \ - || echo " warning: so-telegraf-partition-repair failed; run it manually" >&2 + local status=0 + "$repair" --check >> "$SOUP_LOG" 2>&1 || status=$? + case "$status" in + 0) echo " Telegraf partitions are healthy; nothing to repair." ;; + 1) echo " Repairing stalled Telegraf partitions." + "$repair" --yes \ + || echo " warning: so-telegraf-repair failed; run it manually" >&2 ;; + *) echo " Skipping; Telegraf is not storing metrics in Postgres on this host." ;; + esac } post_to_3.3.0() { # Recollate again since some internal DBs were excluded during 3.2.0 soup recollate_postgres - telegraf_partition_repair + telegraf_repair } ### 3.3.0 End ### diff --git a/salt/postgres/tools/sbin/so-telegraf-partition-repair b/salt/postgres/tools/sbin/so-telegraf-partition-repair deleted file mode 100644 index e6c83a434..000000000 --- a/salt/postgres/tools/sbin/so-telegraf-partition-repair +++ /dev/null @@ -1,153 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -# Repair Telegraf metric partitions on a grid where pg_partman maintenance -# stalled and metrics piled up in _default. -# -# Once a default partition holds rows for a day with no child partition, -# Postgres cannot create that child at all -- attaching it would violate the -# default's constraint -- so maintenance stays broken until the default is -# emptied. This discards the stranded rows rather than repartitioning them: -# most are already past retention, and moving tens of GB just to drop them is -# not worth the WAL. -# -# Self-contained on purpose: it depends only on pg_partman, so it runs against -# a grid that has not yet picked up the current postgres state. -# -# Usage: so-telegraf-partition-repair [--dry-run] [--yes] -# --dry-run Report only; change nothing. -# --yes Skip the confirmation prompt (for soup and other automation). - -set -e - -DRY_RUN=false -ASSUME_YES=false - -usage() { - sed -n '8,24p' "$0" | sed 's/^# \?//' -} - -while [[ $# -gt 0 ]]; do - case "$1" in - --dry-run) DRY_RUN=true ;; - --yes|-y) ASSUME_YES=true ;; - -h|--help) usage; exit 0 ;; - *) echo "Unknown option: $1" >&2; usage >&2; exit 1 ;; - esac - shift -done - -fail() { echo "ERROR: $*" >&2; exit 1; } - -psql_tg() { docker exec -i so-postgres psql -U postgres -d so_telegraf "$@"; } - -# Row counts have to come from dynamic SQL; query_to_xml keeps that in a plain -# query so this needs no helper functions installed in the database. -REPORT=" -WITH parents AS ( - SELECT pc.parent_table, - split_part(pc.parent_table, '.', 1) AS sch, - split_part(pc.parent_table, '.', 2) AS tbl - FROM partman.part_config pc - WHERE pc.parent_table LIKE 'telegraf.%' -), children AS ( - SELECT p.parent_table, - max(to_date(substring(c.relname FROM '_p(\d{8})\$'), 'YYYYMMDD')) AS newest_child - FROM parents p - JOIN pg_class pt ON pt.oid = p.parent_table::regclass - JOIN pg_inherits i ON i.inhparent = pt.oid - JOIN pg_class c ON c.oid = i.inhrelid - WHERE pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT' - GROUP BY p.parent_table -), defaults AS ( - SELECT p.parent_table, - format('%I.%I', p.sch, p.tbl || '_default') AS default_table, - to_regclass(format('%I.%I', p.sch, p.tbl || '_default')) AS default_oid - FROM parents p -) -SELECT d.parent_table, - c.newest_child, - (c.newest_child - current_date) AS days_ahead, - CASE WHEN d.default_oid IS NULL THEN NULL ELSE - (xpath('/row/cnt/text()', - query_to_xml(format('SELECT count(*) AS cnt FROM %s', d.default_table), - false, true, '')))[1]::text::bigint - END AS default_rows, - CASE WHEN d.default_oid IS NULL THEN NULL - ELSE pg_size_pretty(pg_total_relation_size(d.default_oid)) END AS default_size -FROM defaults d -LEFT JOIN children c ON c.parent_table = d.parent_table -ORDER BY 1 -" - -docker ps --format '{{.Names}}' | grep -qx so-postgres \ - || fail "so-postgres is not running." -docker exec so-postgres psql -U postgres -tAc \ - "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1 \ - || fail "The so_telegraf database does not exist; Telegraf is not writing to Postgres." -psql_tg -tAc "SELECT 1 FROM pg_extension WHERE extname='pg_partman'" | grep -q 1 \ - || fail "pg_partman is not installed in so_telegraf." - -echo "Telegraf partition status:" -psql_tg -c "$REPORT" - -stranded=$(psql_tg -tAc "SELECT coalesce(sum(default_rows), 0) FROM ( $REPORT ) t") -echo "Rows stranded in default partitions: $stranded" - -# Report the scheduler too. A grid that needed this script almost always has a -# pg_cron job that has never fired, and repairing partitions without fixing that -# just delays the next occurrence. -cron_db=$(docker exec so-postgres psql -U postgres -tAc \ - "SELECT current_setting('cron.database_name', true)" | tr -d '[:space:]') -if [[ -n "$cron_db" ]]; then - runs=$(docker exec so-postgres psql -U postgres -d "$cron_db" -tAc \ - "SELECT count(*) FROM cron.job_run_details d JOIN cron.job j USING (jobid) - WHERE j.jobname = 'telegraf-partman-maintenance'" 2>/dev/null | tr -d '[:space:]' || true) - if [[ "$runs" == "0" ]]; then - echo - echo "WARNING: the telegraf-partman-maintenance job has never run. Apply the" - echo " postgres state so pg_cron is pointed at a database that exists" - echo " at server start, or this will recur." - fi -fi - -if [[ "$stranded" == "0" ]]; then - echo "Nothing stranded. Running maintenance to premake the current window." - $DRY_RUN && { echo "(dry run: skipping maintenance)"; exit 0; } - psql_tg -v ON_ERROR_STOP=1 -c "CALL partman.run_maintenance_proc()" - exit 0 -fi - -if $DRY_RUN; then - echo "(dry run: would TRUNCATE the default partitions listed above)" - exit 0 -fi - -if ! $ASSUME_YES; then - echo - echo "This will permanently discard the $stranded stranded row(s) above." - [[ -t 0 ]] || fail "Not a terminal; re-run with --yes to confirm." - read -r -p "Continue? [y/N] " answer - [[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 1; } -fi - -psql_tg -v ON_ERROR_STOP=1 <<'EOSQL' -SELECT format('TRUNCATE TABLE %I.%I', n.nspname, c.relname) -FROM partman.part_config pc -JOIN pg_class p ON p.oid = pc.parent_table::regclass -JOIN pg_inherits i ON i.inhparent = p.oid -JOIN pg_class c ON c.oid = i.inhrelid -JOIN pg_namespace n ON n.oid = c.relnamespace -WHERE pc.parent_table LIKE 'telegraf.%' - AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT' -\gexec -CALL partman.run_maintenance_proc(); -EOSQL - -echo -echo "Telegraf partition status after repair:" -psql_tg -c "$REPORT" diff --git a/salt/postgres/tools/sbin/so-telegraf-postgres b/salt/postgres/tools/sbin/so-telegraf-postgres index eb096eb65..537bec6bc 100644 --- a/salt/postgres/tools/sbin/so-telegraf-postgres +++ b/salt/postgres/tools/sbin/so-telegraf-postgres @@ -17,8 +17,9 @@ set -e # Once a default partition holds rows for a day with no child, Postgres cannot # create that child at all -- attaching it would violate the default's # constraint -- so maintenance drains defaults before calling partman. -# To recover a grid that is already in that state, use -# so-telegraf-partition-repair, which discards the backlog instead of moving it. +# To recover a grid that is already in that state, use so-telegraf-repair, +# which discards the backlog instead of moving it and revives pg_cron's +# launcher without waiting for this state to be applied. cmd="${1:?subcommand required}" diff --git a/salt/postgres/tools/sbin/so-telegraf-repair b/salt/postgres/tools/sbin/so-telegraf-repair new file mode 100644 index 000000000..48bb744e0 --- /dev/null +++ b/salt/postgres/tools/sbin/so-telegraf-repair @@ -0,0 +1,261 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Put Telegraf metrics storage back in service on a grid where pg_partman +# maintenance stalled. +# +# pg_cron's launcher connects to cron.database_name at postmaster start and is +# registered BGW_NEVER_RESTART. On a host upgraded onto an existing +# /nsm/postgres volume, init-db.sh never ran, so so_telegraf did not exist when +# PostgreSQL started: the launcher died and never retried. partman maintenance +# therefore never ran, partitions stopped being premade after create_parent's +# initial window, and every metric landed in _default. Retention never +# fired either. +# +# That state is self-perpetuating. Once a default partition holds rows for a day +# with no child, Postgres cannot create that child at all -- attaching it would +# violate the default's constraint -- so maintenance aborts on the first parent +# it reaches and fixing the scheduler alone does not recover the grid. +# +# This repairs both halves: +# * raises premake to 7 on existing parents, so a future outage has a week of +# headroom before anything reaches a default partition +# * discards the rows stranded in default partitions -- most are past +# retention already, and repartitioning tens of GB just to delete most of it +# is not worth the WAL +# * restarts so-postgres when pg_cron's launcher is dead, which is what makes +# the hourly maintenance job start firing again +# * runs maintenance once so partitions exist for the current window +# +# Self-contained on purpose: it depends only on what an affected grid already +# has, so it can be handed to an operator whose grid has not picked up the +# current postgres state yet. Nothing here conflicts with that state -- it +# reuses the pg_cron job the running version already registered, and the +# postgres state migrates the job to the postgres database on the next soup. +# +# Fresh installs need none of this; the check below reports them healthy and +# changes nothing. +# +# Usage: so-telegraf-repair [--check] [--yes] [--no-restart] +# --check Report health and change nothing. +# --yes Skip the confirmation prompt (for soup and other automation). +# --no-restart Never restart so-postgres, even if pg_cron's launcher is dead. +# +# Exit status: +# 0 healthy, or repair completed +# 1 repair is needed (--check only) +# 2 cannot run here: so-postgres, so_telegraf or pg_partman is missing + +set -e + +# Matches p_premake in telegraf.conf's create_parent template. +PREMAKE=7 +JOB_NAME=telegraf-partman-maintenance + +CHECK_ONLY=false +ASSUME_YES=false +NO_RESTART=false + +usage() { sed -n '/^# Usage:/,/^# 2 /p' "$0" | sed 's/^# \?//'; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --check|--dry-run) CHECK_ONLY=true ;; + --yes|-y) ASSUME_YES=true ;; + --no-restart) NO_RESTART=true ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage >&2; exit 2 ;; + esac + shift +done + +skip() { echo "$*"; exit 2; } + +psql_tg() { docker exec -i so-postgres psql -U postgres -d so_telegraf "$@"; } +psql_pg() { docker exec -i so-postgres psql -U postgres -d postgres "$@"; } + +# Row counts have to come from dynamic SQL; query_to_xml keeps that in a plain +# query so this needs no helper functions installed in the database. +REPORT=" +WITH parents AS ( + SELECT pc.parent_table, + pc.premake, + split_part(pc.parent_table, '.', 1) AS sch, + split_part(pc.parent_table, '.', 2) AS tbl + FROM partman.part_config pc + WHERE pc.parent_table LIKE 'telegraf.%' +), children AS ( + SELECT p.parent_table, + max(to_date(substring(c.relname FROM '_p(\d{8})\$'), 'YYYYMMDD')) AS newest_child + FROM parents p + JOIN pg_class pt ON pt.oid = p.parent_table::regclass + JOIN pg_inherits i ON i.inhparent = pt.oid + JOIN pg_class c ON c.oid = i.inhrelid + WHERE pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT' + GROUP BY p.parent_table +), defaults AS ( + SELECT p.parent_table, + p.premake, + format('%I.%I', p.sch, p.tbl || '_default') AS default_table, + to_regclass(format('%I.%I', p.sch, p.tbl || '_default')) AS default_oid + FROM parents p +) +SELECT d.parent_table, + c.newest_child, + (c.newest_child - current_date) AS days_ahead, + d.premake, + CASE WHEN d.default_oid IS NULL THEN NULL ELSE + (xpath('/row/cnt/text()', + query_to_xml(format('SELECT count(*) AS cnt FROM %s', d.default_table), + false, true, '')))[1]::text::bigint + END AS default_rows, + CASE WHEN d.default_oid IS NULL THEN NULL + ELSE pg_size_pretty(pg_total_relation_size(d.default_oid)) END AS default_size +FROM defaults d +LEFT JOIN children c ON c.parent_table = d.parent_table +ORDER BY 1 +" + +docker ps --format '{{.Names}}' | grep -qx so-postgres \ + || skip "so-postgres is not running; nothing to repair." +docker exec so-postgres psql -U postgres -tAc \ + "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1 \ + || skip "The so_telegraf database does not exist; Telegraf is not writing to Postgres." +psql_tg -tAc "SELECT 1 FROM pg_extension WHERE extname='pg_partman'" | grep -q 1 \ + || skip "pg_partman is not installed in so_telegraf; nothing to repair." + +parents=$(psql_tg -tAc \ + "SELECT count(*) FROM partman.part_config WHERE parent_table LIKE 'telegraf.%'") +stranded=$(psql_tg -tAc "SELECT coalesce(sum(default_rows), 0) FROM ( $REPORT ) t") +behind=$(psql_tg -tAc \ + "SELECT count(*) FROM ( $REPORT ) t WHERE coalesce(days_ahead, -1) < 1") +low_premake=$(psql_tg -tAc \ + "SELECT count(*) FROM partman.part_config + WHERE parent_table LIKE 'telegraf.%' AND premake < $PREMAKE") + +# The dead launcher is the root cause, and it is what a restart fixes. pg_cron +# registers it as a background worker, so it shows up in pg_stat_activity for as +# long as it is alive; both columns are matched because which one carries the +# name varies with the pg_cron version. +launcher=$(psql_pg -tAc \ + "SELECT count(*) FROM pg_stat_activity + WHERE backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%'") + +# The job itself lives in whichever database pg_cron keeps its metadata in: +# so_telegraf before the postgres state lands, postgres after. +cron_db=$(docker exec so-postgres psql -U postgres -tAc \ + "SELECT current_setting('cron.database_name', true)" | tr -d '[:space:]') +last_run=never +if [[ -n "$cron_db" ]]; then + last_run=$(docker exec so-postgres psql -U postgres -d "$cron_db" -tAc \ + "SELECT coalesce(max(d.start_time)::text, 'never') + FROM cron.job_run_details d JOIN cron.job j USING (jobid) + WHERE j.jobname = '$JOB_NAME'" 2>/dev/null | tr -d '[:space:]' || echo unknown) + [[ -n "$last_run" ]] || last_run=never +fi + +# Only blame the launcher once Telegraf has actually created something to +# maintain; a grid that has never written a metric has nothing to recover. +restart_needed=false +[[ "$launcher" -eq 0 && "$parents" -gt 0 ]] && restart_needed=true + +repair_needed=false +[[ "$stranded" -gt 0 ]] && repair_needed=true +[[ "$behind" -gt 0 ]] && repair_needed=true +[[ "$low_premake" -gt 0 ]] && repair_needed=true +$restart_needed && repair_needed=true + +echo "Telegraf partition status:" +psql_tg -c "$REPORT" +echo "Rows stranded in default partitions: $stranded" +echo "pg_cron metadata database: ${cron_db:-unset}" +echo "pg_cron launcher running: $([[ "$launcher" -gt 0 ]] && echo yes || echo no)" +echo "Last $JOB_NAME run: $last_run" +echo + +if ! $repair_needed; then + echo "Telegraf partitions are healthy. Nothing to do." + exit 0 +fi + +if $CHECK_ONLY; then + echo "Repair is needed:" + [[ "$stranded" -gt 0 ]] && echo " * $stranded row(s) stranded in default partitions" + [[ "$behind" -gt 0 ]] && echo " * $behind parent(s) with no partition for the current window" + [[ "$low_premake" -gt 0 ]] && echo " * $low_premake parent(s) premaking fewer than $PREMAKE days ahead" + $restart_needed && echo " * pg_cron's launcher is dead; maintenance is not running at all" + echo + echo "Re-run without --check to repair." + exit 1 +fi + +if [[ "$stranded" -gt 0 ]] && ! $ASSUME_YES; then + echo "This will permanently discard the $stranded stranded row(s) above." + $restart_needed && ! $NO_RESTART && \ + echo "so-postgres will also be restarted, which briefly interrupts SOC." + [[ -t 0 ]] || { echo "Not a terminal; re-run with --yes to confirm." >&2; exit 2; } + read -r -p "Continue? [y/N] " answer + [[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 0; } +fi + +if [[ "$low_premake" -gt 0 ]]; then + echo "Raising premake to $PREMAKE on $low_premake parent(s)." + # GREATEST so an operator who raised it further keeps their value. + psql_tg -v ON_ERROR_STOP=1 -c \ + "UPDATE partman.part_config SET premake = GREATEST(premake, $PREMAKE) + WHERE parent_table LIKE 'telegraf.%'" +fi + +if [[ "$stranded" -gt 0 ]]; then + echo "Clearing default partitions." + psql_tg -v ON_ERROR_STOP=1 <<'EOSQL' +SELECT format('TRUNCATE TABLE %I.%I', n.nspname, c.relname) +FROM partman.part_config pc +JOIN pg_class p ON p.oid = pc.parent_table::regclass +JOIN pg_inherits i ON i.inhparent = p.oid +JOIN pg_class c ON c.oid = i.inhrelid +JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE pc.parent_table LIKE 'telegraf.%' + AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT' +\gexec +EOSQL +fi + +if $restart_needed; then + if $NO_RESTART; then + echo "WARNING: pg_cron's launcher is dead and --no-restart was given." + echo " Maintenance will not run on its own until so-postgres is restarted." + else + echo "Restarting so-postgres to revive pg_cron's launcher." + docker restart so-postgres >/dev/null + for _ in $(seq 1 60); do + docker exec so-postgres pg_isready -U postgres -q 2>/dev/null && break + sleep 2 + done + docker exec so-postgres pg_isready -U postgres -q \ + || { echo "so-postgres did not come back; check 'docker logs so-postgres'." >&2; exit 1; } + fi +fi + +echo "Running partition maintenance." +# so_admin.telegraf_maintenance() drains defaults before calling partman, but it +# only exists once the current postgres state has been applied. +psql_tg -v ON_ERROR_STOP=1 <<'EOSQL' +SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL + THEN 'true' ELSE 'false' END AS has_proc \gset +\if :has_proc +CALL so_admin.telegraf_maintenance(); +\else +CALL partman.run_maintenance_proc(); +\endif +EOSQL + +echo +echo "Telegraf partition status after repair:" +psql_tg -c "$REPORT" +echo "The $JOB_NAME job runs hourly at :17. Confirm it fired with:" +echo " so-telegraf-repair --check" From 706d46b3954bd323313a96f7a491b8f29e99428a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2026 15:08:14 -0400 Subject: [PATCH 5/7] Trim the comments on the Telegraf partition tooling The rationale for the repair belongs in the commit history, not in a 40-line header on every script. --- salt/manager/tools/sbin/soup | 12 +---- salt/postgres/tools/sbin/so-telegraf-postgres | 9 ++-- salt/postgres/tools/sbin/so-telegraf-repair | 52 +++---------------- 3 files changed, 13 insertions(+), 60 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index ade67ab54..4f98f0ca0 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1014,16 +1014,8 @@ up_to_3.3.0() { } telegraf_repair() { - # Grids upgraded onto an existing /nsm/postgres never ran init-db.sh, so - # so_telegraf did not exist when PostgreSQL started and pg_cron's launcher - # died without retrying. partman maintenance never ran and every metric piled - # up in _default, which then blocks partition creation outright. - # - # The highstate above has already converted the database to the current state, - # but it deliberately does not discard the backlog those grids accumulated. - # --check reports whether this host is one of them; it exits 1 only when there - # is something to repair, so fresh installs and grids that were never affected - # are left alone rather than truncated. + # Only grids whose Telegraf partitions stalled need this; --check exits 1 + # when there is something to repair, so everyone else is left alone. local repair=/usr/sbin/so-telegraf-repair [[ -x "$repair" ]] || return 0 docker ps --format '{{.Names}}' | grep -qx so-postgres || return 0 diff --git a/salt/postgres/tools/sbin/so-telegraf-postgres b/salt/postgres/tools/sbin/so-telegraf-postgres index 537bec6bc..8a85f9443 100644 --- a/salt/postgres/tools/sbin/so-telegraf-postgres +++ b/salt/postgres/tools/sbin/so-telegraf-postgres @@ -14,12 +14,9 @@ set -e # maintenance Drain default partitions and run partman maintenance. # check Report partition health. Non-zero if any parent is unhealthy. # -# Once a default partition holds rows for a day with no child, Postgres cannot -# create that child at all -- attaching it would violate the default's -# constraint -- so maintenance drains defaults before calling partman. -# To recover a grid that is already in that state, use so-telegraf-repair, -# which discards the backlog instead of moving it and revives pg_cron's -# launcher without waiting for this state to be applied. +# A default partition holding rows for a day blocks creating that day's child, +# so maintenance drains defaults before calling partman. Use so-telegraf-repair +# on a grid already stuck in that state. cmd="${1:?subcommand required}" diff --git a/salt/postgres/tools/sbin/so-telegraf-repair b/salt/postgres/tools/sbin/so-telegraf-repair index 48bb744e0..2076ea1a4 100644 --- a/salt/postgres/tools/sbin/so-telegraf-repair +++ b/salt/postgres/tools/sbin/so-telegraf-repair @@ -6,39 +6,9 @@ # Elastic License 2.0. # Put Telegraf metrics storage back in service on a grid where pg_partman -# maintenance stalled. -# -# pg_cron's launcher connects to cron.database_name at postmaster start and is -# registered BGW_NEVER_RESTART. On a host upgraded onto an existing -# /nsm/postgres volume, init-db.sh never ran, so so_telegraf did not exist when -# PostgreSQL started: the launcher died and never retried. partman maintenance -# therefore never ran, partitions stopped being premade after create_parent's -# initial window, and every metric landed in _default. Retention never -# fired either. -# -# That state is self-perpetuating. Once a default partition holds rows for a day -# with no child, Postgres cannot create that child at all -- attaching it would -# violate the default's constraint -- so maintenance aborts on the first parent -# it reaches and fixing the scheduler alone does not recover the grid. -# -# This repairs both halves: -# * raises premake to 7 on existing parents, so a future outage has a week of -# headroom before anything reaches a default partition -# * discards the rows stranded in default partitions -- most are past -# retention already, and repartitioning tens of GB just to delete most of it -# is not worth the WAL -# * restarts so-postgres when pg_cron's launcher is dead, which is what makes -# the hourly maintenance job start firing again -# * runs maintenance once so partitions exist for the current window -# -# Self-contained on purpose: it depends only on what an affected grid already -# has, so it can be handed to an operator whose grid has not picked up the -# current postgres state yet. Nothing here conflicts with that state -- it -# reuses the pg_cron job the running version already registered, and the -# postgres state migrates the job to the postgres database on the next soup. -# -# Fresh installs need none of this; the check below reports them healthy and -# changes nothing. +# maintenance stalled: raises premake, discards the rows stranded in default +# partitions, restarts so-postgres if pg_cron's launcher is dead, and runs +# maintenance once. Healthy grids are reported and left alone. # # Usage: so-telegraf-repair [--check] [--yes] [--no-restart] # --check Report health and change nothing. @@ -78,8 +48,7 @@ skip() { echo "$*"; exit 2; } psql_tg() { docker exec -i so-postgres psql -U postgres -d so_telegraf "$@"; } psql_pg() { docker exec -i so-postgres psql -U postgres -d postgres "$@"; } -# Row counts have to come from dynamic SQL; query_to_xml keeps that in a plain -# query so this needs no helper functions installed in the database. +# query_to_xml so the per-table row counts need no helper function installed. REPORT=" WITH parents AS ( SELECT pc.parent_table, @@ -137,15 +106,12 @@ low_premake=$(psql_tg -tAc \ "SELECT count(*) FROM partman.part_config WHERE parent_table LIKE 'telegraf.%' AND premake < $PREMAKE") -# The dead launcher is the root cause, and it is what a restart fixes. pg_cron -# registers it as a background worker, so it shows up in pg_stat_activity for as -# long as it is alive; both columns are matched because which one carries the -# name varies with the pg_cron version. +# Both columns are matched because which one carries the launcher's name varies +# with the pg_cron version. launcher=$(psql_pg -tAc \ "SELECT count(*) FROM pg_stat_activity WHERE backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%'") -# The job itself lives in whichever database pg_cron keeps its metadata in: # so_telegraf before the postgres state lands, postgres after. cron_db=$(docker exec so-postgres psql -U postgres -tAc \ "SELECT current_setting('cron.database_name', true)" | tr -d '[:space:]') @@ -158,8 +124,7 @@ if [[ -n "$cron_db" ]]; then [[ -n "$last_run" ]] || last_run=never fi -# Only blame the launcher once Telegraf has actually created something to -# maintain; a grid that has never written a metric has nothing to recover. +# A grid that has never written a metric has nothing to recover. restart_needed=false [[ "$launcher" -eq 0 && "$parents" -gt 0 ]] && restart_needed=true @@ -242,8 +207,7 @@ if $restart_needed; then fi echo "Running partition maintenance." -# so_admin.telegraf_maintenance() drains defaults before calling partman, but it -# only exists once the current postgres state has been applied. +# so_admin.telegraf_maintenance() only exists once the postgres state has landed. psql_tg -v ON_ERROR_STOP=1 <<'EOSQL' SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL THEN 'true' ELSE 'false' END AS has_proc \gset From d69234146ebab0a8c8f9a28fddd74161e5552a37 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2026 15:44:45 -0400 Subject: [PATCH 6/7] Let partman premake forward across the gap a stall leaves Retention drops every child once they all age out, partman refuses to drop the last one, and with infinite_time_partitions off it will not premake forward from a child that far in the past. The set is left with one stale partition and no current one, so metrics land right back in the default. Set infinite_time_partitions on telegraf parents, in both the repair script and the retention subcommand, and stop blaming the launcher when pg_cron is not loaded at all. --- salt/postgres/tools/sbin/so-telegraf-postgres | 5 +++- salt/postgres/tools/sbin/so-telegraf-repair | 26 ++++++++++++------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/salt/postgres/tools/sbin/so-telegraf-postgres b/salt/postgres/tools/sbin/so-telegraf-postgres index 8a85f9443..df7d16e65 100644 --- a/salt/postgres/tools/sbin/so-telegraf-postgres +++ b/salt/postgres/tools/sbin/so-telegraf-postgres @@ -234,10 +234,13 @@ EOSQL SELECT CASE WHEN EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_partman') THEN 'true' ELSE 'false' END AS has_partman \gset \if :has_partman +-- infinite_time_partitions so a gap in metrics does not stop partman from +-- premaking forward, which is what leaves everything in the default. UPDATE partman.part_config SET retention = :'retention_days' || ' days', retention_keep_table = false, - premake = 7 + premake = 7, + infinite_time_partitions = true WHERE parent_table LIKE 'telegraf.%'; \endif EOSQL diff --git a/salt/postgres/tools/sbin/so-telegraf-repair b/salt/postgres/tools/sbin/so-telegraf-repair index 2076ea1a4..ddfa5f7d7 100644 --- a/salt/postgres/tools/sbin/so-telegraf-repair +++ b/salt/postgres/tools/sbin/so-telegraf-repair @@ -102,9 +102,12 @@ parents=$(psql_tg -tAc \ stranded=$(psql_tg -tAc "SELECT coalesce(sum(default_rows), 0) FROM ( $REPORT ) t") behind=$(psql_tg -tAc \ "SELECT count(*) FROM ( $REPORT ) t WHERE coalesce(days_ahead, -1) < 1") -low_premake=$(psql_tg -tAc \ +# premake < 7, or infinite_time_partitions off: without the latter partman +# refuses to premake forward across the gap the stall left behind. +misconfigured=$(psql_tg -tAc \ "SELECT count(*) FROM partman.part_config - WHERE parent_table LIKE 'telegraf.%' AND premake < $PREMAKE") + WHERE parent_table LIKE 'telegraf.%' + AND (premake < $PREMAKE OR NOT infinite_time_partitions)") # Both columns are matched because which one carries the launcher's name varies # with the pg_cron version. @@ -124,14 +127,15 @@ if [[ -n "$cron_db" ]]; then [[ -n "$last_run" ]] || last_run=never fi -# A grid that has never written a metric has nothing to recover. +# A grid that has never written a metric has nothing to recover, and an empty +# cron_db means pg_cron is not loaded at all, which no restart fixes. restart_needed=false -[[ "$launcher" -eq 0 && "$parents" -gt 0 ]] && restart_needed=true +[[ "$launcher" -eq 0 && "$parents" -gt 0 && -n "$cron_db" ]] && restart_needed=true repair_needed=false [[ "$stranded" -gt 0 ]] && repair_needed=true [[ "$behind" -gt 0 ]] && repair_needed=true -[[ "$low_premake" -gt 0 ]] && repair_needed=true +[[ "$misconfigured" -gt 0 ]] && repair_needed=true $restart_needed && repair_needed=true echo "Telegraf partition status:" @@ -151,7 +155,7 @@ if $CHECK_ONLY; then echo "Repair is needed:" [[ "$stranded" -gt 0 ]] && echo " * $stranded row(s) stranded in default partitions" [[ "$behind" -gt 0 ]] && echo " * $behind parent(s) with no partition for the current window" - [[ "$low_premake" -gt 0 ]] && echo " * $low_premake parent(s) premaking fewer than $PREMAKE days ahead" + [[ "$misconfigured" -gt 0 ]] && echo " * $misconfigured parent(s) with stale partman settings" $restart_needed && echo " * pg_cron's launcher is dead; maintenance is not running at all" echo echo "Re-run without --check to repair." @@ -167,11 +171,13 @@ if [[ "$stranded" -gt 0 ]] && ! $ASSUME_YES; then [[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 0; } fi -if [[ "$low_premake" -gt 0 ]]; then - echo "Raising premake to $PREMAKE on $low_premake parent(s)." - # GREATEST so an operator who raised it further keeps their value. +if [[ "$misconfigured" -gt 0 ]]; then + echo "Reconciling partman settings on $misconfigured parent(s)." + # GREATEST so an operator who raised premake further keeps their value. psql_tg -v ON_ERROR_STOP=1 -c \ - "UPDATE partman.part_config SET premake = GREATEST(premake, $PREMAKE) + "UPDATE partman.part_config + SET premake = GREATEST(premake, $PREMAKE), + infinite_time_partitions = true WHERE parent_table LIKE 'telegraf.%'" fi From 9ebf93cc262ea4d969e4095707c7c090486497ba Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2026 15:55:07 -0400 Subject: [PATCH 7/7] Empty a default and create its partition in one transaction Telegraf never stops writing. Clearing 50 defaults with separate TRUNCATEs left the earliest ones refilled by the time maintenance tried to attach today's child, which then failed on the default's constraint and aborted the whole run. Doing both under one transaction makes the concurrent inserts wait and land in the new partition. --- salt/postgres/tools/sbin/so-telegraf-repair | 32 +++++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/salt/postgres/tools/sbin/so-telegraf-repair b/salt/postgres/tools/sbin/so-telegraf-repair index ddfa5f7d7..75004402e 100644 --- a/salt/postgres/tools/sbin/so-telegraf-repair +++ b/salt/postgres/tools/sbin/so-telegraf-repair @@ -183,16 +183,30 @@ fi if [[ "$stranded" -gt 0 ]]; then echo "Clearing default partitions." + # One transaction: Telegraf is still writing, so a default emptied without + # its partition in place is refilled before maintenance can attach one. psql_tg -v ON_ERROR_STOP=1 <<'EOSQL' -SELECT format('TRUNCATE TABLE %I.%I', n.nspname, c.relname) -FROM partman.part_config pc -JOIN pg_class p ON p.oid = pc.parent_table::regclass -JOIN pg_inherits i ON i.inhparent = p.oid -JOIN pg_class c ON c.oid = i.inhrelid -JOIN pg_namespace n ON n.oid = c.relnamespace -WHERE pc.parent_table LIKE 'telegraf.%' - AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT' -\gexec +DO $$ +DECLARE + r record; +BEGIN + FOR r IN + SELECT pc.parent_table, + format('%I.%I', n.nspname, c.relname) AS default_table + FROM partman.part_config pc + JOIN pg_class p ON p.oid = pc.parent_table::regclass + JOIN pg_inherits i ON i.inhparent = p.oid + JOIN pg_class c ON c.oid = i.inhrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE pc.parent_table LIKE 'telegraf.%' + AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT' + LOOP + EXECUTE format('TRUNCATE TABLE %s', r.default_table); + PERFORM partman.create_partition_time( + r.parent_table, ARRAY[date_trunc('day', now())]::timestamptz[]); + END LOOP; +END +$$; EOSQL fi