From a2a4d9314d47cdda234fcbdc3661a50dd306b722 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 6 Aug 2026 12:28:40 -0400 Subject: [PATCH] Repair stalled Telegraf partitions from soup instead of every highstate Truncating default partitions is destructive and premaking them is pg_cron's job, so neither belongs in a state that runs on every checkin. Replace the two telegraf_users states with so-telegraf-partition-repair, a standalone tool that reports partition health and clears the backlog, and call it once from soup. The script depends only on pg_partman, so it also runs against a grid that has not yet picked up the new postgres state. It no-ops when nothing is stranded, refuses to discard rows non-interactively without --yes, and reports when the pg_cron job has never fired, which is the underlying cause rather than a symptom the truncate addresses. Hourly self-healing stays with so_admin.telegraf_maintenance() via pg_cron, so a grid that never soups still recovers, just gradually and without discarding in-retention metrics. --- salt/manager/tools/sbin/soup | 15 ++ salt/postgres/telegraf_users.sls | 19 --- .../tools/sbin/so-telegraf-partition-repair | 153 ++++++++++++++++++ salt/postgres/tools/sbin/so-telegraf-postgres | 55 +------ 4 files changed, 176 insertions(+), 66 deletions(-) create mode 100644 salt/postgres/tools/sbin/so-telegraf-partition-repair diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index b155d1bec..b5366facf 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1013,9 +1013,24 @@ up_to_3.3.0() { INSTALLEDVERSION=3.3.0 } +telegraf_partition_repair() { + # Grids upgraded onto an existing /nsm/postgres never ran init-db.sh, so + # so_telegraf did not exist when PostgreSQL started and pg_cron's launcher + # died without retrying. partman maintenance never ran and every metric piled + # up in _default, which then blocks partition creation outright. + # Fresh installs are unaffected. The script no-ops when nothing is stranded. + [[ -x /usr/sbin/so-telegraf-partition-repair ]] || return 0 + docker ps --format '{{.Names}}' | grep -qx so-postgres || return 0 + echo "Checking Telegraf metric partitions." + /usr/sbin/so-telegraf-partition-repair --yes \ + || echo " warning: so-telegraf-partition-repair failed; run it manually" >&2 +} + post_to_3.3.0() { # Recollate again since some internal DBs were excluded during 3.2.0 soup recollate_postgres + + telegraf_partition_repair } ### 3.3.0 End ### diff --git a/salt/postgres/telegraf_users.sls b/salt/postgres/telegraf_users.sls index 3544755a5..4c63d40b0 100644 --- a/salt/postgres/telegraf_users.sls +++ b/salt/postgres/telegraf_users.sls @@ -70,25 +70,6 @@ postgres_telegraf_retention_reconcile: - cmd: postgres_telegraf_group_role - file: postgres_sbin -# One-time recovery for grids whose pg_cron job never ran. Truncating the -# defaults is destructive, so the watermark keeps it to one pass per host. -postgres_telegraf_partman_default_repair: - cmd.run: - - name: mkdir -p /opt/so/state && /usr/sbin/so-telegraf-postgres repair && touch /opt/so/state/telegraf_partman_default_repair - - creates: /opt/so/state/telegraf_partman_default_repair - - require: - - cmd: postgres_telegraf_retention_reconcile - - file: postgres_sbin - -# Also run from the state, not just hourly from pg_cron, so a grid whose -# pg_cron worker is dead still recovers on its own. -postgres_telegraf_partman_maintenance: - cmd.run: - - name: /usr/sbin/so-telegraf-postgres maintenance - - require: - - cmd: postgres_telegraf_partman_default_repair - - file: postgres_sbin - {% endif %} {% else %} diff --git a/salt/postgres/tools/sbin/so-telegraf-partition-repair b/salt/postgres/tools/sbin/so-telegraf-partition-repair new file mode 100644 index 000000000..e6c83a434 --- /dev/null +++ b/salt/postgres/tools/sbin/so-telegraf-partition-repair @@ -0,0 +1,153 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Repair Telegraf metric partitions on a grid where pg_partman maintenance +# stalled and metrics piled up in _default. +# +# Once a default partition holds rows for a day with no child partition, +# Postgres cannot create that child at all -- attaching it would violate the +# default's constraint -- so maintenance stays broken until the default is +# emptied. This discards the stranded rows rather than repartitioning them: +# most are already past retention, and moving tens of GB just to drop them is +# not worth the WAL. +# +# Self-contained on purpose: it depends only on pg_partman, so it runs against +# a grid that has not yet picked up the current postgres state. +# +# Usage: so-telegraf-partition-repair [--dry-run] [--yes] +# --dry-run Report only; change nothing. +# --yes Skip the confirmation prompt (for soup and other automation). + +set -e + +DRY_RUN=false +ASSUME_YES=false + +usage() { + sed -n '8,24p' "$0" | sed 's/^# \?//' +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) DRY_RUN=true ;; + --yes|-y) ASSUME_YES=true ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage >&2; exit 1 ;; + esac + shift +done + +fail() { echo "ERROR: $*" >&2; exit 1; } + +psql_tg() { docker exec -i so-postgres psql -U postgres -d so_telegraf "$@"; } + +# Row counts have to come from dynamic SQL; query_to_xml keeps that in a plain +# query so this needs no helper functions installed in the database. +REPORT=" +WITH parents AS ( + SELECT pc.parent_table, + split_part(pc.parent_table, '.', 1) AS sch, + split_part(pc.parent_table, '.', 2) AS tbl + FROM partman.part_config pc + WHERE pc.parent_table LIKE 'telegraf.%' +), children AS ( + SELECT p.parent_table, + max(to_date(substring(c.relname FROM '_p(\d{8})\$'), 'YYYYMMDD')) AS newest_child + FROM parents p + JOIN pg_class pt ON pt.oid = p.parent_table::regclass + JOIN pg_inherits i ON i.inhparent = pt.oid + JOIN pg_class c ON c.oid = i.inhrelid + WHERE pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT' + GROUP BY p.parent_table +), defaults AS ( + SELECT p.parent_table, + format('%I.%I', p.sch, p.tbl || '_default') AS default_table, + to_regclass(format('%I.%I', p.sch, p.tbl || '_default')) AS default_oid + FROM parents p +) +SELECT d.parent_table, + c.newest_child, + (c.newest_child - current_date) AS days_ahead, + CASE WHEN d.default_oid IS NULL THEN NULL ELSE + (xpath('/row/cnt/text()', + query_to_xml(format('SELECT count(*) AS cnt FROM %s', d.default_table), + false, true, '')))[1]::text::bigint + END AS default_rows, + CASE WHEN d.default_oid IS NULL THEN NULL + ELSE pg_size_pretty(pg_total_relation_size(d.default_oid)) END AS default_size +FROM defaults d +LEFT JOIN children c ON c.parent_table = d.parent_table +ORDER BY 1 +" + +docker ps --format '{{.Names}}' | grep -qx so-postgres \ + || fail "so-postgres is not running." +docker exec so-postgres psql -U postgres -tAc \ + "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1 \ + || fail "The so_telegraf database does not exist; Telegraf is not writing to Postgres." +psql_tg -tAc "SELECT 1 FROM pg_extension WHERE extname='pg_partman'" | grep -q 1 \ + || fail "pg_partman is not installed in so_telegraf." + +echo "Telegraf partition status:" +psql_tg -c "$REPORT" + +stranded=$(psql_tg -tAc "SELECT coalesce(sum(default_rows), 0) FROM ( $REPORT ) t") +echo "Rows stranded in default partitions: $stranded" + +# Report the scheduler too. A grid that needed this script almost always has a +# pg_cron job that has never fired, and repairing partitions without fixing that +# just delays the next occurrence. +cron_db=$(docker exec so-postgres psql -U postgres -tAc \ + "SELECT current_setting('cron.database_name', true)" | tr -d '[:space:]') +if [[ -n "$cron_db" ]]; then + runs=$(docker exec so-postgres psql -U postgres -d "$cron_db" -tAc \ + "SELECT count(*) FROM cron.job_run_details d JOIN cron.job j USING (jobid) + WHERE j.jobname = 'telegraf-partman-maintenance'" 2>/dev/null | tr -d '[:space:]' || true) + if [[ "$runs" == "0" ]]; then + echo + echo "WARNING: the telegraf-partman-maintenance job has never run. Apply the" + echo " postgres state so pg_cron is pointed at a database that exists" + echo " at server start, or this will recur." + fi +fi + +if [[ "$stranded" == "0" ]]; then + echo "Nothing stranded. Running maintenance to premake the current window." + $DRY_RUN && { echo "(dry run: skipping maintenance)"; exit 0; } + psql_tg -v ON_ERROR_STOP=1 -c "CALL partman.run_maintenance_proc()" + exit 0 +fi + +if $DRY_RUN; then + echo "(dry run: would TRUNCATE the default partitions listed above)" + exit 0 +fi + +if ! $ASSUME_YES; then + echo + echo "This will permanently discard the $stranded stranded row(s) above." + [[ -t 0 ]] || fail "Not a terminal; re-run with --yes to confirm." + read -r -p "Continue? [y/N] " answer + [[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 1; } +fi + +psql_tg -v ON_ERROR_STOP=1 <<'EOSQL' +SELECT format('TRUNCATE TABLE %I.%I', n.nspname, c.relname) +FROM partman.part_config pc +JOIN pg_class p ON p.oid = pc.parent_table::regclass +JOIN pg_inherits i ON i.inhparent = p.oid +JOIN pg_class c ON c.oid = i.inhrelid +JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE pc.parent_table LIKE 'telegraf.%' + AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT' +\gexec +CALL partman.run_maintenance_proc(); +EOSQL + +echo +echo "Telegraf partition status after repair:" +psql_tg -c "$REPORT" diff --git a/salt/postgres/tools/sbin/so-telegraf-postgres b/salt/postgres/tools/sbin/so-telegraf-postgres index fe31fb949..eb096eb65 100644 --- a/salt/postgres/tools/sbin/so-telegraf-postgres +++ b/salt/postgres/tools/sbin/so-telegraf-postgres @@ -12,28 +12,16 @@ set -e # retention Reconcile partman retention and premake on telegraf parents. # Env: RETENTION_DAYS. # maintenance Drain default partitions and run partman maintenance. -# repair One-time recovery: TRUNCATE non-empty defaults, then maintain. # check Report partition health. Non-zero if any parent is unhealthy. # # Once a default partition holds rows for a day with no child, Postgres cannot # create that child at all -- attaching it would violate the default's # constraint -- so maintenance drains defaults before calling partman. +# To recover a grid that is already in that state, use +# so-telegraf-partition-repair, which discards the backlog instead of moving it. cmd="${1:?subcommand required}" -# A function rather than re-invoking $0, which assumes the script is executable. -run_maintenance() { - docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL' -SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL - THEN 'true' ELSE 'false' END AS has_proc \gset -\if :has_proc -CALL so_admin.telegraf_maintenance(); -\else -\echo 'so_admin.telegraf_maintenance() is missing; run so-telegraf-postgres group_role first.' -\endif -EOSQL -} - case "$cmd" in create_db) if ! docker exec so-postgres psql -U postgres -tAc \ @@ -258,42 +246,15 @@ EOSQL ;; maintenance) - run_maintenance - ;; - - repair) - # Stranded rows are discarded, not repartitioned: most are already past - # retention, and moving tens of GB just to drop them isn't worth the WAL. docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL' -SELECT CASE WHEN EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_partman') - THEN 'true' ELSE 'false' END AS has_partman \gset -\if :has_partman -\echo 'Default partitions holding stranded Telegraf metrics:' -SELECT format('%I.%I', n.nspname, c.relname) AS default_partition, - pg_size_pretty(pg_total_relation_size(c.oid)) AS size -FROM partman.part_config pc -JOIN pg_class p ON p.oid = pc.parent_table::regclass -JOIN pg_inherits i ON i.inhparent = p.oid -JOIN pg_class c ON c.oid = i.inhrelid -JOIN pg_namespace n ON n.oid = c.relnamespace -WHERE pc.parent_table LIKE 'telegraf.%' - AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT' - AND pg_total_relation_size(c.oid) > 0 -ORDER BY pg_total_relation_size(c.oid) DESC; - -SELECT format('TRUNCATE TABLE %I.%I', n.nspname, c.relname) -FROM partman.part_config pc -JOIN pg_class p ON p.oid = pc.parent_table::regclass -JOIN pg_inherits i ON i.inhparent = p.oid -JOIN pg_class c ON c.oid = i.inhrelid -JOIN pg_namespace n ON n.oid = c.relnamespace -WHERE pc.parent_table LIKE 'telegraf.%' - AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT' -\gexec +SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL + THEN 'true' ELSE 'false' END AS has_proc \gset +\if :has_proc +CALL so_admin.telegraf_maintenance(); +\else +\echo 'so_admin.telegraf_maintenance() is missing; run so-telegraf-postgres group_role first.' \endif EOSQL - # Premake the window so the next flush has somewhere to land. - run_maintenance ;; check)