From 706d46b3954bd323313a96f7a491b8f29e99428a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2026 15:08:14 -0400 Subject: [PATCH] Trim the comments on the Telegraf partition tooling The rationale for the repair belongs in the commit history, not in a 40-line header on every script. --- salt/manager/tools/sbin/soup | 12 +---- salt/postgres/tools/sbin/so-telegraf-postgres | 9 ++-- salt/postgres/tools/sbin/so-telegraf-repair | 52 +++---------------- 3 files changed, 13 insertions(+), 60 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index ade67ab54..4f98f0ca0 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1014,16 +1014,8 @@ up_to_3.3.0() { } telegraf_repair() { - # Grids upgraded onto an existing /nsm/postgres never ran init-db.sh, so - # so_telegraf did not exist when PostgreSQL started and pg_cron's launcher - # died without retrying. partman maintenance never ran and every metric piled - # up in _default, which then blocks partition creation outright. - # - # The highstate above has already converted the database to the current state, - # but it deliberately does not discard the backlog those grids accumulated. - # --check reports whether this host is one of them; it exits 1 only when there - # is something to repair, so fresh installs and grids that were never affected - # are left alone rather than truncated. + # Only grids whose Telegraf partitions stalled need this; --check exits 1 + # when there is something to repair, so everyone else is left alone. local repair=/usr/sbin/so-telegraf-repair [[ -x "$repair" ]] || return 0 docker ps --format '{{.Names}}' | grep -qx so-postgres || return 0 diff --git a/salt/postgres/tools/sbin/so-telegraf-postgres b/salt/postgres/tools/sbin/so-telegraf-postgres index 537bec6bc..8a85f9443 100644 --- a/salt/postgres/tools/sbin/so-telegraf-postgres +++ b/salt/postgres/tools/sbin/so-telegraf-postgres @@ -14,12 +14,9 @@ set -e # maintenance Drain default partitions and run partman maintenance. # check Report partition health. Non-zero if any parent is unhealthy. # -# Once a default partition holds rows for a day with no child, Postgres cannot -# create that child at all -- attaching it would violate the default's -# constraint -- so maintenance drains defaults before calling partman. -# To recover a grid that is already in that state, use so-telegraf-repair, -# which discards the backlog instead of moving it and revives pg_cron's -# launcher without waiting for this state to be applied. +# A default partition holding rows for a day blocks creating that day's child, +# so maintenance drains defaults before calling partman. Use so-telegraf-repair +# on a grid already stuck in that state. cmd="${1:?subcommand required}" diff --git a/salt/postgres/tools/sbin/so-telegraf-repair b/salt/postgres/tools/sbin/so-telegraf-repair index 48bb744e0..2076ea1a4 100644 --- a/salt/postgres/tools/sbin/so-telegraf-repair +++ b/salt/postgres/tools/sbin/so-telegraf-repair @@ -6,39 +6,9 @@ # Elastic License 2.0. # Put Telegraf metrics storage back in service on a grid where pg_partman -# maintenance stalled. -# -# pg_cron's launcher connects to cron.database_name at postmaster start and is -# registered BGW_NEVER_RESTART. On a host upgraded onto an existing -# /nsm/postgres volume, init-db.sh never ran, so so_telegraf did not exist when -# PostgreSQL started: the launcher died and never retried. partman maintenance -# therefore never ran, partitions stopped being premade after create_parent's -# initial window, and every metric landed in _default. Retention never -# fired either. -# -# That state is self-perpetuating. Once a default partition holds rows for a day -# with no child, Postgres cannot create that child at all -- attaching it would -# violate the default's constraint -- so maintenance aborts on the first parent -# it reaches and fixing the scheduler alone does not recover the grid. -# -# This repairs both halves: -# * raises premake to 7 on existing parents, so a future outage has a week of -# headroom before anything reaches a default partition -# * discards the rows stranded in default partitions -- most are past -# retention already, and repartitioning tens of GB just to delete most of it -# is not worth the WAL -# * restarts so-postgres when pg_cron's launcher is dead, which is what makes -# the hourly maintenance job start firing again -# * runs maintenance once so partitions exist for the current window -# -# Self-contained on purpose: it depends only on what an affected grid already -# has, so it can be handed to an operator whose grid has not picked up the -# current postgres state yet. Nothing here conflicts with that state -- it -# reuses the pg_cron job the running version already registered, and the -# postgres state migrates the job to the postgres database on the next soup. -# -# Fresh installs need none of this; the check below reports them healthy and -# changes nothing. +# maintenance stalled: raises premake, discards the rows stranded in default +# partitions, restarts so-postgres if pg_cron's launcher is dead, and runs +# maintenance once. Healthy grids are reported and left alone. # # Usage: so-telegraf-repair [--check] [--yes] [--no-restart] # --check Report health and change nothing. @@ -78,8 +48,7 @@ skip() { echo "$*"; exit 2; } psql_tg() { docker exec -i so-postgres psql -U postgres -d so_telegraf "$@"; } psql_pg() { docker exec -i so-postgres psql -U postgres -d postgres "$@"; } -# Row counts have to come from dynamic SQL; query_to_xml keeps that in a plain -# query so this needs no helper functions installed in the database. +# query_to_xml so the per-table row counts need no helper function installed. REPORT=" WITH parents AS ( SELECT pc.parent_table, @@ -137,15 +106,12 @@ low_premake=$(psql_tg -tAc \ "SELECT count(*) FROM partman.part_config WHERE parent_table LIKE 'telegraf.%' AND premake < $PREMAKE") -# The dead launcher is the root cause, and it is what a restart fixes. pg_cron -# registers it as a background worker, so it shows up in pg_stat_activity for as -# long as it is alive; both columns are matched because which one carries the -# name varies with the pg_cron version. +# Both columns are matched because which one carries the launcher's name varies +# with the pg_cron version. launcher=$(psql_pg -tAc \ "SELECT count(*) FROM pg_stat_activity WHERE backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%'") -# The job itself lives in whichever database pg_cron keeps its metadata in: # so_telegraf before the postgres state lands, postgres after. cron_db=$(docker exec so-postgres psql -U postgres -tAc \ "SELECT current_setting('cron.database_name', true)" | tr -d '[:space:]') @@ -158,8 +124,7 @@ if [[ -n "$cron_db" ]]; then [[ -n "$last_run" ]] || last_run=never fi -# Only blame the launcher once Telegraf has actually created something to -# maintain; a grid that has never written a metric has nothing to recover. +# A grid that has never written a metric has nothing to recover. restart_needed=false [[ "$launcher" -eq 0 && "$parents" -gt 0 ]] && restart_needed=true @@ -242,8 +207,7 @@ if $restart_needed; then fi echo "Running partition maintenance." -# so_admin.telegraf_maintenance() drains defaults before calling partman, but it -# only exists once the current postgres state has been applied. +# so_admin.telegraf_maintenance() only exists once the postgres state has landed. psql_tg -v ON_ERROR_STOP=1 <<'EOSQL' SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL THEN 'true' ELSE 'false' END AS has_proc \gset