mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-08-24 08:28:21 +02:00
Repair stalled Telegraf partitions from soup instead of every highstate
Truncating default partitions is destructive and premaking them is pg_cron's job, so neither belongs in a state that runs on every checkin. Replace the two telegraf_users states with so-telegraf-partition-repair, a standalone tool that reports partition health and clears the backlog, and call it once from soup. The script depends only on pg_partman, so it also runs against a grid that has not yet picked up the new postgres state. It no-ops when nothing is stranded, refuses to discard rows non-interactively without --yes, and reports when the pg_cron job has never fired, which is the underlying cause rather than a symptom the truncate addresses. Hourly self-healing stays with so_admin.telegraf_maintenance() via pg_cron, so a grid that never soups still recovers, just gradually and without discarding in-retention metrics.
This commit is contained in:
Mike Reeves
@@ -1013,9 +1013,24 @@ up_to_3.3.0() {
|
||||
INSTALLEDVERSION=3.3.0
|
||||
}
|
||||
|
||||
telegraf_partition_repair() {
|
||||
# Grids upgraded onto an existing /nsm/postgres never ran init-db.sh, so
|
||||
# so_telegraf did not exist when PostgreSQL started and pg_cron's launcher
|
||||
# died without retrying. partman maintenance never ran and every metric piled
|
||||
# up in <parent>_default, which then blocks partition creation outright.
|
||||
# Fresh installs are unaffected. The script no-ops when nothing is stranded.
|
||||
[[ -x /usr/sbin/so-telegraf-partition-repair ]] || return 0
|
||||
docker ps --format '{{.Names}}' | grep -qx so-postgres || return 0
|
||||
echo "Checking Telegraf metric partitions."
|
||||
/usr/sbin/so-telegraf-partition-repair --yes \
|
||||
|| echo " warning: so-telegraf-partition-repair failed; run it manually" >&2
|
||||
}
|
||||
|
||||
post_to_3.3.0() {
|
||||
# Recollate again since some internal DBs were excluded during 3.2.0 soup
|
||||
recollate_postgres
|
||||
|
||||
telegraf_partition_repair
|
||||
}
|
||||
### 3.3.0 End ###
|
||||
|
||||
|
||||
@@ -70,25 +70,6 @@ postgres_telegraf_retention_reconcile:
|
||||
- cmd: postgres_telegraf_group_role
|
||||
- file: postgres_sbin
|
||||
|
||||
# One-time recovery for grids whose pg_cron job never ran. Truncating the
|
||||
# defaults is destructive, so the watermark keeps it to one pass per host.
|
||||
postgres_telegraf_partman_default_repair:
|
||||
cmd.run:
|
||||
- name: mkdir -p /opt/so/state && /usr/sbin/so-telegraf-postgres repair && touch /opt/so/state/telegraf_partman_default_repair
|
||||
- creates: /opt/so/state/telegraf_partman_default_repair
|
||||
- require:
|
||||
- cmd: postgres_telegraf_retention_reconcile
|
||||
- file: postgres_sbin
|
||||
|
||||
# Also run from the state, not just hourly from pg_cron, so a grid whose
|
||||
# pg_cron worker is dead still recovers on its own.
|
||||
postgres_telegraf_partman_maintenance:
|
||||
cmd.run:
|
||||
- name: /usr/sbin/so-telegraf-postgres maintenance
|
||||
- require:
|
||||
- cmd: postgres_telegraf_partman_default_repair
|
||||
- file: postgres_sbin
|
||||
|
||||
{% endif %}
|
||||
|
||||
{% else %}
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
# Repair Telegraf metric partitions on a grid where pg_partman maintenance
|
||||
# stalled and metrics piled up in <parent>_default.
|
||||
#
|
||||
# Once a default partition holds rows for a day with no child partition,
|
||||
# Postgres cannot create that child at all -- attaching it would violate the
|
||||
# default's constraint -- so maintenance stays broken until the default is
|
||||
# emptied. This discards the stranded rows rather than repartitioning them:
|
||||
# most are already past retention, and moving tens of GB just to drop them is
|
||||
# not worth the WAL.
|
||||
#
|
||||
# Self-contained on purpose: it depends only on pg_partman, so it runs against
|
||||
# a grid that has not yet picked up the current postgres state.
|
||||
#
|
||||
# Usage: so-telegraf-partition-repair [--dry-run] [--yes]
|
||||
# --dry-run Report only; change nothing.
|
||||
# --yes Skip the confirmation prompt (for soup and other automation).
|
||||
|
||||
set -e
|
||||
|
||||
DRY_RUN=false
|
||||
ASSUME_YES=false
|
||||
|
||||
usage() {
|
||||
sed -n '8,24p' "$0" | sed 's/^# \?//'
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--dry-run) DRY_RUN=true ;;
|
||||
--yes|-y) ASSUME_YES=true ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) echo "Unknown option: $1" >&2; usage >&2; exit 1 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
fail() { echo "ERROR: $*" >&2; exit 1; }
|
||||
|
||||
psql_tg() { docker exec -i so-postgres psql -U postgres -d so_telegraf "$@"; }
|
||||
|
||||
# Row counts have to come from dynamic SQL; query_to_xml keeps that in a plain
|
||||
# query so this needs no helper functions installed in the database.
|
||||
REPORT="
|
||||
WITH parents AS (
|
||||
SELECT pc.parent_table,
|
||||
split_part(pc.parent_table, '.', 1) AS sch,
|
||||
split_part(pc.parent_table, '.', 2) AS tbl
|
||||
FROM partman.part_config pc
|
||||
WHERE pc.parent_table LIKE 'telegraf.%'
|
||||
), children AS (
|
||||
SELECT p.parent_table,
|
||||
max(to_date(substring(c.relname FROM '_p(\d{8})\$'), 'YYYYMMDD')) AS newest_child
|
||||
FROM parents p
|
||||
JOIN pg_class pt ON pt.oid = p.parent_table::regclass
|
||||
JOIN pg_inherits i ON i.inhparent = pt.oid
|
||||
JOIN pg_class c ON c.oid = i.inhrelid
|
||||
WHERE pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT'
|
||||
GROUP BY p.parent_table
|
||||
), defaults AS (
|
||||
SELECT p.parent_table,
|
||||
format('%I.%I', p.sch, p.tbl || '_default') AS default_table,
|
||||
to_regclass(format('%I.%I', p.sch, p.tbl || '_default')) AS default_oid
|
||||
FROM parents p
|
||||
)
|
||||
SELECT d.parent_table,
|
||||
c.newest_child,
|
||||
(c.newest_child - current_date) AS days_ahead,
|
||||
CASE WHEN d.default_oid IS NULL THEN NULL ELSE
|
||||
(xpath('/row/cnt/text()',
|
||||
query_to_xml(format('SELECT count(*) AS cnt FROM %s', d.default_table),
|
||||
false, true, '')))[1]::text::bigint
|
||||
END AS default_rows,
|
||||
CASE WHEN d.default_oid IS NULL THEN NULL
|
||||
ELSE pg_size_pretty(pg_total_relation_size(d.default_oid)) END AS default_size
|
||||
FROM defaults d
|
||||
LEFT JOIN children c ON c.parent_table = d.parent_table
|
||||
ORDER BY 1
|
||||
"
|
||||
|
||||
docker ps --format '{{.Names}}' | grep -qx so-postgres \
|
||||
|| fail "so-postgres is not running."
|
||||
docker exec so-postgres psql -U postgres -tAc \
|
||||
"SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1 \
|
||||
|| fail "The so_telegraf database does not exist; Telegraf is not writing to Postgres."
|
||||
psql_tg -tAc "SELECT 1 FROM pg_extension WHERE extname='pg_partman'" | grep -q 1 \
|
||||
|| fail "pg_partman is not installed in so_telegraf."
|
||||
|
||||
echo "Telegraf partition status:"
|
||||
psql_tg -c "$REPORT"
|
||||
|
||||
stranded=$(psql_tg -tAc "SELECT coalesce(sum(default_rows), 0) FROM ( $REPORT ) t")
|
||||
echo "Rows stranded in default partitions: $stranded"
|
||||
|
||||
# Report the scheduler too. A grid that needed this script almost always has a
|
||||
# pg_cron job that has never fired, and repairing partitions without fixing that
|
||||
# just delays the next occurrence.
|
||||
cron_db=$(docker exec so-postgres psql -U postgres -tAc \
|
||||
"SELECT current_setting('cron.database_name', true)" | tr -d '[:space:]')
|
||||
if [[ -n "$cron_db" ]]; then
|
||||
runs=$(docker exec so-postgres psql -U postgres -d "$cron_db" -tAc \
|
||||
"SELECT count(*) FROM cron.job_run_details d JOIN cron.job j USING (jobid)
|
||||
WHERE j.jobname = 'telegraf-partman-maintenance'" 2>/dev/null | tr -d '[:space:]' || true)
|
||||
if [[ "$runs" == "0" ]]; then
|
||||
echo
|
||||
echo "WARNING: the telegraf-partman-maintenance job has never run. Apply the"
|
||||
echo " postgres state so pg_cron is pointed at a database that exists"
|
||||
echo " at server start, or this will recur."
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$stranded" == "0" ]]; then
|
||||
echo "Nothing stranded. Running maintenance to premake the current window."
|
||||
$DRY_RUN && { echo "(dry run: skipping maintenance)"; exit 0; }
|
||||
psql_tg -v ON_ERROR_STOP=1 -c "CALL partman.run_maintenance_proc()"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if $DRY_RUN; then
|
||||
echo "(dry run: would TRUNCATE the default partitions listed above)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ! $ASSUME_YES; then
|
||||
echo
|
||||
echo "This will permanently discard the $stranded stranded row(s) above."
|
||||
[[ -t 0 ]] || fail "Not a terminal; re-run with --yes to confirm."
|
||||
read -r -p "Continue? [y/N] " answer
|
||||
[[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 1; }
|
||||
fi
|
||||
|
||||
psql_tg -v ON_ERROR_STOP=1 <<'EOSQL'
|
||||
SELECT format('TRUNCATE TABLE %I.%I', n.nspname, c.relname)
|
||||
FROM partman.part_config pc
|
||||
JOIN pg_class p ON p.oid = pc.parent_table::regclass
|
||||
JOIN pg_inherits i ON i.inhparent = p.oid
|
||||
JOIN pg_class c ON c.oid = i.inhrelid
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE pc.parent_table LIKE 'telegraf.%'
|
||||
AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT'
|
||||
\gexec
|
||||
CALL partman.run_maintenance_proc();
|
||||
EOSQL
|
||||
|
||||
echo
|
||||
echo "Telegraf partition status after repair:"
|
||||
psql_tg -c "$REPORT"
|
||||
@@ -12,28 +12,16 @@ set -e
|
||||
# retention Reconcile partman retention and premake on telegraf parents.
|
||||
# Env: RETENTION_DAYS.
|
||||
# maintenance Drain default partitions and run partman maintenance.
|
||||
# repair One-time recovery: TRUNCATE non-empty defaults, then maintain.
|
||||
# check Report partition health. Non-zero if any parent is unhealthy.
|
||||
#
|
||||
# Once a default partition holds rows for a day with no child, Postgres cannot
|
||||
# create that child at all -- attaching it would violate the default's
|
||||
# constraint -- so maintenance drains defaults before calling partman.
|
||||
# To recover a grid that is already in that state, use
|
||||
# so-telegraf-partition-repair, which discards the backlog instead of moving it.
|
||||
|
||||
cmd="${1:?subcommand required}"
|
||||
|
||||
# A function rather than re-invoking $0, which assumes the script is executable.
|
||||
run_maintenance() {
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
|
||||
SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL
|
||||
THEN 'true' ELSE 'false' END AS has_proc \gset
|
||||
\if :has_proc
|
||||
CALL so_admin.telegraf_maintenance();
|
||||
\else
|
||||
\echo 'so_admin.telegraf_maintenance() is missing; run so-telegraf-postgres group_role first.'
|
||||
\endif
|
||||
EOSQL
|
||||
}
|
||||
|
||||
case "$cmd" in
|
||||
create_db)
|
||||
if ! docker exec so-postgres psql -U postgres -tAc \
|
||||
@@ -258,42 +246,15 @@ EOSQL
|
||||
;;
|
||||
|
||||
maintenance)
|
||||
run_maintenance
|
||||
;;
|
||||
|
||||
repair)
|
||||
# Stranded rows are discarded, not repartitioned: most are already past
|
||||
# retention, and moving tens of GB just to drop them isn't worth the WAL.
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
|
||||
SELECT CASE WHEN EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_partman')
|
||||
THEN 'true' ELSE 'false' END AS has_partman \gset
|
||||
\if :has_partman
|
||||
\echo 'Default partitions holding stranded Telegraf metrics:'
|
||||
SELECT format('%I.%I', n.nspname, c.relname) AS default_partition,
|
||||
pg_size_pretty(pg_total_relation_size(c.oid)) AS size
|
||||
FROM partman.part_config pc
|
||||
JOIN pg_class p ON p.oid = pc.parent_table::regclass
|
||||
JOIN pg_inherits i ON i.inhparent = p.oid
|
||||
JOIN pg_class c ON c.oid = i.inhrelid
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE pc.parent_table LIKE 'telegraf.%'
|
||||
AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT'
|
||||
AND pg_total_relation_size(c.oid) > 0
|
||||
ORDER BY pg_total_relation_size(c.oid) DESC;
|
||||
|
||||
SELECT format('TRUNCATE TABLE %I.%I', n.nspname, c.relname)
|
||||
FROM partman.part_config pc
|
||||
JOIN pg_class p ON p.oid = pc.parent_table::regclass
|
||||
JOIN pg_inherits i ON i.inhparent = p.oid
|
||||
JOIN pg_class c ON c.oid = i.inhrelid
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE pc.parent_table LIKE 'telegraf.%'
|
||||
AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT'
|
||||
\gexec
|
||||
SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL
|
||||
THEN 'true' ELSE 'false' END AS has_proc \gset
|
||||
\if :has_proc
|
||||
CALL so_admin.telegraf_maintenance();
|
||||
\else
|
||||
\echo 'so_admin.telegraf_maintenance() is missing; run so-telegraf-postgres group_role first.'
|
||||
\endif
|
||||
EOSQL
|
||||
# Premake the window so the next flush has somewhere to land.
|
||||
run_maintenance
|
||||
;;
|
||||
|
||||
check)
|
||||
|
||||
Reference in New Issue
Block a user