mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-08-27 09:58:29 +02:00
Merge pull request #16143 from Security-Onion-Solutions/TOoSmOotH/telegraf-partman-maintenance
Fix Telegraf metrics falling into pg_partman default partitions
This commit is contained in:
Mike Reeves
@@ -1013,9 +1013,30 @@ up_to_3.3.0() {
|
||||
INSTALLEDVERSION=3.3.0
|
||||
}
|
||||
|
||||
telegraf_repair() {
|
||||
# Only grids whose Telegraf partitions stalled need this; --check exits 1
|
||||
# when there is something to repair, so everyone else is left alone.
|
||||
local repair=/usr/sbin/so-telegraf-repair
|
||||
[[ -x "$repair" ]] || return 0
|
||||
docker ps --format '{{.Names}}' | grep -qx so-postgres || return 0
|
||||
|
||||
echo "Checking Telegraf metric partitions."
|
||||
local status=0
|
||||
"$repair" --check >> "$SOUP_LOG" 2>&1 || status=$?
|
||||
case "$status" in
|
||||
0) echo " Telegraf partitions are healthy; nothing to repair." ;;
|
||||
1) echo " Repairing stalled Telegraf partitions."
|
||||
"$repair" --yes \
|
||||
|| echo " warning: so-telegraf-repair failed; run it manually" >&2 ;;
|
||||
*) echo " Skipping; Telegraf is not storing metrics in Postgres on this host." ;;
|
||||
esac
|
||||
}
|
||||
|
||||
post_to_3.3.0() {
|
||||
# Recollate again since some internal DBs were excluded during 3.2.0 soup
|
||||
recollate_postgres
|
||||
|
||||
telegraf_repair
|
||||
}
|
||||
### 3.3.0 End ###
|
||||
|
||||
|
||||
@@ -16,4 +16,4 @@ postgres:
|
||||
logging_collector: 'off'
|
||||
log_min_messages: 'warning'
|
||||
shared_preload_libraries: pg_cron
|
||||
cron.database_name: so_telegraf
|
||||
cron.database_name: postgres
|
||||
|
||||
@@ -83,7 +83,7 @@ postgres:
|
||||
advanced: True
|
||||
helpLink: postgres
|
||||
cron.database_name:
|
||||
description: Database pg_cron schedules jobs in. Must be so_telegraf so partman maintenance runs in the right database context.
|
||||
description: Database pg_cron keeps its job metadata in. Must already exist when PostgreSQL starts, because pg_cron's launcher connects to it at startup and never retries if it is missing. The maintenance job itself targets so_telegraf.
|
||||
global: True
|
||||
advanced: True
|
||||
helpLink: postgres
|
||||
|
||||
@@ -47,7 +47,12 @@ trap 'rm -f "$TMPFILE"' EXIT
|
||||
|
||||
# Dump all databases and roles, compress. Write to a temp file so the final
|
||||
# filename only ever appears for a complete, verified backup.
|
||||
if ! docker exec so-postgres pg_dumpall -U postgres | gzip > "$TMPFILE"; then
|
||||
#
|
||||
# so_telegraf is excluded: it is transient metrics on a short retention window,
|
||||
# it dominates the dump size, and it is rebuilt automatically after a restore --
|
||||
# init-db.sh recreates the database and Telegraf recreates its tables on first
|
||||
# write. Roles are globals, so the per-minion telegraf logins are still dumped.
|
||||
if ! docker exec so-postgres pg_dumpall -U postgres --exclude-database=so_telegraf | gzip > "$TMPFILE"; then
|
||||
log "ERROR: pg_dumpall/gzip failed; backup aborted"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -5,11 +5,18 @@ set -e
|
||||
# Usage: so-telegraf-postgres <subcommand>
|
||||
# create_db Ensure the so_telegraf database exists.
|
||||
# group_role Provision the so_telegraf group role, telegraf/partman schemas,
|
||||
# pg_partman, pg_cron, and the hourly partman maintenance job.
|
||||
# pg_partman, the so_admin maintenance routines, and the hourly
|
||||
# pg_cron maintenance job.
|
||||
# user Create or update a per-minion login role granted to so_telegraf.
|
||||
# Env: ROLE_USER, ROLE_PASS.
|
||||
# retention Reconcile partman retention on telegraf parents.
|
||||
# retention Reconcile partman retention and premake on telegraf parents.
|
||||
# Env: RETENTION_DAYS.
|
||||
# maintenance Drain default partitions and run partman maintenance.
|
||||
# check Report partition health. Non-zero if any parent is unhealthy.
|
||||
#
|
||||
# A default partition holding rows for a day blocks creating that day's child,
|
||||
# so maintenance drains defaults before calling partman. Use so-telegraf-repair
|
||||
# on a grid already stuck in that state.
|
||||
|
||||
cmd="${1:?subcommand required}"
|
||||
|
||||
@@ -36,7 +43,6 @@ CREATE SCHEMA IF NOT EXISTS telegraf AUTHORIZATION so_telegraf;
|
||||
GRANT USAGE, CREATE ON SCHEMA telegraf TO so_telegraf;
|
||||
CREATE SCHEMA IF NOT EXISTS partman;
|
||||
CREATE EXTENSION IF NOT EXISTS pg_partman SCHEMA partman;
|
||||
CREATE EXTENSION IF NOT EXISTS pg_cron;
|
||||
-- Telegraf (running as so_telegraf) calls partman.create_parent()
|
||||
-- on first write of each metric, which needs USAGE on the partman
|
||||
-- schema, EXECUTE on its functions/procedures, and write access to
|
||||
@@ -51,12 +57,141 @@ ALTER DEFAULT PRIVILEGES IN SCHEMA partman
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO so_telegraf;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA partman
|
||||
GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO so_telegraf;
|
||||
-- Hourly partman maintenance. cron.schedule is idempotent by jobname.
|
||||
SELECT cron.schedule(
|
||||
|
||||
-- pg_cron runs these as postgres, so they must not sit in a schema any
|
||||
-- Telegraf role can create objects in.
|
||||
CREATE SCHEMA IF NOT EXISTS so_admin AUTHORIZATION postgres;
|
||||
REVOKE ALL ON SCHEMA so_admin FROM PUBLIC;
|
||||
|
||||
CREATE OR REPLACE PROCEDURE so_admin.telegraf_maintenance()
|
||||
LANGUAGE plpgsql
|
||||
AS $proc$
|
||||
DECLARE
|
||||
r record;
|
||||
v_default text;
|
||||
v_rows bigint;
|
||||
BEGIN
|
||||
-- No per-parent EXCEPTION handler: partition_data_proc commits internally,
|
||||
-- and COMMIT is illegal while a subtransaction is active. A failing parent
|
||||
-- aborts the run and the next pass retries.
|
||||
FOR r IN
|
||||
SELECT parent_table, retention
|
||||
FROM partman.part_config
|
||||
WHERE parent_table LIKE 'telegraf.%'
|
||||
ORDER BY parent_table
|
||||
LOOP
|
||||
v_default := format('%I.%I',
|
||||
split_part(r.parent_table, '.', 1),
|
||||
split_part(r.parent_table, '.', 2) || '_default');
|
||||
|
||||
CONTINUE WHEN to_regclass(v_default) IS NULL;
|
||||
|
||||
EXECUTE format('SELECT count(*) FROM %s', v_default) INTO v_rows;
|
||||
CONTINUE WHEN v_rows = 0;
|
||||
|
||||
RAISE WARNING 'so_admin.telegraf_maintenance: % rows stranded in %, draining',
|
||||
v_rows, v_default;
|
||||
|
||||
-- Cheaper to delete expired rows than to repartition and then drop them.
|
||||
IF r.retention IS NOT NULL THEN
|
||||
EXECUTE format('DELETE FROM %s WHERE "time" < now() - %L::interval',
|
||||
v_default, r.retention);
|
||||
COMMIT;
|
||||
END IF;
|
||||
|
||||
-- Bounded so a large backlog drains across several runs.
|
||||
CALL partman.partition_data_proc(
|
||||
p_parent_table := r.parent_table,
|
||||
p_loop_count := 200,
|
||||
p_source_table := v_default
|
||||
);
|
||||
COMMIT;
|
||||
END LOOP;
|
||||
|
||||
CALL partman.run_maintenance_proc();
|
||||
END;
|
||||
$proc$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION so_admin.telegraf_partition_status()
|
||||
RETURNS TABLE (
|
||||
parent_table text,
|
||||
oldest_child date,
|
||||
newest_child date,
|
||||
days_ahead int,
|
||||
retention text,
|
||||
default_rows bigint,
|
||||
default_size text
|
||||
)
|
||||
LANGUAGE plpgsql
|
||||
AS $func$
|
||||
DECLARE
|
||||
r record;
|
||||
v_default regclass;
|
||||
BEGIN
|
||||
FOR r IN
|
||||
SELECT pc.parent_table AS pt, pc.retention AS ret
|
||||
FROM partman.part_config pc
|
||||
WHERE pc.parent_table LIKE 'telegraf.%'
|
||||
ORDER BY pc.parent_table
|
||||
LOOP
|
||||
parent_table := r.pt;
|
||||
retention := r.ret;
|
||||
|
||||
SELECT min(d), max(d) INTO oldest_child, newest_child
|
||||
FROM (
|
||||
SELECT to_date(substring(c.relname FROM '_p(\d{8})$'), 'YYYYMMDD') AS d
|
||||
FROM pg_inherits i
|
||||
JOIN pg_class c ON c.oid = i.inhrelid
|
||||
WHERE i.inhparent = r.pt::regclass
|
||||
AND pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT'
|
||||
) s;
|
||||
|
||||
days_ahead := newest_child - current_date;
|
||||
|
||||
v_default := to_regclass(format('%I.%I',
|
||||
split_part(r.pt, '.', 1),
|
||||
split_part(r.pt, '.', 2) || '_default'));
|
||||
IF v_default IS NULL THEN
|
||||
default_rows := NULL;
|
||||
default_size := NULL;
|
||||
ELSE
|
||||
EXECUTE format('SELECT count(*) FROM %s', v_default::text) INTO default_rows;
|
||||
default_size := pg_size_pretty(pg_total_relation_size(v_default));
|
||||
END IF;
|
||||
|
||||
RETURN NEXT;
|
||||
END LOOP;
|
||||
END;
|
||||
$func$;
|
||||
|
||||
-- Drop the registration older releases left in so_telegraf.
|
||||
SELECT CASE
|
||||
WHEN current_setting('cron.database_name', true) IS DISTINCT FROM current_database()
|
||||
AND EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_cron')
|
||||
THEN 'true' ELSE 'false'
|
||||
END AS drop_stale_cron \gset
|
||||
\if :drop_stale_cron
|
||||
DROP EXTENSION pg_cron CASCADE;
|
||||
\endif
|
||||
EOSQL
|
||||
|
||||
# Guarded on the live GUC so applying this before the postgresql.conf change
|
||||
# has restarted the container skips rather than failing.
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d postgres <<'EOSQL'
|
||||
SELECT CASE WHEN current_setting('cron.database_name', true) = current_database()
|
||||
THEN 'true' ELSE 'false' END AS cron_here \gset
|
||||
\if :cron_here
|
||||
CREATE EXTENSION IF NOT EXISTS pg_cron;
|
||||
-- cron.schedule_in_database is idempotent by jobname.
|
||||
SELECT cron.schedule_in_database(
|
||||
'telegraf-partman-maintenance',
|
||||
'17 * * * *',
|
||||
'CALL partman.run_maintenance_proc()'
|
||||
'CALL so_admin.telegraf_maintenance()',
|
||||
'so_telegraf'
|
||||
);
|
||||
\else
|
||||
\echo 'pg_cron metadata database is not `postgres` yet; skipping job registration.'
|
||||
\endif
|
||||
EOSQL
|
||||
;;
|
||||
|
||||
@@ -90,6 +225,8 @@ EOSQL
|
||||
: "${RETENTION_DAYS:?RETENTION_DAYS is required}"
|
||||
# \gset + \if guards against a missing pg_partman without using a DO
|
||||
# block (psql :var substitution doesn't reach into dollar-quoted code).
|
||||
# premake is reconciled here because telegraf.conf only applies it to
|
||||
# parents created from now on.
|
||||
docker exec -i so-postgres psql \
|
||||
-v ON_ERROR_STOP=1 \
|
||||
-v retention_days="$RETENTION_DAYS" \
|
||||
@@ -97,14 +234,60 @@ EOSQL
|
||||
SELECT CASE WHEN EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_partman')
|
||||
THEN 'true' ELSE 'false' END AS has_partman \gset
|
||||
\if :has_partman
|
||||
-- infinite_time_partitions so a gap in metrics does not stop partman from
|
||||
-- premaking forward, which is what leaves everything in the default.
|
||||
UPDATE partman.part_config
|
||||
SET retention = :'retention_days' || ' days',
|
||||
retention_keep_table = false
|
||||
retention_keep_table = false,
|
||||
premake = 7,
|
||||
infinite_time_partitions = true
|
||||
WHERE parent_table LIKE 'telegraf.%';
|
||||
\endif
|
||||
EOSQL
|
||||
;;
|
||||
|
||||
maintenance)
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
|
||||
SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL
|
||||
THEN 'true' ELSE 'false' END AS has_proc \gset
|
||||
\if :has_proc
|
||||
CALL so_admin.telegraf_maintenance();
|
||||
\else
|
||||
\echo 'so_admin.telegraf_maintenance() is missing; run so-telegraf-postgres group_role first.'
|
||||
\endif
|
||||
EOSQL
|
||||
;;
|
||||
|
||||
check)
|
||||
docker exec -i so-postgres psql -U postgres -d so_telegraf <<'EOSQL'
|
||||
\pset border 2
|
||||
SELECT * FROM so_admin.telegraf_partition_status();
|
||||
EOSQL
|
||||
docker exec -i so-postgres psql -U postgres -d postgres <<'EOSQL'
|
||||
\pset border 2
|
||||
SELECT CASE WHEN to_regclass('cron.job_run_details') IS NOT NULL
|
||||
THEN 'true' ELSE 'false' END AS has_cron \gset
|
||||
\if :has_cron
|
||||
SELECT d.status, d.return_message, d.start_time
|
||||
FROM cron.job_run_details d
|
||||
JOIN cron.job j ON j.jobid = d.jobid
|
||||
WHERE j.jobname = 'telegraf-partman-maintenance'
|
||||
ORDER BY d.start_time DESC
|
||||
LIMIT 5;
|
||||
\else
|
||||
\echo 'pg_cron is not installed in this database.'
|
||||
\endif
|
||||
EOSQL
|
||||
unhealthy=$(docker exec so-postgres psql -U postgres -d so_telegraf -tAc \
|
||||
"SELECT count(*) FROM so_admin.telegraf_partition_status()
|
||||
WHERE coalesce(default_rows, 0) > 0 OR coalesce(days_ahead, -1) < 1")
|
||||
if [ "${unhealthy:-1}" != "0" ]; then
|
||||
echo "so-telegraf-postgres check: $unhealthy telegraf parent(s) unhealthy" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "so-telegraf-postgres check: all telegraf parents healthy"
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "Unknown subcommand: $cmd" >&2
|
||||
exit 1
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
# Put Telegraf metrics storage back in service on a grid where pg_partman
|
||||
# maintenance stalled: raises premake, discards the rows stranded in default
|
||||
# partitions, restarts so-postgres if pg_cron's launcher is dead, and runs
|
||||
# maintenance once. Healthy grids are reported and left alone.
|
||||
#
|
||||
# Usage: so-telegraf-repair [--check] [--yes] [--no-restart]
|
||||
# --check Report health and change nothing.
|
||||
# --yes Skip the confirmation prompt (for soup and other automation).
|
||||
# --no-restart Never restart so-postgres, even if pg_cron's launcher is dead.
|
||||
#
|
||||
# Exit status:
|
||||
# 0 healthy, or repair completed
|
||||
# 1 repair is needed (--check only)
|
||||
# 2 cannot run here: so-postgres, so_telegraf or pg_partman is missing
|
||||
|
||||
set -e
|
||||
|
||||
# Matches p_premake in telegraf.conf's create_parent template.
|
||||
PREMAKE=7
|
||||
JOB_NAME=telegraf-partman-maintenance
|
||||
|
||||
CHECK_ONLY=false
|
||||
ASSUME_YES=false
|
||||
NO_RESTART=false
|
||||
|
||||
usage() { sed -n '/^# Usage:/,/^# 2 /p' "$0" | sed 's/^# \?//'; }
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--check|--dry-run) CHECK_ONLY=true ;;
|
||||
--yes|-y) ASSUME_YES=true ;;
|
||||
--no-restart) NO_RESTART=true ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) echo "Unknown option: $1" >&2; usage >&2; exit 2 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
skip() { echo "$*"; exit 2; }
|
||||
|
||||
psql_tg() { docker exec -i so-postgres psql -U postgres -d so_telegraf "$@"; }
|
||||
psql_pg() { docker exec -i so-postgres psql -U postgres -d postgres "$@"; }
|
||||
|
||||
# query_to_xml so the per-table row counts need no helper function installed.
|
||||
REPORT="
|
||||
WITH parents AS (
|
||||
SELECT pc.parent_table,
|
||||
pc.premake,
|
||||
split_part(pc.parent_table, '.', 1) AS sch,
|
||||
split_part(pc.parent_table, '.', 2) AS tbl
|
||||
FROM partman.part_config pc
|
||||
WHERE pc.parent_table LIKE 'telegraf.%'
|
||||
), children AS (
|
||||
SELECT p.parent_table,
|
||||
max(to_date(substring(c.relname FROM '_p(\d{8})\$'), 'YYYYMMDD')) AS newest_child
|
||||
FROM parents p
|
||||
JOIN pg_class pt ON pt.oid = p.parent_table::regclass
|
||||
JOIN pg_inherits i ON i.inhparent = pt.oid
|
||||
JOIN pg_class c ON c.oid = i.inhrelid
|
||||
WHERE pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT'
|
||||
GROUP BY p.parent_table
|
||||
), defaults AS (
|
||||
SELECT p.parent_table,
|
||||
p.premake,
|
||||
format('%I.%I', p.sch, p.tbl || '_default') AS default_table,
|
||||
to_regclass(format('%I.%I', p.sch, p.tbl || '_default')) AS default_oid
|
||||
FROM parents p
|
||||
)
|
||||
SELECT d.parent_table,
|
||||
c.newest_child,
|
||||
(c.newest_child - current_date) AS days_ahead,
|
||||
d.premake,
|
||||
CASE WHEN d.default_oid IS NULL THEN NULL ELSE
|
||||
(xpath('/row/cnt/text()',
|
||||
query_to_xml(format('SELECT count(*) AS cnt FROM %s', d.default_table),
|
||||
false, true, '')))[1]::text::bigint
|
||||
END AS default_rows,
|
||||
CASE WHEN d.default_oid IS NULL THEN NULL
|
||||
ELSE pg_size_pretty(pg_total_relation_size(d.default_oid)) END AS default_size
|
||||
FROM defaults d
|
||||
LEFT JOIN children c ON c.parent_table = d.parent_table
|
||||
ORDER BY 1
|
||||
"
|
||||
|
||||
docker ps --format '{{.Names}}' | grep -qx so-postgres \
|
||||
|| skip "so-postgres is not running; nothing to repair."
|
||||
docker exec so-postgres psql -U postgres -tAc \
|
||||
"SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1 \
|
||||
|| skip "The so_telegraf database does not exist; Telegraf is not writing to Postgres."
|
||||
psql_tg -tAc "SELECT 1 FROM pg_extension WHERE extname='pg_partman'" | grep -q 1 \
|
||||
|| skip "pg_partman is not installed in so_telegraf; nothing to repair."
|
||||
|
||||
parents=$(psql_tg -tAc \
|
||||
"SELECT count(*) FROM partman.part_config WHERE parent_table LIKE 'telegraf.%'")
|
||||
stranded=$(psql_tg -tAc "SELECT coalesce(sum(default_rows), 0) FROM ( $REPORT ) t")
|
||||
behind=$(psql_tg -tAc \
|
||||
"SELECT count(*) FROM ( $REPORT ) t WHERE coalesce(days_ahead, -1) < 1")
|
||||
# premake < 7, or infinite_time_partitions off: without the latter partman
|
||||
# refuses to premake forward across the gap the stall left behind.
|
||||
misconfigured=$(psql_tg -tAc \
|
||||
"SELECT count(*) FROM partman.part_config
|
||||
WHERE parent_table LIKE 'telegraf.%'
|
||||
AND (premake < $PREMAKE OR NOT infinite_time_partitions)")
|
||||
|
||||
# Both columns are matched because which one carries the launcher's name varies
|
||||
# with the pg_cron version.
|
||||
launcher=$(psql_pg -tAc \
|
||||
"SELECT count(*) FROM pg_stat_activity
|
||||
WHERE backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%'")
|
||||
|
||||
# so_telegraf before the postgres state lands, postgres after.
|
||||
cron_db=$(docker exec so-postgres psql -U postgres -tAc \
|
||||
"SELECT current_setting('cron.database_name', true)" | tr -d '[:space:]')
|
||||
last_run=never
|
||||
if [[ -n "$cron_db" ]]; then
|
||||
last_run=$(docker exec so-postgres psql -U postgres -d "$cron_db" -tAc \
|
||||
"SELECT coalesce(max(d.start_time)::text, 'never')
|
||||
FROM cron.job_run_details d JOIN cron.job j USING (jobid)
|
||||
WHERE j.jobname = '$JOB_NAME'" 2>/dev/null | tr -d '[:space:]' || echo unknown)
|
||||
[[ -n "$last_run" ]] || last_run=never
|
||||
fi
|
||||
|
||||
# A grid that has never written a metric has nothing to recover, and an empty
|
||||
# cron_db means pg_cron is not loaded at all, which no restart fixes.
|
||||
restart_needed=false
|
||||
[[ "$launcher" -eq 0 && "$parents" -gt 0 && -n "$cron_db" ]] && restart_needed=true
|
||||
|
||||
repair_needed=false
|
||||
[[ "$stranded" -gt 0 ]] && repair_needed=true
|
||||
[[ "$behind" -gt 0 ]] && repair_needed=true
|
||||
[[ "$misconfigured" -gt 0 ]] && repair_needed=true
|
||||
$restart_needed && repair_needed=true
|
||||
|
||||
echo "Telegraf partition status:"
|
||||
psql_tg -c "$REPORT"
|
||||
echo "Rows stranded in default partitions: $stranded"
|
||||
echo "pg_cron metadata database: ${cron_db:-unset}"
|
||||
echo "pg_cron launcher running: $([[ "$launcher" -gt 0 ]] && echo yes || echo no)"
|
||||
echo "Last $JOB_NAME run: $last_run"
|
||||
echo
|
||||
|
||||
if ! $repair_needed; then
|
||||
echo "Telegraf partitions are healthy. Nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if $CHECK_ONLY; then
|
||||
echo "Repair is needed:"
|
||||
[[ "$stranded" -gt 0 ]] && echo " * $stranded row(s) stranded in default partitions"
|
||||
[[ "$behind" -gt 0 ]] && echo " * $behind parent(s) with no partition for the current window"
|
||||
[[ "$misconfigured" -gt 0 ]] && echo " * $misconfigured parent(s) with stale partman settings"
|
||||
$restart_needed && echo " * pg_cron's launcher is dead; maintenance is not running at all"
|
||||
echo
|
||||
echo "Re-run without --check to repair."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$stranded" -gt 0 ]] && ! $ASSUME_YES; then
|
||||
echo "This will permanently discard the $stranded stranded row(s) above."
|
||||
$restart_needed && ! $NO_RESTART && \
|
||||
echo "so-postgres will also be restarted, which briefly interrupts SOC."
|
||||
[[ -t 0 ]] || { echo "Not a terminal; re-run with --yes to confirm." >&2; exit 2; }
|
||||
read -r -p "Continue? [y/N] " answer
|
||||
[[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 0; }
|
||||
fi
|
||||
|
||||
if [[ "$misconfigured" -gt 0 ]]; then
|
||||
echo "Reconciling partman settings on $misconfigured parent(s)."
|
||||
# GREATEST so an operator who raised premake further keeps their value.
|
||||
psql_tg -v ON_ERROR_STOP=1 -c \
|
||||
"UPDATE partman.part_config
|
||||
SET premake = GREATEST(premake, $PREMAKE),
|
||||
infinite_time_partitions = true
|
||||
WHERE parent_table LIKE 'telegraf.%'"
|
||||
fi
|
||||
|
||||
if [[ "$stranded" -gt 0 ]]; then
|
||||
echo "Clearing default partitions."
|
||||
# One transaction: Telegraf is still writing, so a default emptied without
|
||||
# its partition in place is refilled before maintenance can attach one.
|
||||
psql_tg -v ON_ERROR_STOP=1 <<'EOSQL'
|
||||
DO $$
|
||||
DECLARE
|
||||
r record;
|
||||
BEGIN
|
||||
FOR r IN
|
||||
SELECT pc.parent_table,
|
||||
format('%I.%I', n.nspname, c.relname) AS default_table
|
||||
FROM partman.part_config pc
|
||||
JOIN pg_class p ON p.oid = pc.parent_table::regclass
|
||||
JOIN pg_inherits i ON i.inhparent = p.oid
|
||||
JOIN pg_class c ON c.oid = i.inhrelid
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE pc.parent_table LIKE 'telegraf.%'
|
||||
AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT'
|
||||
LOOP
|
||||
EXECUTE format('TRUNCATE TABLE %s', r.default_table);
|
||||
PERFORM partman.create_partition_time(
|
||||
r.parent_table, ARRAY[date_trunc('day', now())]::timestamptz[]);
|
||||
END LOOP;
|
||||
END
|
||||
$$;
|
||||
EOSQL
|
||||
fi
|
||||
|
||||
if $restart_needed; then
|
||||
if $NO_RESTART; then
|
||||
echo "WARNING: pg_cron's launcher is dead and --no-restart was given."
|
||||
echo " Maintenance will not run on its own until so-postgres is restarted."
|
||||
else
|
||||
echo "Restarting so-postgres to revive pg_cron's launcher."
|
||||
docker restart so-postgres >/dev/null
|
||||
for _ in $(seq 1 60); do
|
||||
docker exec so-postgres pg_isready -U postgres -q 2>/dev/null && break
|
||||
sleep 2
|
||||
done
|
||||
docker exec so-postgres pg_isready -U postgres -q \
|
||||
|| { echo "so-postgres did not come back; check 'docker logs so-postgres'." >&2; exit 1; }
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Running partition maintenance."
|
||||
# so_admin.telegraf_maintenance() only exists once the postgres state has landed.
|
||||
psql_tg -v ON_ERROR_STOP=1 <<'EOSQL'
|
||||
SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL
|
||||
THEN 'true' ELSE 'false' END AS has_proc \gset
|
||||
\if :has_proc
|
||||
CALL so_admin.telegraf_maintenance();
|
||||
\else
|
||||
CALL partman.run_maintenance_proc();
|
||||
\endif
|
||||
EOSQL
|
||||
|
||||
echo
|
||||
echo "Telegraf partition status after repair:"
|
||||
psql_tg -c "$REPORT"
|
||||
echo "The $JOB_NAME job runs hourly at :17. Confirm it fired with:"
|
||||
echo " so-telegraf-repair --check"
|
||||
@@ -122,7 +122,7 @@
|
||||
create_templates = [
|
||||
'''CREATE TABLE IF NOT EXISTS {{ .table }} ({{ .columns }}) PARTITION BY RANGE ("time")''',
|
||||
'''ALTER TABLE {{ .table }} ALTER COLUMN "time" SET NOT NULL''',
|
||||
'''SELECT partman.create_parent(p_parent_table := {{ printf "%s.%s" .table.Schema .table.Name | quoteLiteral }}, p_control := 'time', p_type := 'range', p_interval := '1 day', p_premake := 3) WHERE NOT EXISTS (SELECT 1 FROM partman.part_config WHERE parent_table = {{ printf "%s.%s" .table.Schema .table.Name | quoteLiteral }})'''
|
||||
'''SELECT partman.create_parent(p_parent_table := {{ printf "%s.%s" .table.Schema .table.Name | quoteLiteral }}, p_control := 'time', p_type := 'range', p_interval := '1 day', p_premake := 7) WHERE NOT EXISTS (SELECT 1 FROM partman.part_config WHERE parent_table = {{ printf "%s.%s" .table.Schema .table.Name | quoteLiteral }})'''
|
||||
]
|
||||
tag_table_create_templates = [
|
||||
'''CREATE TABLE IF NOT EXISTS {{ .table }} ({{ .columns }}, PRIMARY KEY (tag_id))'''
|
||||
|
||||
Reference in New Issue
Block a user