mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-11-18 18:16:02 +01:00
388 lines
14 KiB
Bash
388 lines
14 KiB
Bash
#!/bin/bash
|
|
|
|
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
|
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
|
# https://securityonion.net/license; you may not use this file except in compliance with the
|
|
# Elastic License 2.0.
|
|
|
|
# Put Telegraf metrics storage back in service on a grid where pg_partman
|
|
# maintenance stalled: raises premake, discards the rows stranded in default
|
|
# partitions, restarts so-postgres if pg_cron's launcher is dead, and runs
|
|
# maintenance once. Healthy grids are reported and left alone.
|
|
#
|
|
# Usage: so-telegraf-repair [--check] [--yes] [--no-restart]
|
|
# --check Report health and change nothing. Also reports a missing
|
|
# so_maint_telegraf role and a maintenance job that is missing
|
|
# or inactive.
|
|
# --yes Skip the confirmation prompt (for soup and other automation).
|
|
# --no-restart Never restart so-postgres, even if pg_cron's launcher is dead.
|
|
#
|
|
# Repair mode (without --check) registers the so_maint_telegraf role and the
|
|
# maintenance job if they are missing, pauses the job while it repairs, and
|
|
# always leaves it active afterwards. Data work runs as so_maint_telegraf.
|
|
#
|
|
# Exit status:
|
|
# 0 healthy, or repair completed
|
|
# 1 repair is needed (--check), or the repair failed after it started
|
|
# 2 skipped: Telegraf output is not Postgres, or so-postgres, so_telegraf or pg_partman is missing
|
|
# 3 could not check, for example a connection error
|
|
|
|
set -eE
|
|
|
|
# Matches p_premake in telegraf.conf's create_parent template.
|
|
PREMAKE=7
|
|
JOB_NAME=telegraf-partman-maintenance
|
|
MAINT_ROLE=so_maint_telegraf
|
|
# Long enough for an hourly maintenance run to release its locks, short enough
|
|
# that soup and admins are not left hanging.
|
|
LOCK_TIMEOUT=60s
|
|
PG_OPTS="-c lock_timeout=$LOCK_TIMEOUT -c application_name=so-telegraf-repair"
|
|
|
|
CHECK_ONLY=false
|
|
ASSUME_YES=false
|
|
NO_RESTART=false
|
|
|
|
usage() { sed -n '/^# Usage:/,/^# 3 /p' "$0" | sed 's/^# \?//'; }
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--check|--dry-run) CHECK_ONLY=true ;;
|
|
--yes|-y) ASSUME_YES=true ;;
|
|
--no-restart) NO_RESTART=true ;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) echo "Unknown option: $1" >&2; usage >&2; exit 3 ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
# Strips C0 (except tab and newline), DEL and C1 characters, so names and
|
|
# messages written with Telegraf credentials can't drive the terminal.
|
|
sanitise() { LC_ALL=C tr -d '\000-\010\013-\037\177' | LC_ALL=C sed 's/\xc2[\x80-\x9f]//g'; }
|
|
|
|
ERR_FILE=$(mktemp)
|
|
ERR_STATUS=3
|
|
ERR_PREFIX="Could not check Telegraf partitions"
|
|
job_id=
|
|
restore_active=
|
|
|
|
on_err() {
|
|
# Command substitutions inherit this trap; only the top level reports.
|
|
[[ $BASH_SUBSHELL -eq 0 ]] || exit "$ERR_STATUS"
|
|
local line
|
|
line=$(tail -n 1 "$ERR_FILE" | sanitise)
|
|
echo "$ERR_PREFIX: ${line:-command failed}" >&2
|
|
exit "$ERR_STATUS"
|
|
}
|
|
|
|
on_exit() {
|
|
if [[ -n "$restore_active" ]]; then
|
|
psql_cron -tAc "SELECT cron.alter_job($job_id, active := $restore_active)" >/dev/null || true
|
|
fi
|
|
rm -f "$ERR_FILE"
|
|
}
|
|
|
|
trap on_err ERR
|
|
trap on_exit EXIT
|
|
|
|
finish() { trap - ERR; exit "$1"; }
|
|
skip() { echo "$*"; finish 2; }
|
|
|
|
# Keeps the last stderr line for on_err while still showing all of it.
|
|
capture() {
|
|
local rc=0
|
|
: >"$ERR_FILE"
|
|
"$@" 2>"$ERR_FILE" || rc=$?
|
|
sanitise <"$ERR_FILE" >&2
|
|
return "$rc"
|
|
}
|
|
|
|
psql_tg() {
|
|
capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U "$MAINT_ROLE" -d so_telegraf "$@"
|
|
}
|
|
psql_pg() {
|
|
capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U postgres -d postgres "$@"
|
|
}
|
|
psql_cron() {
|
|
capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U postgres -d "$cron_db" "$@"
|
|
}
|
|
|
|
# A part_config row counts only while its parent is a partitioned table.
|
|
TELEGRAF_PARENTS="parent_table LIKE 'telegraf.%'
|
|
AND EXISTS (SELECT 1 FROM pg_class c
|
|
WHERE c.oid = to_regclass(parent_table) AND c.relkind = 'p')"
|
|
|
|
REPORT="
|
|
SELECT d.parent_table,
|
|
c.newest_child,
|
|
(c.newest_child - current_date) AS days_ahead,
|
|
pc.premake,
|
|
d.has_rows AS default_has_rows,
|
|
pg_size_pretty(d.default_size) AS default_size
|
|
FROM so_admin.telegraf_defaults() d
|
|
JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table
|
|
JOIN pg_class p ON p.oid = d.parent_table AND p.relkind = 'p'
|
|
LEFT JOIN LATERAL (
|
|
SELECT max(to_date(substring(ch.relname FROM '_p(\d{8})\$'), 'YYYYMMDD')) AS newest_child
|
|
FROM pg_inherits i
|
|
JOIN pg_class ch ON ch.oid = i.inhrelid
|
|
WHERE i.inhparent = d.parent_table
|
|
AND pg_get_expr(ch.relpartbound, ch.oid) <> 'DEFAULT'
|
|
) c ON true
|
|
ORDER BY d.parent_table::text
|
|
"
|
|
|
|
print_report() {
|
|
local report
|
|
report=$(psql_tg -c "$REPORT")
|
|
sanitise <<<"$report"
|
|
}
|
|
|
|
has_cron() {
|
|
[[ -n "$cron_db" ]] || return 0
|
|
psql_cron -tAc "SELECT 1 FROM pg_extension WHERE extname = 'pg_cron'"
|
|
}
|
|
|
|
lookup_job() {
|
|
local cron
|
|
cron=$(has_cron)
|
|
[[ -n "$cron" ]] || return 0
|
|
psql_cron -tA -F '|' -c \
|
|
"SELECT jobid, active FROM cron.job
|
|
WHERE jobname = '$JOB_NAME' AND username = '$MAINT_ROLE'
|
|
ORDER BY jobid LIMIT 1"
|
|
}
|
|
|
|
output=$(capture salt-call pillar.get telegraf:output --out=newline_values_only)
|
|
output=$(tr -d '[:space:]' <<<"$output" | tr '[:lower:]' '[:upper:]')
|
|
output=${output:-INFLUXDB}
|
|
case "$output" in
|
|
POSTGRES|BOTH) ;;
|
|
*) skip "Telegraf output is $(sanitise <<<"$output"); not checking Postgres partitions." ;;
|
|
esac
|
|
|
|
containers=$(capture docker ps --format '{{.Names}}')
|
|
grep -qx so-postgres <<<"$containers" \
|
|
|| skip "so-postgres is not running; nothing to repair."
|
|
has_db=$(psql_pg -tAc "SELECT 1 FROM pg_database WHERE datname = 'so_telegraf'")
|
|
[[ -n "$has_db" ]] \
|
|
|| skip "The so_telegraf database does not exist; Telegraf is not writing to Postgres."
|
|
|
|
# so_telegraf before the postgres state lands, postgres after.
|
|
cron_db=$(psql_pg -tAc "SELECT current_setting('cron.database_name', true)")
|
|
cron_db=$(tr -d '[:space:]' <<<"$cron_db")
|
|
|
|
role_ok=$(psql_pg -tAc \
|
|
"SELECT rolcanlogin AND NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls)
|
|
FROM pg_roles WHERE rolname = '$MAINT_ROLE'")
|
|
job=
|
|
[[ "$cron_db" == postgres ]] && job=$(lookup_job)
|
|
|
|
if [[ "$role_ok" != t ]] && $CHECK_ONLY; then
|
|
echo "Repair is needed:"
|
|
if [[ -z "$role_ok" ]]; then
|
|
echo " * maintenance role $MAINT_ROLE is missing"
|
|
else
|
|
echo " * maintenance role $MAINT_ROLE is misconfigured"
|
|
fi
|
|
echo
|
|
echo "Re-run without --check to repair."
|
|
finish 1
|
|
fi
|
|
|
|
if ! $CHECK_ONLY && [[ "$role_ok" != t || ( "$cron_db" == postgres && -z "$job" ) ]]; then
|
|
echo "Registering the maintenance role and job."
|
|
if ! /usr/sbin/so-telegraf-postgres group_role; then
|
|
echo "Could not register the maintenance job; run a highstate and try again." >&2
|
|
finish 3
|
|
fi
|
|
[[ "$cron_db" == postgres ]] && job=$(lookup_job)
|
|
fi
|
|
|
|
has_partman=$(psql_tg -tAc "SELECT 1 FROM pg_extension WHERE extname = 'pg_partman'")
|
|
[[ -n "$has_partman" ]] \
|
|
|| skip "pg_partman is not installed in so_telegraf; nothing to repair."
|
|
|
|
parents=$(psql_tg -tAc \
|
|
"SELECT count(*) FROM partman.part_config WHERE $TELEGRAF_PARENTS")
|
|
counts=$(psql_tg -tAc \
|
|
"SELECT count(*) FILTER (WHERE default_has_rows) || '|' ||
|
|
count(*) FILTER (WHERE coalesce(days_ahead, -1) < 1)
|
|
FROM ( $REPORT ) t")
|
|
IFS='|' read -r stranded behind <<<"$counts"
|
|
# premake < 7, or infinite_time_partitions off: without the latter partman
|
|
# refuses to premake forward across the gap the stall left behind.
|
|
misconfigured=$(psql_tg -tAc \
|
|
"SELECT count(*) FROM partman.part_config
|
|
WHERE $TELEGRAF_PARENTS
|
|
AND (premake < $PREMAKE OR NOT infinite_time_partitions)")
|
|
|
|
# Both columns are matched because which one carries the launcher's name varies
|
|
# with the pg_cron version.
|
|
launcher=$(psql_pg -tAc \
|
|
"SELECT count(*) FROM pg_stat_activity
|
|
WHERE backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%'")
|
|
|
|
last_run=never
|
|
cron_installed=$(has_cron)
|
|
if [[ -n "$cron_installed" ]]; then
|
|
run=$(psql_cron -tAc \
|
|
"SELECT d.status || ' at ' || d.start_time
|
|
FROM cron.job_run_details d JOIN cron.job j USING (jobid)
|
|
WHERE j.jobname = '$JOB_NAME'
|
|
ORDER BY d.start_time DESC LIMIT 1")
|
|
if [[ -n "$run" ]]; then
|
|
message=$(psql_cron -tAc \
|
|
"SELECT coalesce(d.return_message, '')
|
|
FROM cron.job_run_details d JOIN cron.job j USING (jobid)
|
|
WHERE j.jobname = '$JOB_NAME'
|
|
ORDER BY d.start_time DESC LIMIT 1")
|
|
message=$(sanitise <<<"$message")
|
|
message=${message//$'\n'/ }
|
|
last_run="$(sanitise <<<"$run"): ${message:0:200}"
|
|
fi
|
|
fi
|
|
|
|
# A grid that has never written a metric has nothing to recover, and an empty
|
|
# cron_db means pg_cron is not loaded at all, which no restart fixes.
|
|
restart_needed=false
|
|
[[ "$launcher" -eq 0 && "$parents" -gt 0 && -n "$cron_db" ]] && restart_needed=true
|
|
|
|
# group_role registers the job only in the postgres database.
|
|
job_inactive=false
|
|
if [[ "$cron_db" == postgres ]]; then
|
|
[[ -z "$job" || "${job##*|}" == f ]] && job_inactive=true
|
|
fi
|
|
|
|
repair_needed=false
|
|
[[ "$stranded" -gt 0 ]] && repair_needed=true
|
|
[[ "$behind" -gt 0 ]] && repair_needed=true
|
|
[[ "$misconfigured" -gt 0 ]] && repair_needed=true
|
|
$restart_needed && repair_needed=true
|
|
$job_inactive && repair_needed=true
|
|
|
|
echo "Telegraf partition status:"
|
|
print_report
|
|
echo "Default partitions with rows: $stranded"
|
|
echo "pg_cron metadata database: $(sanitise <<<"${cron_db:-unset}")"
|
|
[[ "$cron_db" == postgres ]] \
|
|
|| echo "pg_cron metadata database is $(sanitise <<<"${cron_db:-unset}"); not checking the maintenance job."
|
|
echo "pg_cron launcher running: $([[ "$launcher" -gt 0 ]] && echo yes || echo no)"
|
|
echo "Last $JOB_NAME run: $last_run"
|
|
echo
|
|
|
|
if ! $repair_needed; then
|
|
echo "Telegraf partitions are healthy. Nothing to do."
|
|
finish 0
|
|
fi
|
|
|
|
if $CHECK_ONLY; then
|
|
echo "Repair is needed:"
|
|
[[ "$stranded" -gt 0 ]] && echo " * $stranded default partition(s) hold rows"
|
|
[[ "$behind" -gt 0 ]] && echo " * $behind parent(s) with no partition for the current window"
|
|
[[ "$misconfigured" -gt 0 ]] && echo " * $misconfigured parent(s) with stale partman settings"
|
|
$restart_needed && echo " * pg_cron's launcher is dead; maintenance is not running at all"
|
|
$job_inactive && echo " * the maintenance job is missing or inactive"
|
|
echo
|
|
echo "Re-run without --check to repair."
|
|
finish 1
|
|
fi
|
|
|
|
if [[ "$stranded" -gt 0 ]] && ! $ASSUME_YES; then
|
|
echo "This will permanently discard the rows in the $stranded default partition(s) above."
|
|
$restart_needed && ! $NO_RESTART && \
|
|
echo "so-postgres will also be restarted, which briefly interrupts SOC."
|
|
[[ -t 0 ]] || { echo "Not a terminal; re-run with --yes to confirm." >&2; finish 1; }
|
|
read -r -p "Continue? [y/N] " answer
|
|
[[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; finish 0; }
|
|
fi
|
|
|
|
ERR_STATUS=1
|
|
ERR_PREFIX="Telegraf partition repair failed"
|
|
|
|
if [[ "$cron_db" == postgres ]]; then
|
|
if [[ -z "$job" ]]; then
|
|
echo "Could not register the maintenance job; run a highstate and try again." >&2
|
|
finish 3
|
|
fi
|
|
job_id=${job%%|*}
|
|
[[ "${job##*|}" == t ]] && restore_active=true || restore_active=false
|
|
echo "Pausing the $JOB_NAME job during the repair."
|
|
psql_cron -tAc "SELECT cron.alter_job($job_id, active := false)" >/dev/null
|
|
fi
|
|
|
|
if [[ "$misconfigured" -gt 0 ]]; then
|
|
echo "Reconciling partman settings on $misconfigured parent(s)."
|
|
# GREATEST so an operator who raised premake further keeps their value.
|
|
psql_tg -v ON_ERROR_STOP=1 -c \
|
|
"UPDATE partman.part_config
|
|
SET premake = GREATEST(premake, $PREMAKE),
|
|
infinite_time_partitions = true
|
|
WHERE $TELEGRAF_PARENTS"
|
|
fi
|
|
|
|
if [[ "$stranded" -gt 0 ]]; then
|
|
echo "Clearing default partitions."
|
|
# One transaction: Telegraf is still writing, so a default emptied without
|
|
# its partition in place is refilled before maintenance can attach one.
|
|
psql_tg -v ON_ERROR_STOP=1 <<'EOSQL'
|
|
BEGIN;
|
|
DO $$
|
|
DECLARE
|
|
r record;
|
|
BEGIN
|
|
FOR r IN
|
|
SELECT pc.parent_table, d.default_part
|
|
FROM so_admin.telegraf_defaults() d
|
|
JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table
|
|
WHERE d.default_part IS NOT NULL
|
|
LOOP
|
|
EXECUTE format('TRUNCATE TABLE %s', r.default_part);
|
|
PERFORM partman.create_partition_time(
|
|
r.parent_table, ARRAY[date_trunc('day', now())]::timestamptz[]);
|
|
END LOOP;
|
|
END
|
|
$$;
|
|
COMMIT;
|
|
EOSQL
|
|
fi
|
|
|
|
if $restart_needed; then
|
|
if $NO_RESTART; then
|
|
echo "WARNING: pg_cron's launcher is dead and --no-restart was given."
|
|
echo " Maintenance will not run on its own until so-postgres is restarted."
|
|
else
|
|
echo "Restarting so-postgres to revive pg_cron's launcher."
|
|
docker restart so-postgres >/dev/null
|
|
for _ in $(seq 1 60); do
|
|
docker exec so-postgres pg_isready -U postgres -q 2>/dev/null && break
|
|
sleep 2
|
|
done
|
|
docker exec so-postgres pg_isready -U postgres -q \
|
|
|| { echo "so-postgres did not come back; check 'docker logs so-postgres'." >&2; finish 1; }
|
|
fi
|
|
fi
|
|
|
|
echo "Running partition maintenance."
|
|
# so_admin.telegraf_maintenance() only exists once the postgres state has landed.
|
|
psql_tg -v ON_ERROR_STOP=1 <<'EOSQL'
|
|
SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL
|
|
THEN 'true' ELSE 'false' END AS has_proc \gset
|
|
\if :has_proc
|
|
CALL so_admin.telegraf_maintenance();
|
|
\else
|
|
CALL partman.run_maintenance_proc();
|
|
\endif
|
|
EOSQL
|
|
|
|
if [[ -n "$job_id" ]]; then
|
|
psql_cron -tAc "SELECT cron.alter_job($job_id, active := true)" >/dev/null
|
|
restore_active=
|
|
echo "The $JOB_NAME job is active."
|
|
fi
|
|
|
|
echo
|
|
echo "Telegraf partition status after repair:"
|
|
print_report
|
|
echo "The $JOB_NAME job runs hourly at :17. Confirm it fired with:"
|
|
echo " so-telegraf-repair --check"
|