#!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. # Put Telegraf metrics storage back in service on a grid where pg_partman # maintenance stalled: raises premake, discards the rows stranded in default # partitions, restarts so-postgres if pg_cron's launcher is dead, and runs # maintenance once. Healthy grids are reported and left alone. # # Usage: so-telegraf-repair [--check] [--yes] [--no-restart] # --check Report health and change nothing. Also reports a missing # so_maint_telegraf role and a maintenance job that is missing # or inactive. # --yes Skip the confirmation prompt (for soup and other automation). # --no-restart Never restart so-postgres, even if pg_cron's launcher is dead. # # Repair mode (without --check) registers the so_maint_telegraf role and the # maintenance job if they are missing, pauses the job while it repairs, and # always leaves it active afterwards. Data work runs as so_maint_telegraf. # # Exit status: # 0 healthy, or repair completed # 1 repair is needed (--check), or the repair failed after it started # 2 skipped: Telegraf output is not Postgres, or so-postgres, so_telegraf or pg_partman is missing # 3 could not check, for example a connection error set -eE # Matches p_premake in telegraf.conf's create_parent template. PREMAKE=7 JOB_NAME=telegraf-partman-maintenance MAINT_ROLE=so_maint_telegraf # Long enough for an hourly maintenance run to release its locks, short enough # that soup and admins are not left hanging. LOCK_TIMEOUT=60s PG_OPTS="-c lock_timeout=$LOCK_TIMEOUT -c application_name=so-telegraf-repair" CHECK_ONLY=false ASSUME_YES=false NO_RESTART=false usage() { sed -n '/^# Usage:/,/^# 3 /p' "$0" | sed 's/^# \?//'; } while [[ $# -gt 0 ]]; do case "$1" in --check|--dry-run) CHECK_ONLY=true ;; --yes|-y) ASSUME_YES=true ;; --no-restart) NO_RESTART=true ;; -h|--help) usage; exit 0 ;; *) echo "Unknown option: $1" >&2; usage >&2; exit 3 ;; esac shift done # Strips C0 (except tab and newline), DEL and C1 characters, so names and # messages written with Telegraf credentials can't drive the terminal. sanitise() { LC_ALL=C tr -d '\000-\010\013-\037\177' | LC_ALL=C sed 's/\xc2[\x80-\x9f]//g'; } ERR_FILE=$(mktemp) ERR_STATUS=3 ERR_PREFIX="Could not check Telegraf partitions" job_id= restore_active= on_err() { # Command substitutions inherit this trap; only the top level reports. [[ $BASH_SUBSHELL -eq 0 ]] || exit "$ERR_STATUS" local line line=$(tail -n 1 "$ERR_FILE" | sanitise) echo "$ERR_PREFIX: ${line:-command failed}" >&2 exit "$ERR_STATUS" } on_exit() { if [[ -n "$restore_active" ]]; then psql_cron -tAc "SELECT cron.alter_job($job_id, active := $restore_active)" >/dev/null || true fi rm -f "$ERR_FILE" } trap on_err ERR trap on_exit EXIT finish() { trap - ERR; exit "$1"; } skip() { echo "$*"; finish 2; } # Keeps the last stderr line for on_err while still showing all of it. capture() { local rc=0 : >"$ERR_FILE" "$@" 2>"$ERR_FILE" || rc=$? sanitise <"$ERR_FILE" >&2 return "$rc" } psql_tg() { capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U "$MAINT_ROLE" -d so_telegraf "$@" } psql_pg() { capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U postgres -d postgres "$@" } psql_cron() { capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U postgres -d "$cron_db" "$@" } # A part_config row counts only while its parent is a partitioned table. TELEGRAF_PARENTS="parent_table LIKE 'telegraf.%' AND EXISTS (SELECT 1 FROM pg_class c WHERE c.oid = to_regclass(parent_table) AND c.relkind = 'p')" REPORT=" SELECT d.parent_table, c.newest_child, (c.newest_child - current_date) AS days_ahead, pc.premake, d.has_rows AS default_has_rows, pg_size_pretty(d.default_size) AS default_size FROM so_admin.telegraf_defaults() d JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table JOIN pg_class p ON p.oid = d.parent_table AND p.relkind = 'p' LEFT JOIN LATERAL ( SELECT max(to_date(substring(ch.relname FROM '_p(\d{8})\$'), 'YYYYMMDD')) AS newest_child FROM pg_inherits i JOIN pg_class ch ON ch.oid = i.inhrelid WHERE i.inhparent = d.parent_table AND pg_get_expr(ch.relpartbound, ch.oid) <> 'DEFAULT' ) c ON true ORDER BY d.parent_table::text " print_report() { local report report=$(psql_tg -c "$REPORT") sanitise <<<"$report" } has_cron() { [[ -n "$cron_db" ]] || return 0 psql_cron -tAc "SELECT 1 FROM pg_extension WHERE extname = 'pg_cron'" } lookup_job() { local cron cron=$(has_cron) [[ -n "$cron" ]] || return 0 psql_cron -tA -F '|' -c \ "SELECT jobid, active FROM cron.job WHERE jobname = '$JOB_NAME' AND username = '$MAINT_ROLE' ORDER BY jobid LIMIT 1" } output=$(capture salt-call pillar.get telegraf:output --out=newline_values_only) output=$(tr -d '[:space:]' <<<"$output" | tr '[:lower:]' '[:upper:]') output=${output:-INFLUXDB} case "$output" in POSTGRES|BOTH) ;; *) skip "Telegraf output is $(sanitise <<<"$output"); not checking Postgres partitions." ;; esac containers=$(capture docker ps --format '{{.Names}}') grep -qx so-postgres <<<"$containers" \ || skip "so-postgres is not running; nothing to repair." has_db=$(psql_pg -tAc "SELECT 1 FROM pg_database WHERE datname = 'so_telegraf'") [[ -n "$has_db" ]] \ || skip "The so_telegraf database does not exist; Telegraf is not writing to Postgres." # so_telegraf before the postgres state lands, postgres after. cron_db=$(psql_pg -tAc "SELECT current_setting('cron.database_name', true)") cron_db=$(tr -d '[:space:]' <<<"$cron_db") role_ok=$(psql_pg -tAc \ "SELECT rolcanlogin AND NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls) FROM pg_roles WHERE rolname = '$MAINT_ROLE'") job= [[ "$cron_db" == postgres ]] && job=$(lookup_job) if [[ "$role_ok" != t ]] && $CHECK_ONLY; then echo "Repair is needed:" if [[ -z "$role_ok" ]]; then echo " * maintenance role $MAINT_ROLE is missing" else echo " * maintenance role $MAINT_ROLE is misconfigured" fi echo echo "Re-run without --check to repair." finish 1 fi if ! $CHECK_ONLY && [[ "$role_ok" != t || ( "$cron_db" == postgres && -z "$job" ) ]]; then echo "Registering the maintenance role and job." if ! /usr/sbin/so-telegraf-postgres group_role; then echo "Could not register the maintenance job; run a highstate and try again." >&2 finish 3 fi [[ "$cron_db" == postgres ]] && job=$(lookup_job) fi has_partman=$(psql_tg -tAc "SELECT 1 FROM pg_extension WHERE extname = 'pg_partman'") [[ -n "$has_partman" ]] \ || skip "pg_partman is not installed in so_telegraf; nothing to repair." parents=$(psql_tg -tAc \ "SELECT count(*) FROM partman.part_config WHERE $TELEGRAF_PARENTS") counts=$(psql_tg -tAc \ "SELECT count(*) FILTER (WHERE default_has_rows) || '|' || count(*) FILTER (WHERE coalesce(days_ahead, -1) < 1) FROM ( $REPORT ) t") IFS='|' read -r stranded behind <<<"$counts" # premake < 7, or infinite_time_partitions off: without the latter partman # refuses to premake forward across the gap the stall left behind. misconfigured=$(psql_tg -tAc \ "SELECT count(*) FROM partman.part_config WHERE $TELEGRAF_PARENTS AND (premake < $PREMAKE OR NOT infinite_time_partitions)") # Both columns are matched because which one carries the launcher's name varies # with the pg_cron version. launcher=$(psql_pg -tAc \ "SELECT count(*) FROM pg_stat_activity WHERE backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%'") last_run=never cron_installed=$(has_cron) if [[ -n "$cron_installed" ]]; then run=$(psql_cron -tAc \ "SELECT d.status || ' at ' || d.start_time FROM cron.job_run_details d JOIN cron.job j USING (jobid) WHERE j.jobname = '$JOB_NAME' ORDER BY d.start_time DESC LIMIT 1") if [[ -n "$run" ]]; then message=$(psql_cron -tAc \ "SELECT coalesce(d.return_message, '') FROM cron.job_run_details d JOIN cron.job j USING (jobid) WHERE j.jobname = '$JOB_NAME' ORDER BY d.start_time DESC LIMIT 1") message=$(sanitise <<<"$message") message=${message//$'\n'/ } last_run="$(sanitise <<<"$run"): ${message:0:200}" fi fi # A grid that has never written a metric has nothing to recover, and an empty # cron_db means pg_cron is not loaded at all, which no restart fixes. restart_needed=false [[ "$launcher" -eq 0 && "$parents" -gt 0 && -n "$cron_db" ]] && restart_needed=true # group_role registers the job only in the postgres database. job_inactive=false if [[ "$cron_db" == postgres ]]; then [[ -z "$job" || "${job##*|}" == f ]] && job_inactive=true fi repair_needed=false [[ "$stranded" -gt 0 ]] && repair_needed=true [[ "$behind" -gt 0 ]] && repair_needed=true [[ "$misconfigured" -gt 0 ]] && repair_needed=true $restart_needed && repair_needed=true $job_inactive && repair_needed=true echo "Telegraf partition status:" print_report echo "Default partitions with rows: $stranded" echo "pg_cron metadata database: $(sanitise <<<"${cron_db:-unset}")" [[ "$cron_db" == postgres ]] \ || echo "pg_cron metadata database is $(sanitise <<<"${cron_db:-unset}"); not checking the maintenance job." echo "pg_cron launcher running: $([[ "$launcher" -gt 0 ]] && echo yes || echo no)" echo "Last $JOB_NAME run: $last_run" echo if ! $repair_needed; then echo "Telegraf partitions are healthy. Nothing to do." finish 0 fi if $CHECK_ONLY; then echo "Repair is needed:" [[ "$stranded" -gt 0 ]] && echo " * $stranded default partition(s) hold rows" [[ "$behind" -gt 0 ]] && echo " * $behind parent(s) with no partition for the current window" [[ "$misconfigured" -gt 0 ]] && echo " * $misconfigured parent(s) with stale partman settings" $restart_needed && echo " * pg_cron's launcher is dead; maintenance is not running at all" $job_inactive && echo " * the maintenance job is missing or inactive" echo echo "Re-run without --check to repair." finish 1 fi if [[ "$stranded" -gt 0 ]] && ! $ASSUME_YES; then echo "This will permanently discard the rows in the $stranded default partition(s) above." $restart_needed && ! $NO_RESTART && \ echo "so-postgres will also be restarted, which briefly interrupts SOC." [[ -t 0 ]] || { echo "Not a terminal; re-run with --yes to confirm." >&2; finish 1; } read -r -p "Continue? [y/N] " answer [[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; finish 0; } fi ERR_STATUS=1 ERR_PREFIX="Telegraf partition repair failed" if [[ "$cron_db" == postgres ]]; then if [[ -z "$job" ]]; then echo "Could not register the maintenance job; run a highstate and try again." >&2 finish 3 fi job_id=${job%%|*} [[ "${job##*|}" == t ]] && restore_active=true || restore_active=false echo "Pausing the $JOB_NAME job during the repair." psql_cron -tAc "SELECT cron.alter_job($job_id, active := false)" >/dev/null fi if [[ "$misconfigured" -gt 0 ]]; then echo "Reconciling partman settings on $misconfigured parent(s)." # GREATEST so an operator who raised premake further keeps their value. psql_tg -v ON_ERROR_STOP=1 -c \ "UPDATE partman.part_config SET premake = GREATEST(premake, $PREMAKE), infinite_time_partitions = true WHERE $TELEGRAF_PARENTS" fi if [[ "$stranded" -gt 0 ]]; then echo "Clearing default partitions." # One transaction: Telegraf is still writing, so a default emptied without # its partition in place is refilled before maintenance can attach one. psql_tg -v ON_ERROR_STOP=1 <<'EOSQL' BEGIN; DO $$ DECLARE r record; BEGIN FOR r IN SELECT pc.parent_table, d.default_part FROM so_admin.telegraf_defaults() d JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table WHERE d.default_part IS NOT NULL LOOP EXECUTE format('TRUNCATE TABLE %s', r.default_part); PERFORM partman.create_partition_time( r.parent_table, ARRAY[date_trunc('day', now())]::timestamptz[]); END LOOP; END $$; COMMIT; EOSQL fi if $restart_needed; then if $NO_RESTART; then echo "WARNING: pg_cron's launcher is dead and --no-restart was given." echo " Maintenance will not run on its own until so-postgres is restarted." else echo "Restarting so-postgres to revive pg_cron's launcher." docker restart so-postgres >/dev/null for _ in $(seq 1 60); do docker exec so-postgres pg_isready -U postgres -q 2>/dev/null && break sleep 2 done docker exec so-postgres pg_isready -U postgres -q \ || { echo "so-postgres did not come back; check 'docker logs so-postgres'." >&2; finish 1; } fi fi echo "Running partition maintenance." # so_admin.telegraf_maintenance() only exists once the postgres state has landed. psql_tg -v ON_ERROR_STOP=1 <<'EOSQL' SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL THEN 'true' ELSE 'false' END AS has_proc \gset \if :has_proc CALL so_admin.telegraf_maintenance(); \else CALL partman.run_maintenance_proc(); \endif EOSQL if [[ -n "$job_id" ]]; then psql_cron -tAc "SELECT cron.alter_job($job_id, active := true)" >/dev/null restore_active= echo "The $JOB_NAME job is active." fi echo echo "Telegraf partition status after repair:" print_report echo "The $JOB_NAME job runs hourly at :17. Confirm it fired with:" echo " so-telegraf-repair --check"