#!/bin/bash

# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.

# Put Telegraf metrics storage back in service on a grid where pg_partman
# maintenance stalled: raises premake, discards the rows stranded in default
# partitions, restarts so-postgres if pg_cron's launcher is dead, and runs
# maintenance once. Healthy grids are reported and left alone.
#
# Usage: so-telegraf-repair [--check] [--yes] [--no-restart]
#   --check       Report health and change nothing. Also reports a missing
#                 so_maint_telegraf role and a maintenance job that is missing
#                 or inactive.
#   --yes         Skip the confirmation prompt (for soup and other automation).
#   --no-restart  Never restart so-postgres, even if pg_cron's launcher is dead.
#
# Repair mode (without --check) registers the so_maint_telegraf role and the
# maintenance job if they are missing, pauses the job while it repairs, and
# always leaves it active afterwards. Data work runs as so_maint_telegraf.
#
# Exit status:
#   0  healthy, or repair completed
#   1  repair is needed (--check), or the repair failed after it started
#   2  skipped: Telegraf output is not Postgres, or so-postgres, so_telegraf or pg_partman is missing
#   3  could not check, for example a connection error

set -eE

# Matches p_premake in telegraf.conf's create_parent template.
PREMAKE=7
JOB_NAME=telegraf-partman-maintenance
MAINT_ROLE=so_maint_telegraf
# Long enough for an hourly maintenance run to release its locks, short enough
# that soup and admins are not left hanging.
LOCK_TIMEOUT=60s
PG_OPTS="-c lock_timeout=$LOCK_TIMEOUT -c application_name=so-telegraf-repair"

CHECK_ONLY=false
ASSUME_YES=false
NO_RESTART=false

usage() { sed -n '/^# Usage:/,/^#   3 /p' "$0" | sed 's/^# \?//'; }

while [[ $# -gt 0 ]]; do
  case "$1" in
    --check|--dry-run) CHECK_ONLY=true ;;
    --yes|-y)          ASSUME_YES=true ;;
    --no-restart)      NO_RESTART=true ;;
    -h|--help)         usage; exit 0 ;;
    *) echo "Unknown option: $1" >&2; usage >&2; exit 3 ;;
  esac
  shift
done

# Strips C0 (except tab and newline), DEL and C1 characters, so names and
# messages written with Telegraf credentials can't drive the terminal.
sanitise() { LC_ALL=C tr -d '\000-\010\013-\037\177' | LC_ALL=C sed 's/\xc2[\x80-\x9f]//g'; }

ERR_FILE=$(mktemp)
ERR_STATUS=3
ERR_PREFIX="Could not check Telegraf partitions"
job_id=
restore_active=

on_err() {
  # Command substitutions inherit this trap; only the top level reports.
  [[ $BASH_SUBSHELL -eq 0 ]] || exit "$ERR_STATUS"
  local line
  line=$(tail -n 1 "$ERR_FILE" | sanitise)
  echo "$ERR_PREFIX: ${line:-command failed}" >&2
  exit "$ERR_STATUS"
}

on_exit() {
  if [[ -n "$restore_active" ]]; then
    psql_cron -tAc "SELECT cron.alter_job($job_id, active := $restore_active)" >/dev/null || true
  fi
  rm -f "$ERR_FILE"
}

trap on_err ERR
trap on_exit EXIT

finish() { trap - ERR; exit "$1"; }
skip() { echo "$*"; finish 2; }

# Keeps the last stderr line for on_err while still showing all of it.
capture() {
  local rc=0
  : >"$ERR_FILE"
  "$@" 2>"$ERR_FILE" || rc=$?
  sanitise <"$ERR_FILE" >&2
  return "$rc"
}

psql_tg() {
  capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U "$MAINT_ROLE" -d so_telegraf "$@"
}
psql_pg() {
  capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U postgres -d postgres "$@"
}
psql_cron() {
  capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U postgres -d "$cron_db" "$@"
}

# A part_config row counts only while its parent is a partitioned table.
TELEGRAF_PARENTS="parent_table LIKE 'telegraf.%'
  AND EXISTS (SELECT 1 FROM pg_class c
              WHERE c.oid = to_regclass(parent_table) AND c.relkind = 'p')"

REPORT="
SELECT d.parent_table,
       c.newest_child,
       (c.newest_child - current_date) AS days_ahead,
       pc.premake,
       d.has_rows AS default_has_rows,
       pg_size_pretty(d.default_size) AS default_size
FROM so_admin.telegraf_defaults() d
JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table
JOIN pg_class p ON p.oid = d.parent_table AND p.relkind = 'p'
LEFT JOIN LATERAL (
    SELECT max(to_date(substring(ch.relname FROM '_p(\d{8})\$'), 'YYYYMMDD')) AS newest_child
    FROM pg_inherits i
    JOIN pg_class ch ON ch.oid = i.inhrelid
    WHERE i.inhparent = d.parent_table
      AND pg_get_expr(ch.relpartbound, ch.oid) <> 'DEFAULT'
) c ON true
ORDER BY d.parent_table::text
"

print_report() {
  local report
  report=$(psql_tg -c "$REPORT")
  sanitise <<<"$report"
}

has_cron() {
  [[ -n "$cron_db" ]] || return 0
  psql_cron -tAc "SELECT 1 FROM pg_extension WHERE extname = 'pg_cron'"
}

lookup_job() {
  local cron
  cron=$(has_cron)
  [[ -n "$cron" ]] || return 0
  psql_cron -tA -F '|' -c \
    "SELECT jobid, active FROM cron.job
      WHERE jobname = '$JOB_NAME' AND username = '$MAINT_ROLE'
      ORDER BY jobid LIMIT 1"
}

output=$(capture salt-call pillar.get telegraf:output --out=newline_values_only)
output=$(tr -d '[:space:]' <<<"$output" | tr '[:lower:]' '[:upper:]')
output=${output:-INFLUXDB}
case "$output" in
  POSTGRES|BOTH) ;;
  *) skip "Telegraf output is $(sanitise <<<"$output"); not checking Postgres partitions." ;;
esac

containers=$(capture docker ps --format '{{.Names}}')
grep -qx so-postgres <<<"$containers" \
  || skip "so-postgres is not running; nothing to repair."
has_db=$(psql_pg -tAc "SELECT 1 FROM pg_database WHERE datname = 'so_telegraf'")
[[ -n "$has_db" ]] \
  || skip "The so_telegraf database does not exist; Telegraf is not writing to Postgres."

# so_telegraf before the postgres state lands, postgres after.
cron_db=$(psql_pg -tAc "SELECT current_setting('cron.database_name', true)")
cron_db=$(tr -d '[:space:]' <<<"$cron_db")

role_ok=$(psql_pg -tAc \
  "SELECT rolcanlogin AND NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls)
     FROM pg_roles WHERE rolname = '$MAINT_ROLE'")
job=
[[ "$cron_db" == postgres ]] && job=$(lookup_job)

if [[ "$role_ok" != t ]] && $CHECK_ONLY; then
  echo "Repair is needed:"
  if [[ -z "$role_ok" ]]; then
    echo "  * maintenance role $MAINT_ROLE is missing"
  else
    echo "  * maintenance role $MAINT_ROLE is misconfigured"
  fi
  echo
  echo "Re-run without --check to repair."
  finish 1
fi

if ! $CHECK_ONLY && [[ "$role_ok" != t || ( "$cron_db" == postgres && -z "$job" ) ]]; then
  echo "Registering the maintenance role and job."
  if ! /usr/sbin/so-telegraf-postgres group_role; then
    echo "Could not register the maintenance job; run a highstate and try again." >&2
    finish 3
  fi
  [[ "$cron_db" == postgres ]] && job=$(lookup_job)
fi

has_partman=$(psql_tg -tAc "SELECT 1 FROM pg_extension WHERE extname = 'pg_partman'")
[[ -n "$has_partman" ]] \
  || skip "pg_partman is not installed in so_telegraf; nothing to repair."

parents=$(psql_tg -tAc \
  "SELECT count(*) FROM partman.part_config WHERE $TELEGRAF_PARENTS")
counts=$(psql_tg -tAc \
  "SELECT count(*) FILTER (WHERE default_has_rows) || '|' ||
          count(*) FILTER (WHERE coalesce(days_ahead, -1) < 1)
     FROM ( $REPORT ) t")
IFS='|' read -r stranded behind <<<"$counts"
# premake < 7, or infinite_time_partitions off: without the latter partman
# refuses to premake forward across the gap the stall left behind.
misconfigured=$(psql_tg -tAc \
  "SELECT count(*) FROM partman.part_config
    WHERE $TELEGRAF_PARENTS
      AND (premake < $PREMAKE OR NOT infinite_time_partitions)")

# Both columns are matched because which one carries the launcher's name varies
# with the pg_cron version.
launcher=$(psql_pg -tAc \
  "SELECT count(*) FROM pg_stat_activity
    WHERE backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%'")

last_run=never
cron_installed=$(has_cron)
if [[ -n "$cron_installed" ]]; then
  run=$(psql_cron -tAc \
    "SELECT d.status || ' at ' || d.start_time
       FROM cron.job_run_details d JOIN cron.job j USING (jobid)
      WHERE j.jobname = '$JOB_NAME'
      ORDER BY d.start_time DESC LIMIT 1")
  if [[ -n "$run" ]]; then
    message=$(psql_cron -tAc \
      "SELECT coalesce(d.return_message, '')
         FROM cron.job_run_details d JOIN cron.job j USING (jobid)
        WHERE j.jobname = '$JOB_NAME'
        ORDER BY d.start_time DESC LIMIT 1")
    message=$(sanitise <<<"$message")
    message=${message//$'\n'/ }
    last_run="$(sanitise <<<"$run"): ${message:0:200}"
  fi
fi

# A grid that has never written a metric has nothing to recover, and an empty
# cron_db means pg_cron is not loaded at all, which no restart fixes.
restart_needed=false
[[ "$launcher" -eq 0 && "$parents" -gt 0 && -n "$cron_db" ]] && restart_needed=true

# group_role registers the job only in the postgres database.
job_inactive=false
if [[ "$cron_db" == postgres ]]; then
  [[ -z "$job" || "${job##*|}" == f ]] && job_inactive=true
fi

repair_needed=false
[[ "$stranded" -gt 0 ]] && repair_needed=true
[[ "$behind" -gt 0 ]] && repair_needed=true
[[ "$misconfigured" -gt 0 ]] && repair_needed=true
$restart_needed && repair_needed=true
$job_inactive && repair_needed=true

echo "Telegraf partition status:"
print_report
echo "Default partitions with rows: $stranded"
echo "pg_cron metadata database: $(sanitise <<<"${cron_db:-unset}")"
[[ "$cron_db" == postgres ]] \
  || echo "pg_cron metadata database is $(sanitise <<<"${cron_db:-unset}"); not checking the maintenance job."
echo "pg_cron launcher running: $([[ "$launcher" -gt 0 ]] && echo yes || echo no)"
echo "Last $JOB_NAME run: $last_run"
echo

if ! $repair_needed; then
  echo "Telegraf partitions are healthy. Nothing to do."
  finish 0
fi

if $CHECK_ONLY; then
  echo "Repair is needed:"
  [[ "$stranded" -gt 0 ]]    && echo "  * $stranded default partition(s) hold rows"
  [[ "$behind" -gt 0 ]]      && echo "  * $behind parent(s) with no partition for the current window"
  [[ "$misconfigured" -gt 0 ]] && echo "  * $misconfigured parent(s) with stale partman settings"
  $restart_needed            && echo "  * pg_cron's launcher is dead; maintenance is not running at all"
  $job_inactive              && echo "  * the maintenance job is missing or inactive"
  echo
  echo "Re-run without --check to repair."
  finish 1
fi

if [[ "$stranded" -gt 0 ]] && ! $ASSUME_YES; then
  echo "This will permanently discard the rows in the $stranded default partition(s) above."
  $restart_needed && ! $NO_RESTART && \
    echo "so-postgres will also be restarted, which briefly interrupts SOC."
  [[ -t 0 ]] || { echo "Not a terminal; re-run with --yes to confirm." >&2; finish 1; }
  read -r -p "Continue? [y/N] " answer
  [[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; finish 0; }
fi

ERR_STATUS=1
ERR_PREFIX="Telegraf partition repair failed"

if [[ "$cron_db" == postgres ]]; then
  if [[ -z "$job" ]]; then
    echo "Could not register the maintenance job; run a highstate and try again." >&2
    finish 3
  fi
  job_id=${job%%|*}
  [[ "${job##*|}" == t ]] && restore_active=true || restore_active=false
  echo "Pausing the $JOB_NAME job during the repair."
  psql_cron -tAc "SELECT cron.alter_job($job_id, active := false)" >/dev/null
fi

if [[ "$misconfigured" -gt 0 ]]; then
  echo "Reconciling partman settings on $misconfigured parent(s)."
  # GREATEST so an operator who raised premake further keeps their value.
  psql_tg -v ON_ERROR_STOP=1 -c \
    "UPDATE partman.part_config
        SET premake = GREATEST(premake, $PREMAKE),
            infinite_time_partitions = true
      WHERE $TELEGRAF_PARENTS"
fi

if [[ "$stranded" -gt 0 ]]; then
  echo "Clearing default partitions."
  # One transaction: Telegraf is still writing, so a default emptied without
  # its partition in place is refilled before maintenance can attach one.
  psql_tg -v ON_ERROR_STOP=1 <<'EOSQL'
BEGIN;
DO $$
DECLARE
    r record;
BEGIN
    FOR r IN
        SELECT pc.parent_table, d.default_part
        FROM so_admin.telegraf_defaults() d
        JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table
        WHERE d.default_part IS NOT NULL
    LOOP
        EXECUTE format('TRUNCATE TABLE %s', r.default_part);
        PERFORM partman.create_partition_time(
            r.parent_table, ARRAY[date_trunc('day', now())]::timestamptz[]);
    END LOOP;
END
$$;
COMMIT;
EOSQL
fi

if $restart_needed; then
  if $NO_RESTART; then
    echo "WARNING: pg_cron's launcher is dead and --no-restart was given."
    echo "         Maintenance will not run on its own until so-postgres is restarted."
  else
    echo "Restarting so-postgres to revive pg_cron's launcher."
    docker restart so-postgres >/dev/null
    for _ in $(seq 1 60); do
      docker exec so-postgres pg_isready -U postgres -q 2>/dev/null && break
      sleep 2
    done
    docker exec so-postgres pg_isready -U postgres -q \
      || { echo "so-postgres did not come back; check 'docker logs so-postgres'." >&2; finish 1; }
  fi
fi

echo "Running partition maintenance."
# so_admin.telegraf_maintenance() only exists once the postgres state has landed.
psql_tg -v ON_ERROR_STOP=1 <<'EOSQL'
SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL
            THEN 'true' ELSE 'false' END AS has_proc \gset
\if :has_proc
CALL so_admin.telegraf_maintenance();
\else
CALL partman.run_maintenance_proc();
\endif
EOSQL

if [[ -n "$job_id" ]]; then
  psql_cron -tAc "SELECT cron.alter_job($job_id, active := true)" >/dev/null
  restore_active=
  echo "The $JOB_NAME job is active."
fi

echo
echo "Telegraf partition status after repair:"
print_report
echo "The $JOB_NAME job runs hourly at :17. Confirm it fired with:"
echo "  so-telegraf-repair --check"
