diff --git a/salt/common/tools/sbin/so-status b/salt/common/tools/sbin/so-status index b50cdff14..18b714d70 100755 --- a/salt/common/tools/sbin/so-status +++ b/salt/common/tools/sbin/so-status @@ -17,6 +17,7 @@ import time EXPECTED_CONTAINERS_FILE = "/opt/so/conf/so-status/so-status.conf" HIGHSTATE_COMPLETE_FILE = "/opt/so/log/salt/lasthighstate" UPTIME_FILE = "/proc/uptime" +POSTGRES_MAINTENANCE_STATUS_FILE = "/opt/so/log/postgres/maintenance.status" def showUsage(options, args): print('Usage: {} [OPTIONS]'.format(sys.argv[0])) @@ -149,6 +150,20 @@ def check_container_status(options, console): return code, container_list +def check_postgres_maintenance(code, container_list): + try: + with open(POSTGRES_MAINTENANCE_STATUS_FILE, "r") as status_file: + status = status_file.readline().strip() + except OSError: + return code + if status.startswith("failed"): + reason = status[len("failed"):].lstrip(":").strip() + container_list.append({ "Name": "so-postgres-maintenance", "Status": "failed", "Details": reason }) + if code == 0: + code = 1 + return code + + def check_status(options, console): container_list = [] code = check_installation_status(options, console) @@ -160,6 +175,7 @@ def check_status(options, console): # on the Grid until a highstate completes. if code == 0 or code == 2: container_code, container_list = check_container_status(options, console) + container_code = check_postgres_maintenance(container_code, container_list) if code == 0: code = container_code output(options, console, code, container_list) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index ba27ca004..b5c27aa22 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -546,6 +546,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0 [[ "$POSTVERSION" == "3.2.0" ]] && post_to_3.3.0 [[ "$POSTVERSION" == "3.3.0" ]] && post_to_3.4.0 + [[ "$POSTVERSION" == "3.4.0" ]] && post_to_3.5.0 # All applicable post-upgrade steps completed; clear the resume marker. rm -f "$POSTVERSION_FILE" true @@ -1069,11 +1070,14 @@ telegraf_repair() { echo "Checking Telegraf metric partitions." local status=0 - "$repair" --check >> "$SOUP_LOG" 2>&1 || status=$? + local max_age + max_age=$(lookup_pillar maintenance_max_age_hours postgres:telegraf 2>/dev/null) || true + max_age=${max_age:-3} + MAINTENANCE_MAX_AGE_HOURS="$max_age" "$repair" --check >> "$SOUP_LOG" 2>&1 || status=$? case "$status" in 0) echo " Telegraf partitions are healthy; nothing to repair." ;; 1) echo " Repairing stalled Telegraf partitions." - "$repair" --yes \ + MAINTENANCE_MAX_AGE_HOURS="$max_age" "$repair" --yes \ || echo " warning: so-telegraf-repair failed; run it manually" >&2 ;; *) echo " Skipping; Telegraf is not storing metrics in Postgres on this host." ;; esac @@ -1202,6 +1206,14 @@ add_detection_runtime_fields() { } ### 3.4.0 End ### +### 3.5.0 Scripts ### +post_to_3.5.0() { + telegraf_repair + + set_postversion 3.5.0 +} +### 3.5.0 End ### + repo_sync() { echo "Sync the local repo." diff --git a/salt/postgres/defaults.yaml b/salt/postgres/defaults.yaml index 754a0d44c..c72b96109 100644 --- a/salt/postgres/defaults.yaml +++ b/salt/postgres/defaults.yaml @@ -2,6 +2,7 @@ postgres: enabled: True telegraf: retention_days: 14 + maintenance_max_age_hours: 3 config: listen_addresses: '*' port: 5432 @@ -17,3 +18,6 @@ postgres: log_min_messages: 'warning' shared_preload_libraries: pg_cron cron.database_name: postgres + # pg_hba.conf only trusts local socket connections; over TCP pg_cron has no + # password to send and every job fails to connect. + cron.host: /run/postgresql diff --git a/salt/postgres/disabled.sls b/salt/postgres/disabled.sls index 4b5b62328..38227ec96 100644 --- a/salt/postgres/disabled.sls +++ b/salt/postgres/disabled.sls @@ -24,6 +24,15 @@ so_postgres_backup: - identifier: so_postgres_backup - user: root +so_postgres_maintenance_status: + cron.absent: + - identifier: so_postgres_maintenance_status + - user: root + +so_postgres_maintenance_status_file: + file.absent: + - name: /opt/so/log/postgres/maintenance.status + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/postgres/enabled.sls b/salt/postgres/enabled.sls index 088ad6f54..100f975bc 100644 --- a/salt/postgres/enabled.sls +++ b/salt/postgres/enabled.sls @@ -8,6 +8,7 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'ca/trust.map.jinja' import CATRUST %} {% from 'docker/docker.map.jinja' import DOCKERMERGED %} +{% from 'telegraf/map.jinja' import TELEGRAFMERGED %} {% set SO_POSTGRES_USER = salt['pillar.get']('postgres:auth:users:so_postgres_user:user', 'so_postgres') %} include: @@ -122,6 +123,29 @@ so_postgres_backup: - month: '*' - dayweek: '*' +{% if TELEGRAFMERGED.output | upper in ['POSTGRES', 'BOTH'] %} +{% set max_age = salt['pillar.get']('postgres:telegraf:maintenance_max_age_hours', 3) | int %} +so_postgres_maintenance_status: + cron.present: + - name: MAINTENANCE_MAX_AGE_HOURS={{ max_age }} /usr/sbin/so-telegraf-postgres status > /dev/null 2>&1 + - identifier: so_postgres_maintenance_status + - user: root + - minute: '*/15' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' +{% else %} +so_postgres_maintenance_status: + cron.absent: + - identifier: so_postgres_maintenance_status + - user: root + +so_postgres_maintenance_status_file: + file.absent: + - name: /opt/so/log/postgres/maintenance.status +{% endif %} + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/postgres/soc_postgres.yaml b/salt/postgres/soc_postgres.yaml index 4ab1e4dac..47314e0bc 100644 --- a/salt/postgres/soc_postgres.yaml +++ b/salt/postgres/soc_postgres.yaml @@ -9,6 +9,11 @@ postgres: description: Number of days of Telegraf metrics to keep in the so_telegraf database. Older partitions are dropped hourly by pg_partman. forcedType: int helpLink: postgres + maintenance_max_age_hours: + description: Number of hours without a successful run of the hourly pg_partman maintenance job before the Telegraf metrics database is reported as unhealthy in grid status and repaired by soup. + forcedType: int + advanced: True + helpLink: postgres config: max_connections: description: Maximum number of concurrent PostgreSQL connections. @@ -87,3 +92,8 @@ postgres: global: True advanced: True helpLink: postgres + cron.host: + description: Unix socket directory pg_cron uses to run scheduled jobs. Must be a local socket directory so jobs authenticate with the local trust rule; a TCP host requires a password and jobs will fail. + global: True + advanced: True + helpLink: postgres diff --git a/salt/postgres/tools/sbin/so-telegraf-postgres b/salt/postgres/tools/sbin/so-telegraf-postgres index df7d16e65..7fbb42219 100644 --- a/salt/postgres/tools/sbin/so-telegraf-postgres +++ b/salt/postgres/tools/sbin/so-telegraf-postgres @@ -12,7 +12,11 @@ set -e # retention Reconcile partman retention and premake on telegraf parents. # Env: RETENTION_DAYS. # maintenance Drain default partitions and run partman maintenance. -# check Report partition health. Non-zero if any parent is unhealthy. +# check Report partition and maintenance health. Non-zero if any parent +# is unhealthy or the maintenance job has no successful run within +# the last MAINTENANCE_MAX_AGE_HOURS (default 3). +# status Run check and write `ok` or `failed: ` to +# /opt/so/log/postgres/maintenance.status for so-status. # # A default partition holding rows for a day blocks creating that day's child, # so maintenance drains defaults before calling partman. Use so-telegraf-repair @@ -189,6 +193,11 @@ SELECT cron.schedule_in_database( 'CALL so_admin.telegraf_maintenance()', 'so_telegraf' ); +-- pg_cron copies cron.host into nodename only when a job is first created, so +-- jobs registered before cron.host changed keep connecting to the old host. +UPDATE cron.job SET nodename = current_setting('cron.host') + WHERE jobname = 'telegraf-partman-maintenance' + AND nodename IS DISTINCT FROM current_setting('cron.host'); \else \echo 'pg_cron metadata database is not `postgres` yet; skipping job registration.' \endif @@ -278,16 +287,80 @@ LIMIT 5; \echo 'pg_cron is not installed in this database.' \endif EOSQL + max_age="${MAINTENANCE_MAX_AGE_HOURS:-3}" + [[ "$max_age" =~ ^[0-9]+$ ]] || { echo "MAINTENANCE_MAX_AGE_HOURS must be an integer" >&2; exit 1; } + failures=() unhealthy=$(docker exec so-postgres psql -U postgres -d so_telegraf -tAc \ "SELECT count(*) FROM so_admin.telegraf_partition_status() WHERE coalesce(default_rows, 0) > 0 OR coalesce(days_ahead, -1) < 1") if [ "${unhealthy:-1}" != "0" ]; then - echo "so-telegraf-postgres check: $unhealthy telegraf parent(s) unhealthy" >&2 + failures+=("$unhealthy telegraf parent(s) unhealthy") + fi + has_cron=$(docker exec so-postgres psql -U postgres -d postgres -tAc \ + "SELECT to_regclass('cron.job_run_details') IS NOT NULL") + if [ "$has_cron" != "t" ]; then + failures+=("pg_cron is not installed in the postgres database") + else + runs=$(docker exec so-postgres psql -U postgres -d postgres -tA -F '|' -c \ + "SELECT (SELECT count(*) FROM cron.job WHERE jobname = 'telegraf-partman-maintenance'), + (SELECT count(*) FROM cron.job WHERE jobname = 'telegraf-partman-maintenance' AND active), + (SELECT count(*) FROM pg_stat_activity + WHERE backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%'), + (SELECT count(*) FROM pg_stat_activity + WHERE (backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%') + AND backend_start < now() - make_interval(hours => $max_age)), + count(*), + count(*) FILTER (WHERE d.status = 'succeeded' + AND d.start_time > now() - make_interval(hours => $max_age)), + coalesce((array_agg(format('last run %s at %s: %s', d.status, d.start_time, + regexp_replace(coalesce(d.return_message, ''), '\s+', ' ', 'g')) + ORDER BY d.start_time DESC))[1], '') + FROM cron.job j + JOIN cron.job_run_details d ON d.jobid = j.jobid + WHERE j.jobname = 'telegraf-partman-maintenance'") || runs="" + IFS='|' read -r jobs active launchers old_launchers run_count recent_ok last_run <<< "$runs" + # With no runs recorded, a live launcher younger than N hours means the + # job was only just scheduled (fresh install or a restart for a config + # change) and has not reached its first :17 yet. A launcher older than + # that should have fired it by now. Tradeoff: a job first scheduled long + # after the launcher started reports failed until its first run. + if [ -z "$runs" ]; then + failures+=("could not read telegraf-partman-maintenance runs from cron.job_run_details") + elif [ "$jobs" = "0" ]; then + failures+=("telegraf-partman-maintenance is not scheduled") + elif [ "$active" = "0" ]; then + failures+=("telegraf-partman-maintenance is inactive") + elif [ "$launchers" = "0" ]; then + failures+=("pg_cron launcher is not running; telegraf-partman-maintenance cannot run") + elif [ "$run_count" = "0" ]; then + if [ "$old_launchers" != "0" ]; then + failures+=("telegraf-partman-maintenance has never run although pg_cron has been up for more than $max_age hour(s)") + fi + elif [ "$recent_ok" = "0" ]; then + failures+=("no successful telegraf-partman-maintenance run in the last $max_age hour(s); $last_run") + fi + fi + if [ "${#failures[@]}" -gt 0 ]; then + msg=$(printf '%s; ' "${failures[@]}") + echo "so-telegraf-postgres check: ${msg%; }" >&2 exit 1 fi echo "so-telegraf-postgres check: all telegraf parents healthy" ;; + status) + status_file=/opt/so/log/postgres/maintenance.status + tmp=$(mktemp "$status_file.XXXXXX") + if reason=$("$0" check 2>&1 >/dev/null); then + echo ok > "$tmp" + else + reason=$(printf '%s\n' "$reason" | tail -n 1) + echo "failed: ${reason#so-telegraf-postgres check: }" > "$tmp" + fi + chmod 644 "$tmp" + mv -f "$tmp" "$status_file" + ;; + *) echo "Unknown subcommand: $cmd" >&2 exit 1 diff --git a/salt/postgres/tools/sbin/so-telegraf-repair b/salt/postgres/tools/sbin/so-telegraf-repair index 75004402e..0f0ecde21 100644 --- a/salt/postgres/tools/sbin/so-telegraf-repair +++ b/salt/postgres/tools/sbin/so-telegraf-repair @@ -15,6 +15,9 @@ # --yes Skip the confirmation prompt (for soup and other automation). # --no-restart Never restart so-postgres, even if pg_cron's launcher is dead. # +# Env: MAINTENANCE_MAX_AGE_HOURS (default 3) is how long the maintenance job may +# go without a successful run before repair is needed. +# # Exit status: # 0 healthy, or repair completed # 1 repair is needed (--check only) @@ -25,6 +28,8 @@ set -e # Matches p_premake in telegraf.conf's create_parent template. PREMAKE=7 JOB_NAME=telegraf-partman-maintenance +MAX_AGE_HOURS="${MAINTENANCE_MAX_AGE_HOURS:-3}" +[[ "$MAX_AGE_HOURS" =~ ^[0-9]+$ ]] || { echo "MAINTENANCE_MAX_AGE_HOURS must be an integer" >&2; exit 2; } CHECK_ONLY=false ASSUME_YES=false @@ -118,14 +123,30 @@ launcher=$(psql_pg -tAc \ # so_telegraf before the postgres state lands, postgres after. cron_db=$(docker exec so-postgres psql -U postgres -tAc \ "SELECT current_setting('cron.database_name', true)" | tr -d '[:space:]') +run_count=0 +recent_ok=0 last_run=never -if [[ -n "$cron_db" ]]; then - last_run=$(docker exec so-postgres psql -U postgres -d "$cron_db" -tAc \ - "SELECT coalesce(max(d.start_time)::text, 'never') +has_run_details=$([[ -n "$cron_db" ]] && docker exec so-postgres psql -U postgres -d "$cron_db" -tAc \ + "SELECT to_regclass('cron.job_run_details') IS NOT NULL" 2>/dev/null || echo f) +if [[ "$has_run_details" == t ]]; then + runs=$(docker exec so-postgres psql -U postgres -d "$cron_db" -tA -F '|' -c \ + "SELECT count(*), + count(*) FILTER (WHERE d.status = 'succeeded' + AND d.start_time > now() - make_interval(hours => $MAX_AGE_HOURS)), + coalesce((array_agg(format('%s at %s', d.status, d.start_time) + ORDER BY d.start_time DESC))[1], 'never') FROM cron.job_run_details d JOIN cron.job j USING (jobid) - WHERE j.jobname = '$JOB_NAME'" 2>/dev/null | tr -d '[:space:]' || echo unknown) - [[ -n "$last_run" ]] || last_run=never + WHERE j.jobname = '$JOB_NAME'" 2>/dev/null) \ + || { echo "Could not read $JOB_NAME runs from cron.job_run_details; cannot judge maintenance health." >&2; exit 2; } + IFS='|' read -r run_count recent_ok last_run <<< "$runs" fi +# An unreadable run history exits 2 rather than 1: soup repairs on 1, which +# truncates the default partitions, and a transient query error must not do that. +# No runs at all means a freshly scheduled job (or pg_cron not loaded, handled +# by restart_needed); the partition checks catch a job that never fires. Runs +# with no recent success mean the job fires but fails, e.g. cannot connect. +maintenance_failing=false +[[ "$run_count" -gt 0 && "$recent_ok" -eq 0 ]] && maintenance_failing=true # A grid that has never written a metric has nothing to recover, and an empty # cron_db means pg_cron is not loaded at all, which no restart fixes. @@ -137,6 +158,7 @@ repair_needed=false [[ "$behind" -gt 0 ]] && repair_needed=true [[ "$misconfigured" -gt 0 ]] && repair_needed=true $restart_needed && repair_needed=true +$maintenance_failing && repair_needed=true echo "Telegraf partition status:" psql_tg -c "$REPORT" @@ -157,6 +179,7 @@ if $CHECK_ONLY; then [[ "$behind" -gt 0 ]] && echo " * $behind parent(s) with no partition for the current window" [[ "$misconfigured" -gt 0 ]] && echo " * $misconfigured parent(s) with stale partman settings" $restart_needed && echo " * pg_cron's launcher is dead; maintenance is not running at all" + $maintenance_failing && echo " * no successful $JOB_NAME run in the last $MAX_AGE_HOURS hour(s)" echo echo "Re-run without --check to repair." exit 1 @@ -241,5 +264,6 @@ EOSQL echo echo "Telegraf partition status after repair:" psql_tg -c "$REPORT" -echo "The $JOB_NAME job runs hourly at :17. Confirm it fired with:" +echo "The $JOB_NAME job runs hourly at :17. Until its next run succeeds, grid status" +echo "may still report so-postgres-maintenance as failed. Confirm it fired with:" echo " so-telegraf-repair --check"