Postgres: run pg_partman maintenance over the local socket and detect failed runs

salt/common/tools/sbin/so-status

@@ -17,6 +17,7 @@ import time
 EXPECTED_CONTAINERS_FILE = "/opt/so/conf/so-status/so-status.conf"
 HIGHSTATE_COMPLETE_FILE = "/opt/so/log/salt/lasthighstate"
 UPTIME_FILE = "/proc/uptime"
+POSTGRES_MAINTENANCE_STATUS_FILE = "/opt/so/log/postgres/maintenance.status"
 
 def showUsage(options, args):
   print('Usage: {} [OPTIONS]'.format(sys.argv[0]))
@@ -149,6 +150,20 @@ def check_container_status(options, console):
   return code, container_list
 
 
+def check_postgres_maintenance(code, container_list):
+  try:
+    with open(POSTGRES_MAINTENANCE_STATUS_FILE, "r") as status_file:
+      status = status_file.readline().strip()
+  except OSError:
+    return code
+  if status.startswith("failed"):
+    reason = status[len("failed"):].lstrip(":").strip()
+    container_list.append({ "Name": "so-postgres-maintenance", "Status": "failed", "Details": reason })
+    if code == 0:
+      code = 1
+  return code
+
+
 def check_status(options, console):
   container_list = []
   code = check_installation_status(options, console)
@@ -160,6 +175,7 @@ def check_status(options, console):
   # on the Grid until a highstate completes.
   if code == 0 or code == 2:
     container_code, container_list = check_container_status(options, console)
+    container_code = check_postgres_maintenance(container_code, container_list)
     if code == 0:
       code = container_code
   output(options, console, code, container_list)

salt/manager/tools/sbin/soup

@@ -546,6 +546,7 @@ postupgrade_changes() {
     [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0
     [[ "$POSTVERSION" == "3.2.0" ]] && post_to_3.3.0
     [[ "$POSTVERSION" == "3.3.0" ]] && post_to_3.4.0
+    [[ "$POSTVERSION" == "3.4.0" ]] && post_to_3.5.0
     # All applicable post-upgrade steps completed; clear the resume marker.
     rm -f "$POSTVERSION_FILE"
     true
@@ -1069,11 +1070,14 @@ telegraf_repair() {
 
   echo "Checking Telegraf metric partitions."
   local status=0
-  "$repair" --check >> "$SOUP_LOG" 2>&1 || status=$?
+  local max_age
+  max_age=$(lookup_pillar maintenance_max_age_hours postgres:telegraf 2>/dev/null) || true
+  max_age=${max_age:-3}
+  MAINTENANCE_MAX_AGE_HOURS="$max_age" "$repair" --check >> "$SOUP_LOG" 2>&1 || status=$?
   case "$status" in
     0) echo "  Telegraf partitions are healthy; nothing to repair." ;;
     1) echo "  Repairing stalled Telegraf partitions."
-       "$repair" --yes \
+       MAINTENANCE_MAX_AGE_HOURS="$max_age" "$repair" --yes \
          || echo "  warning: so-telegraf-repair failed; run it manually" >&2 ;;
     *) echo "  Skipping; Telegraf is not storing metrics in Postgres on this host." ;;
   esac
@@ -1202,6 +1206,14 @@ add_detection_runtime_fields() {
 }
 ### 3.4.0 End ###
 
+### 3.5.0 Scripts ###
+post_to_3.5.0() {
+  telegraf_repair
+
+  set_postversion 3.5.0
+}
+### 3.5.0 End ###
+
 
 repo_sync() {
   echo "Sync the local repo."

salt/postgres/defaults.yaml

@@ -2,6 +2,7 @@ postgres:
   enabled: True
   telegraf:
     retention_days: 14
+    maintenance_max_age_hours: 3
   config:
     listen_addresses: '*'
     port: 5432
@@ -17,3 +18,6 @@ postgres:
     log_min_messages: 'warning'
     shared_preload_libraries: pg_cron
     cron.database_name: postgres
+    # pg_hba.conf only trusts local socket connections; over TCP pg_cron has no
+    # password to send and every job fails to connect.
+    cron.host: /run/postgresql

salt/postgres/disabled.sls

@@ -24,6 +24,15 @@ so_postgres_backup:
     - identifier: so_postgres_backup
     - user: root
 
+so_postgres_maintenance_status:
+  cron.absent:
+    - identifier: so_postgres_maintenance_status
+    - user: root
+
+so_postgres_maintenance_status_file:
+  file.absent:
+    - name: /opt/so/log/postgres/maintenance.status
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/postgres/enabled.sls

@@ -8,6 +8,7 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   from 'telegraf/map.jinja' import TELEGRAFMERGED %}
 {%   set SO_POSTGRES_USER = salt['pillar.get']('postgres:auth:users:so_postgres_user:user', 'so_postgres') %}
 
 include:
@@ -122,6 +123,29 @@ so_postgres_backup:
     - month: '*'
     - dayweek: '*'
 
+{% if TELEGRAFMERGED.output | upper in ['POSTGRES', 'BOTH'] %}
+{%   set max_age = salt['pillar.get']('postgres:telegraf:maintenance_max_age_hours', 3) | int %}
+so_postgres_maintenance_status:
+  cron.present:
+    - name: MAINTENANCE_MAX_AGE_HOURS={{ max_age }} /usr/sbin/so-telegraf-postgres status > /dev/null 2>&1
+    - identifier: so_postgres_maintenance_status
+    - user: root
+    - minute: '*/15'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+{% else %}
+so_postgres_maintenance_status:
+  cron.absent:
+    - identifier: so_postgres_maintenance_status
+    - user: root
+
+so_postgres_maintenance_status_file:
+  file.absent:
+    - name: /opt/so/log/postgres/maintenance.status
+{% endif %}
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/postgres/soc_postgres.yaml

@@ -9,6 +9,11 @@ postgres:
       description: Number of days of Telegraf metrics to keep in the so_telegraf database. Older partitions are dropped hourly by pg_partman.
       forcedType: int
       helpLink: postgres
+    maintenance_max_age_hours:
+      description: Number of hours without a successful run of the hourly pg_partman maintenance job before the Telegraf metrics database is reported as unhealthy in grid status and repaired by soup.
+      forcedType: int
+      advanced: True
+      helpLink: postgres
   config:
     max_connections:
       description: Maximum number of concurrent PostgreSQL connections.
@@ -87,3 +92,8 @@ postgres:
       global: True
       advanced: True
       helpLink: postgres
+    cron.host:
+      description: Unix socket directory pg_cron uses to run scheduled jobs. Must be a local socket directory so jobs authenticate with the local trust rule; a TCP host requires a password and jobs will fail.
+      global: True
+      advanced: True
+      helpLink: postgres

salt/postgres/tools/sbin/so-telegraf-postgres

@@ -12,7 +12,11 @@ set -e
 #   retention    Reconcile partman retention and premake on telegraf parents.
 #                Env: RETENTION_DAYS.
 #   maintenance  Drain default partitions and run partman maintenance.
-#   check        Report partition health. Non-zero if any parent is unhealthy.
+#   check        Report partition and maintenance health. Non-zero if any parent
+#                is unhealthy or the maintenance job has no successful run within
+#                the last MAINTENANCE_MAX_AGE_HOURS (default 3).
+#   status       Run check and write `ok` or `failed: <reason>` to
+#                /opt/so/log/postgres/maintenance.status for so-status.
 #
 # A default partition holding rows for a day blocks creating that day's child,
 # so maintenance drains defaults before calling partman. Use so-telegraf-repair
@@ -189,6 +193,11 @@ SELECT cron.schedule_in_database(
   'CALL so_admin.telegraf_maintenance()',
   'so_telegraf'
 );
+-- pg_cron copies cron.host into nodename only when a job is first created, so
+-- jobs registered before cron.host changed keep connecting to the old host.
+UPDATE cron.job SET nodename = current_setting('cron.host')
+ WHERE jobname = 'telegraf-partman-maintenance'
+   AND nodename IS DISTINCT FROM current_setting('cron.host');
 \else
 \echo 'pg_cron metadata database is not `postgres` yet; skipping job registration.'
 \endif
@@ -278,16 +287,80 @@ LIMIT 5;
 \echo 'pg_cron is not installed in this database.'
 \endif
 EOSQL
+    max_age="${MAINTENANCE_MAX_AGE_HOURS:-3}"
+    [[ "$max_age" =~ ^[0-9]+$ ]] || { echo "MAINTENANCE_MAX_AGE_HOURS must be an integer" >&2; exit 1; }
+    failures=()
     unhealthy=$(docker exec so-postgres psql -U postgres -d so_telegraf -tAc \
       "SELECT count(*) FROM so_admin.telegraf_partition_status()
        WHERE coalesce(default_rows, 0) > 0 OR coalesce(days_ahead, -1) < 1")
     if [ "${unhealthy:-1}" != "0" ]; then
-      echo "so-telegraf-postgres check: $unhealthy telegraf parent(s) unhealthy" >&2
+      failures+=("$unhealthy telegraf parent(s) unhealthy")
+    fi
+    has_cron=$(docker exec so-postgres psql -U postgres -d postgres -tAc \
+      "SELECT to_regclass('cron.job_run_details') IS NOT NULL")
+    if [ "$has_cron" != "t" ]; then
+      failures+=("pg_cron is not installed in the postgres database")
+    else
+      runs=$(docker exec so-postgres psql -U postgres -d postgres -tA -F '|' -c \
+        "SELECT (SELECT count(*) FROM cron.job WHERE jobname = 'telegraf-partman-maintenance'),
+                (SELECT count(*) FROM cron.job WHERE jobname = 'telegraf-partman-maintenance' AND active),
+                (SELECT count(*) FROM pg_stat_activity
+                  WHERE backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%'),
+                (SELECT count(*) FROM pg_stat_activity
+                  WHERE (backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%')
+                    AND backend_start < now() - make_interval(hours => $max_age)),
+                count(*),
+                count(*) FILTER (WHERE d.status = 'succeeded'
+                                   AND d.start_time > now() - make_interval(hours => $max_age)),
+                coalesce((array_agg(format('last run %s at %s: %s', d.status, d.start_time,
+                                           regexp_replace(coalesce(d.return_message, ''), '\s+', ' ', 'g'))
+                                    ORDER BY d.start_time DESC))[1], '')
+           FROM cron.job j
+           JOIN cron.job_run_details d ON d.jobid = j.jobid
+          WHERE j.jobname = 'telegraf-partman-maintenance'") || runs=""
+      IFS='|' read -r jobs active launchers old_launchers run_count recent_ok last_run <<< "$runs"
+      # With no runs recorded, a live launcher younger than N hours means the
+      # job was only just scheduled (fresh install or a restart for a config
+      # change) and has not reached its first :17 yet. A launcher older than
+      # that should have fired it by now. Tradeoff: a job first scheduled long
+      # after the launcher started reports failed until its first run.
+      if [ -z "$runs" ]; then
+        failures+=("could not read telegraf-partman-maintenance runs from cron.job_run_details")
+      elif [ "$jobs" = "0" ]; then
+        failures+=("telegraf-partman-maintenance is not scheduled")
+      elif [ "$active" = "0" ]; then
+        failures+=("telegraf-partman-maintenance is inactive")
+      elif [ "$launchers" = "0" ]; then
+        failures+=("pg_cron launcher is not running; telegraf-partman-maintenance cannot run")
+      elif [ "$run_count" = "0" ]; then
+        if [ "$old_launchers" != "0" ]; then
+          failures+=("telegraf-partman-maintenance has never run although pg_cron has been up for more than $max_age hour(s)")
+        fi
+      elif [ "$recent_ok" = "0" ]; then
+        failures+=("no successful telegraf-partman-maintenance run in the last $max_age hour(s); $last_run")
+      fi
+    fi
+    if [ "${#failures[@]}" -gt 0 ]; then
+      msg=$(printf '%s; ' "${failures[@]}")
+      echo "so-telegraf-postgres check: ${msg%; }" >&2
       exit 1
     fi
     echo "so-telegraf-postgres check: all telegraf parents healthy"
     ;;
 
+  status)
+    status_file=/opt/so/log/postgres/maintenance.status
+    tmp=$(mktemp "$status_file.XXXXXX")
+    if reason=$("$0" check 2>&1 >/dev/null); then
+      echo ok > "$tmp"
+    else
+      reason=$(printf '%s\n' "$reason" | tail -n 1)
+      echo "failed: ${reason#so-telegraf-postgres check: }" > "$tmp"
+    fi
+    chmod 644 "$tmp"
+    mv -f "$tmp" "$status_file"
+    ;;
+
   *)
     echo "Unknown subcommand: $cmd" >&2
     exit 1

salt/postgres/tools/sbin/so-telegraf-repair

@@ -15,6 +15,9 @@
 #   --yes         Skip the confirmation prompt (for soup and other automation).
 #   --no-restart  Never restart so-postgres, even if pg_cron's launcher is dead.
 #
+# Env: MAINTENANCE_MAX_AGE_HOURS (default 3) is how long the maintenance job may
+# go without a successful run before repair is needed.
+#
 # Exit status:
 #   0  healthy, or repair completed
 #   1  repair is needed (--check only)
@@ -25,6 +28,8 @@ set -e
 # Matches p_premake in telegraf.conf's create_parent template.
 PREMAKE=7
 JOB_NAME=telegraf-partman-maintenance
+MAX_AGE_HOURS="${MAINTENANCE_MAX_AGE_HOURS:-3}"
+[[ "$MAX_AGE_HOURS" =~ ^[0-9]+$ ]] || { echo "MAINTENANCE_MAX_AGE_HOURS must be an integer" >&2; exit 2; }
 
 CHECK_ONLY=false
 ASSUME_YES=false
@@ -118,14 +123,30 @@ launcher=$(psql_pg -tAc \
 # so_telegraf before the postgres state lands, postgres after.
 cron_db=$(docker exec so-postgres psql -U postgres -tAc \
   "SELECT current_setting('cron.database_name', true)" | tr -d '[:space:]')
+run_count=0
+recent_ok=0
 last_run=never
-if [[ -n "$cron_db" ]]; then
-  last_run=$(docker exec so-postgres psql -U postgres -d "$cron_db" -tAc \
-    "SELECT coalesce(max(d.start_time)::text, 'never')
+has_run_details=$([[ -n "$cron_db" ]] && docker exec so-postgres psql -U postgres -d "$cron_db" -tAc \
+  "SELECT to_regclass('cron.job_run_details') IS NOT NULL" 2>/dev/null || echo f)
+if [[ "$has_run_details" == t ]]; then
+  runs=$(docker exec so-postgres psql -U postgres -d "$cron_db" -tA -F '|' -c \
+    "SELECT count(*),
+            count(*) FILTER (WHERE d.status = 'succeeded'
+                               AND d.start_time > now() - make_interval(hours => $MAX_AGE_HOURS)),
+            coalesce((array_agg(format('%s at %s', d.status, d.start_time)
+                                ORDER BY d.start_time DESC))[1], 'never')
        FROM cron.job_run_details d JOIN cron.job j USING (jobid)
-      WHERE j.jobname = '$JOB_NAME'" 2>/dev/null | tr -d '[:space:]' || echo unknown)
-  [[ -n "$last_run" ]] || last_run=never
+      WHERE j.jobname = '$JOB_NAME'" 2>/dev/null) \
+    || { echo "Could not read $JOB_NAME runs from cron.job_run_details; cannot judge maintenance health." >&2; exit 2; }
+  IFS='|' read -r run_count recent_ok last_run <<< "$runs"
 fi
+# An unreadable run history exits 2 rather than 1: soup repairs on 1, which
+# truncates the default partitions, and a transient query error must not do that.
+# No runs at all means a freshly scheduled job (or pg_cron not loaded, handled
+# by restart_needed); the partition checks catch a job that never fires. Runs
+# with no recent success mean the job fires but fails, e.g. cannot connect.
+maintenance_failing=false
+[[ "$run_count" -gt 0 && "$recent_ok" -eq 0 ]] && maintenance_failing=true
 
 # A grid that has never written a metric has nothing to recover, and an empty
 # cron_db means pg_cron is not loaded at all, which no restart fixes.
@@ -137,6 +158,7 @@ repair_needed=false
 [[ "$behind" -gt 0 ]] && repair_needed=true
 [[ "$misconfigured" -gt 0 ]] && repair_needed=true
 $restart_needed && repair_needed=true
+$maintenance_failing && repair_needed=true
 
 echo "Telegraf partition status:"
 psql_tg -c "$REPORT"
@@ -157,6 +179,7 @@ if $CHECK_ONLY; then
   [[ "$behind" -gt 0 ]]      && echo "  * $behind parent(s) with no partition for the current window"
   [[ "$misconfigured" -gt 0 ]] && echo "  * $misconfigured parent(s) with stale partman settings"
   $restart_needed            && echo "  * pg_cron's launcher is dead; maintenance is not running at all"
+  $maintenance_failing       && echo "  * no successful $JOB_NAME run in the last $MAX_AGE_HOURS hour(s)"
   echo
   echo "Re-run without --check to repair."
   exit 1
@@ -241,5 +264,6 @@ EOSQL
 echo
 echo "Telegraf partition status after repair:"
 psql_tg -c "$REPORT"
-echo "The $JOB_NAME job runs hourly at :17. Confirm it fired with:"
+echo "The $JOB_NAME job runs hourly at :17. Until its next run succeeds, grid status"
+echo "may still report so-postgres-maintenance as failed. Confirm it fired with:"
 echo "  so-telegraf-repair --check"