Run zeekctl cron when cron-only zeekctl settings change

LogExpireInterval, StatsLogExpireInterval, CrashExpireInterval, MinDiskSpace
and MailHostUpDown are only acted on by ZeekControl's 'cron' command, which
nothing ever ran, so setting them in SOC had no effect.

Track those five in /opt/so/state/zeek/zeekctl_cron_settings and trigger the
cron run off that file, so unrelated edits like CompressLogs don't fire it.
Runs as the zeek user to avoid root-owned files under /nsm/zeek, with
--no-watch so the crashed-node watchdog doesn't race the container restart.

salt/common/tools/sbin/so-log-check

@@ -169,6 +169,11 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error while parsing document for index \[.ds-logs-kratos-so-.*object mapping for \[file\]" # false positive (mapping error occuring BEFORE kratos index has rolled over in 2.4.210)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|No such container"            # false positive (telegraf trying to run stats on an old container)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|passwords do not match"       # false positive (automated hydra test)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Request did not pass preprocessing" # expected WARN log lines indicating invalid auth header
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Missing or invalid authorization header for bearer token" # expected WARN log lines indicating invalid auth header
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unexpected authorization header" # expected WARN log lines indicating invalid auth header
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Missing ory_kratos_session cookie" # expected WARN log lines indicating invalid auth header
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Static assets preprocessor only supports GET and HEAD requests" # expected WARN log lines indicating invalid auth header
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then

salt/manager/tools/sbin/soup

@@ -1041,9 +1041,30 @@ up_to_3.3.0() {
   INSTALLEDVERSION=3.3.0
 }
 
+telegraf_repair() {
+  # Only grids whose Telegraf partitions stalled need this; --check exits 1
+  # when there is something to repair, so everyone else is left alone.
+  local repair=/usr/sbin/so-telegraf-repair
+  [[ -x "$repair" ]] || return 0
+  docker ps --format '{{.Names}}' | grep -qx so-postgres || return 0
+
+  echo "Checking Telegraf metric partitions."
+  local status=0
+  "$repair" --check >> "$SOUP_LOG" 2>&1 || status=$?
+  case "$status" in
+    0) echo "  Telegraf partitions are healthy; nothing to repair." ;;
+    1) echo "  Repairing stalled Telegraf partitions."
+       "$repair" --yes \
+         || echo "  warning: so-telegraf-repair failed; run it manually" >&2 ;;
+    *) echo "  Skipping; Telegraf is not storing metrics in Postgres on this host." ;;
+  esac
+}
+
 post_to_3.3.0() {
   # Recollate again since some internal DBs were excluded during 3.2.0 soup
   recollate_postgres
+
+  telegraf_repair
 }
 ### 3.3.0 End ###
 

salt/postgres/defaults.yaml

@@ -16,4 +16,4 @@ postgres:
     logging_collector: 'off'
     log_min_messages: 'warning'
     shared_preload_libraries: pg_cron
-    cron.database_name: so_telegraf
+    cron.database_name: postgres

salt/postgres/soc_postgres.yaml

@@ -83,7 +83,7 @@ postgres:
       advanced: True
       helpLink: postgres
     cron.database_name:
-      description: Database pg_cron schedules jobs in. Must be so_telegraf so partman maintenance runs in the right database context.
+      description: Database pg_cron keeps its job metadata in. Must already exist when PostgreSQL starts, because pg_cron's launcher connects to it at startup and never retries if it is missing. The maintenance job itself targets so_telegraf.
       global: True
       advanced: True
       helpLink: postgres

salt/postgres/tools/sbin/so-postgres-backup

@@ -47,7 +47,12 @@ trap 'rm -f "$TMPFILE"' EXIT
 
 # Dump all databases and roles, compress. Write to a temp file so the final
 # filename only ever appears for a complete, verified backup.
-if ! docker exec so-postgres pg_dumpall -U postgres | gzip > "$TMPFILE"; then
+#
+# so_telegraf is excluded: it is transient metrics on a short retention window,
+# it dominates the dump size, and it is rebuilt automatically after a restore --
+# init-db.sh recreates the database and Telegraf recreates its tables on first
+# write. Roles are globals, so the per-minion telegraf logins are still dumped.
+if ! docker exec so-postgres pg_dumpall -U postgres --exclude-database=so_telegraf | gzip > "$TMPFILE"; then
   log "ERROR: pg_dumpall/gzip failed; backup aborted"
   exit 1
 fi

salt/postgres/tools/sbin/so-telegraf-postgres

@@ -5,11 +5,18 @@ set -e
 # Usage: so-telegraf-postgres <subcommand>
 #   create_db    Ensure the so_telegraf database exists.
 #   group_role   Provision the so_telegraf group role, telegraf/partman schemas,
-#                pg_partman, pg_cron, and the hourly partman maintenance job.
+#                pg_partman, the so_admin maintenance routines, and the hourly
+#                pg_cron maintenance job.
 #   user         Create or update a per-minion login role granted to so_telegraf.
 #                Env: ROLE_USER, ROLE_PASS.
-#   retention    Reconcile partman retention on telegraf parents.
+#   retention    Reconcile partman retention and premake on telegraf parents.
 #                Env: RETENTION_DAYS.
+#   maintenance  Drain default partitions and run partman maintenance.
+#   check        Report partition health. Non-zero if any parent is unhealthy.
+#
+# A default partition holding rows for a day blocks creating that day's child,
+# so maintenance drains defaults before calling partman. Use so-telegraf-repair
+# on a grid already stuck in that state.
 
 cmd="${1:?subcommand required}"
 
@@ -36,7 +43,6 @@ CREATE SCHEMA IF NOT EXISTS telegraf AUTHORIZATION so_telegraf;
 GRANT USAGE, CREATE ON SCHEMA telegraf TO so_telegraf;
 CREATE SCHEMA IF NOT EXISTS partman;
 CREATE EXTENSION IF NOT EXISTS pg_partman SCHEMA partman;
-CREATE EXTENSION IF NOT EXISTS pg_cron;
 -- Telegraf (running as so_telegraf) calls partman.create_parent()
 -- on first write of each metric, which needs USAGE on the partman
 -- schema, EXECUTE on its functions/procedures, and write access to
@@ -51,12 +57,141 @@ ALTER DEFAULT PRIVILEGES IN SCHEMA partman
     GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO so_telegraf;
 ALTER DEFAULT PRIVILEGES IN SCHEMA partman
     GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO so_telegraf;
--- Hourly partman maintenance. cron.schedule is idempotent by jobname.
-SELECT cron.schedule(
+
+-- pg_cron runs these as postgres, so they must not sit in a schema any
+-- Telegraf role can create objects in.
+CREATE SCHEMA IF NOT EXISTS so_admin AUTHORIZATION postgres;
+REVOKE ALL ON SCHEMA so_admin FROM PUBLIC;
+
+CREATE OR REPLACE PROCEDURE so_admin.telegraf_maintenance()
+LANGUAGE plpgsql
+AS $proc$
+DECLARE
+    r          record;
+    v_default  text;
+    v_rows     bigint;
+BEGIN
+    -- No per-parent EXCEPTION handler: partition_data_proc commits internally,
+    -- and COMMIT is illegal while a subtransaction is active. A failing parent
+    -- aborts the run and the next pass retries.
+    FOR r IN
+        SELECT parent_table, retention
+        FROM partman.part_config
+        WHERE parent_table LIKE 'telegraf.%'
+        ORDER BY parent_table
+    LOOP
+        v_default := format('%I.%I',
+                            split_part(r.parent_table, '.', 1),
+                            split_part(r.parent_table, '.', 2) || '_default');
+
+        CONTINUE WHEN to_regclass(v_default) IS NULL;
+
+        EXECUTE format('SELECT count(*) FROM %s', v_default) INTO v_rows;
+        CONTINUE WHEN v_rows = 0;
+
+        RAISE WARNING 'so_admin.telegraf_maintenance: % rows stranded in %, draining',
+                      v_rows, v_default;
+
+        -- Cheaper to delete expired rows than to repartition and then drop them.
+        IF r.retention IS NOT NULL THEN
+            EXECUTE format('DELETE FROM %s WHERE "time" < now() - %L::interval',
+                           v_default, r.retention);
+            COMMIT;
+        END IF;
+
+        -- Bounded so a large backlog drains across several runs.
+        CALL partman.partition_data_proc(
+            p_parent_table := r.parent_table,
+            p_loop_count   := 200,
+            p_source_table := v_default
+        );
+        COMMIT;
+    END LOOP;
+
+    CALL partman.run_maintenance_proc();
+END;
+$proc$;
+
+CREATE OR REPLACE FUNCTION so_admin.telegraf_partition_status()
+RETURNS TABLE (
+    parent_table  text,
+    oldest_child  date,
+    newest_child  date,
+    days_ahead    int,
+    retention     text,
+    default_rows  bigint,
+    default_size  text
+)
+LANGUAGE plpgsql
+AS $func$
+DECLARE
+    r          record;
+    v_default  regclass;
+BEGIN
+    FOR r IN
+        SELECT pc.parent_table AS pt, pc.retention AS ret
+        FROM partman.part_config pc
+        WHERE pc.parent_table LIKE 'telegraf.%'
+        ORDER BY pc.parent_table
+    LOOP
+        parent_table := r.pt;
+        retention    := r.ret;
+
+        SELECT min(d), max(d) INTO oldest_child, newest_child
+        FROM (
+            SELECT to_date(substring(c.relname FROM '_p(\d{8})$'), 'YYYYMMDD') AS d
+            FROM pg_inherits i
+            JOIN pg_class c ON c.oid = i.inhrelid
+            WHERE i.inhparent = r.pt::regclass
+              AND pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT'
+        ) s;
+
+        days_ahead := newest_child - current_date;
+
+        v_default := to_regclass(format('%I.%I',
+                                        split_part(r.pt, '.', 1),
+                                        split_part(r.pt, '.', 2) || '_default'));
+        IF v_default IS NULL THEN
+            default_rows := NULL;
+            default_size := NULL;
+        ELSE
+            EXECUTE format('SELECT count(*) FROM %s', v_default::text) INTO default_rows;
+            default_size := pg_size_pretty(pg_total_relation_size(v_default));
+        END IF;
+
+        RETURN NEXT;
+    END LOOP;
+END;
+$func$;
+
+-- Drop the registration older releases left in so_telegraf.
+SELECT CASE
+         WHEN current_setting('cron.database_name', true) IS DISTINCT FROM current_database()
+              AND EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_cron')
+         THEN 'true' ELSE 'false'
+       END AS drop_stale_cron \gset
+\if :drop_stale_cron
+DROP EXTENSION pg_cron CASCADE;
+\endif
+EOSQL
+
+    # Guarded on the live GUC so applying this before the postgresql.conf change
+    # has restarted the container skips rather than failing.
+    docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d postgres <<'EOSQL'
+SELECT CASE WHEN current_setting('cron.database_name', true) = current_database()
+            THEN 'true' ELSE 'false' END AS cron_here \gset
+\if :cron_here
+CREATE EXTENSION IF NOT EXISTS pg_cron;
+-- cron.schedule_in_database is idempotent by jobname.
+SELECT cron.schedule_in_database(
   'telegraf-partman-maintenance',
   '17 * * * *',
-  'CALL partman.run_maintenance_proc()'
+  'CALL so_admin.telegraf_maintenance()',
+  'so_telegraf'
 );
+\else
+\echo 'pg_cron metadata database is not `postgres` yet; skipping job registration.'
+\endif
 EOSQL
     ;;
 
@@ -90,6 +225,8 @@ EOSQL
     : "${RETENTION_DAYS:?RETENTION_DAYS is required}"
     # \gset + \if guards against a missing pg_partman without using a DO
     # block (psql :var substitution doesn't reach into dollar-quoted code).
+    # premake is reconciled here because telegraf.conf only applies it to
+    # parents created from now on.
     docker exec -i so-postgres psql \
       -v ON_ERROR_STOP=1 \
       -v retention_days="$RETENTION_DAYS" \
@@ -97,14 +234,60 @@ EOSQL
 SELECT CASE WHEN EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_partman')
             THEN 'true' ELSE 'false' END AS has_partman \gset
 \if :has_partman
+-- infinite_time_partitions so a gap in metrics does not stop partman from
+-- premaking forward, which is what leaves everything in the default.
 UPDATE partman.part_config
 SET retention = :'retention_days' || ' days',
-    retention_keep_table = false
+    retention_keep_table = false,
+    premake = 7,
+    infinite_time_partitions = true
 WHERE parent_table LIKE 'telegraf.%';
 \endif
 EOSQL
     ;;
 
+  maintenance)
+    docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
+SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL
+            THEN 'true' ELSE 'false' END AS has_proc \gset
+\if :has_proc
+CALL so_admin.telegraf_maintenance();
+\else
+\echo 'so_admin.telegraf_maintenance() is missing; run so-telegraf-postgres group_role first.'
+\endif
+EOSQL
+    ;;
+
+  check)
+    docker exec -i so-postgres psql -U postgres -d so_telegraf <<'EOSQL'
+\pset border 2
+SELECT * FROM so_admin.telegraf_partition_status();
+EOSQL
+    docker exec -i so-postgres psql -U postgres -d postgres <<'EOSQL'
+\pset border 2
+SELECT CASE WHEN to_regclass('cron.job_run_details') IS NOT NULL
+            THEN 'true' ELSE 'false' END AS has_cron \gset
+\if :has_cron
+SELECT d.status, d.return_message, d.start_time
+FROM cron.job_run_details d
+JOIN cron.job j ON j.jobid = d.jobid
+WHERE j.jobname = 'telegraf-partman-maintenance'
+ORDER BY d.start_time DESC
+LIMIT 5;
+\else
+\echo 'pg_cron is not installed in this database.'
+\endif
+EOSQL
+    unhealthy=$(docker exec so-postgres psql -U postgres -d so_telegraf -tAc \
+      "SELECT count(*) FROM so_admin.telegraf_partition_status()
+       WHERE coalesce(default_rows, 0) > 0 OR coalesce(days_ahead, -1) < 1")
+    if [ "${unhealthy:-1}" != "0" ]; then
+      echo "so-telegraf-postgres check: $unhealthy telegraf parent(s) unhealthy" >&2
+      exit 1
+    fi
+    echo "so-telegraf-postgres check: all telegraf parents healthy"
+    ;;
+
   *)
     echo "Unknown subcommand: $cmd" >&2
     exit 1

salt/postgres/tools/sbin/so-telegraf-repair

@@ -0,0 +1,245 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+# Put Telegraf metrics storage back in service on a grid where pg_partman
+# maintenance stalled: raises premake, discards the rows stranded in default
+# partitions, restarts so-postgres if pg_cron's launcher is dead, and runs
+# maintenance once. Healthy grids are reported and left alone.
+#
+# Usage: so-telegraf-repair [--check] [--yes] [--no-restart]
+#   --check       Report health and change nothing.
+#   --yes         Skip the confirmation prompt (for soup and other automation).
+#   --no-restart  Never restart so-postgres, even if pg_cron's launcher is dead.
+#
+# Exit status:
+#   0  healthy, or repair completed
+#   1  repair is needed (--check only)
+#   2  cannot run here: so-postgres, so_telegraf or pg_partman is missing
+
+set -e
+
+# Matches p_premake in telegraf.conf's create_parent template.
+PREMAKE=7
+JOB_NAME=telegraf-partman-maintenance
+
+CHECK_ONLY=false
+ASSUME_YES=false
+NO_RESTART=false
+
+usage() { sed -n '/^# Usage:/,/^#   2 /p' "$0" | sed 's/^# \?//'; }
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --check|--dry-run) CHECK_ONLY=true ;;
+    --yes|-y)          ASSUME_YES=true ;;
+    --no-restart)      NO_RESTART=true ;;
+    -h|--help)         usage; exit 0 ;;
+    *) echo "Unknown option: $1" >&2; usage >&2; exit 2 ;;
+  esac
+  shift
+done
+
+skip() { echo "$*"; exit 2; }
+
+psql_tg() { docker exec -i so-postgres psql -U postgres -d so_telegraf "$@"; }
+psql_pg() { docker exec -i so-postgres psql -U postgres -d postgres "$@"; }
+
+# query_to_xml so the per-table row counts need no helper function installed.
+REPORT="
+WITH parents AS (
+    SELECT pc.parent_table,
+           pc.premake,
+           split_part(pc.parent_table, '.', 1) AS sch,
+           split_part(pc.parent_table, '.', 2) AS tbl
+    FROM partman.part_config pc
+    WHERE pc.parent_table LIKE 'telegraf.%'
+), children AS (
+    SELECT p.parent_table,
+           max(to_date(substring(c.relname FROM '_p(\d{8})\$'), 'YYYYMMDD')) AS newest_child
+    FROM parents p
+    JOIN pg_class pt ON pt.oid = p.parent_table::regclass
+    JOIN pg_inherits i ON i.inhparent = pt.oid
+    JOIN pg_class c ON c.oid = i.inhrelid
+    WHERE pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT'
+    GROUP BY p.parent_table
+), defaults AS (
+    SELECT p.parent_table,
+           p.premake,
+           format('%I.%I', p.sch, p.tbl || '_default') AS default_table,
+           to_regclass(format('%I.%I', p.sch, p.tbl || '_default')) AS default_oid
+    FROM parents p
+)
+SELECT d.parent_table,
+       c.newest_child,
+       (c.newest_child - current_date) AS days_ahead,
+       d.premake,
+       CASE WHEN d.default_oid IS NULL THEN NULL ELSE
+            (xpath('/row/cnt/text()',
+                   query_to_xml(format('SELECT count(*) AS cnt FROM %s', d.default_table),
+                                false, true, '')))[1]::text::bigint
+       END AS default_rows,
+       CASE WHEN d.default_oid IS NULL THEN NULL
+            ELSE pg_size_pretty(pg_total_relation_size(d.default_oid)) END AS default_size
+FROM defaults d
+LEFT JOIN children c ON c.parent_table = d.parent_table
+ORDER BY 1
+"
+
+docker ps --format '{{.Names}}' | grep -qx so-postgres \
+  || skip "so-postgres is not running; nothing to repair."
+docker exec so-postgres psql -U postgres -tAc \
+  "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1 \
+  || skip "The so_telegraf database does not exist; Telegraf is not writing to Postgres."
+psql_tg -tAc "SELECT 1 FROM pg_extension WHERE extname='pg_partman'" | grep -q 1 \
+  || skip "pg_partman is not installed in so_telegraf; nothing to repair."
+
+parents=$(psql_tg -tAc \
+  "SELECT count(*) FROM partman.part_config WHERE parent_table LIKE 'telegraf.%'")
+stranded=$(psql_tg -tAc "SELECT coalesce(sum(default_rows), 0) FROM ( $REPORT ) t")
+behind=$(psql_tg -tAc \
+  "SELECT count(*) FROM ( $REPORT ) t WHERE coalesce(days_ahead, -1) < 1")
+# premake < 7, or infinite_time_partitions off: without the latter partman
+# refuses to premake forward across the gap the stall left behind.
+misconfigured=$(psql_tg -tAc \
+  "SELECT count(*) FROM partman.part_config
+    WHERE parent_table LIKE 'telegraf.%'
+      AND (premake < $PREMAKE OR NOT infinite_time_partitions)")
+
+# Both columns are matched because which one carries the launcher's name varies
+# with the pg_cron version.
+launcher=$(psql_pg -tAc \
+  "SELECT count(*) FROM pg_stat_activity
+    WHERE backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%'")
+
+# so_telegraf before the postgres state lands, postgres after.
+cron_db=$(docker exec so-postgres psql -U postgres -tAc \
+  "SELECT current_setting('cron.database_name', true)" | tr -d '[:space:]')
+last_run=never
+if [[ -n "$cron_db" ]]; then
+  last_run=$(docker exec so-postgres psql -U postgres -d "$cron_db" -tAc \
+    "SELECT coalesce(max(d.start_time)::text, 'never')
+       FROM cron.job_run_details d JOIN cron.job j USING (jobid)
+      WHERE j.jobname = '$JOB_NAME'" 2>/dev/null | tr -d '[:space:]' || echo unknown)
+  [[ -n "$last_run" ]] || last_run=never
+fi
+
+# A grid that has never written a metric has nothing to recover, and an empty
+# cron_db means pg_cron is not loaded at all, which no restart fixes.
+restart_needed=false
+[[ "$launcher" -eq 0 && "$parents" -gt 0 && -n "$cron_db" ]] && restart_needed=true
+
+repair_needed=false
+[[ "$stranded" -gt 0 ]] && repair_needed=true
+[[ "$behind" -gt 0 ]] && repair_needed=true
+[[ "$misconfigured" -gt 0 ]] && repair_needed=true
+$restart_needed && repair_needed=true
+
+echo "Telegraf partition status:"
+psql_tg -c "$REPORT"
+echo "Rows stranded in default partitions: $stranded"
+echo "pg_cron metadata database: ${cron_db:-unset}"
+echo "pg_cron launcher running: $([[ "$launcher" -gt 0 ]] && echo yes || echo no)"
+echo "Last $JOB_NAME run: $last_run"
+echo
+
+if ! $repair_needed; then
+  echo "Telegraf partitions are healthy. Nothing to do."
+  exit 0
+fi
+
+if $CHECK_ONLY; then
+  echo "Repair is needed:"
+  [[ "$stranded" -gt 0 ]]    && echo "  * $stranded row(s) stranded in default partitions"
+  [[ "$behind" -gt 0 ]]      && echo "  * $behind parent(s) with no partition for the current window"
+  [[ "$misconfigured" -gt 0 ]] && echo "  * $misconfigured parent(s) with stale partman settings"
+  $restart_needed            && echo "  * pg_cron's launcher is dead; maintenance is not running at all"
+  echo
+  echo "Re-run without --check to repair."
+  exit 1
+fi
+
+if [[ "$stranded" -gt 0 ]] && ! $ASSUME_YES; then
+  echo "This will permanently discard the $stranded stranded row(s) above."
+  $restart_needed && ! $NO_RESTART && \
+    echo "so-postgres will also be restarted, which briefly interrupts SOC."
+  [[ -t 0 ]] || { echo "Not a terminal; re-run with --yes to confirm." >&2; exit 2; }
+  read -r -p "Continue? [y/N] " answer
+  [[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 0; }
+fi
+
+if [[ "$misconfigured" -gt 0 ]]; then
+  echo "Reconciling partman settings on $misconfigured parent(s)."
+  # GREATEST so an operator who raised premake further keeps their value.
+  psql_tg -v ON_ERROR_STOP=1 -c \
+    "UPDATE partman.part_config
+        SET premake = GREATEST(premake, $PREMAKE),
+            infinite_time_partitions = true
+      WHERE parent_table LIKE 'telegraf.%'"
+fi
+
+if [[ "$stranded" -gt 0 ]]; then
+  echo "Clearing default partitions."
+  # One transaction: Telegraf is still writing, so a default emptied without
+  # its partition in place is refilled before maintenance can attach one.
+  psql_tg -v ON_ERROR_STOP=1 <<'EOSQL'
+DO $$
+DECLARE
+    r record;
+BEGIN
+    FOR r IN
+        SELECT pc.parent_table,
+               format('%I.%I', n.nspname, c.relname) AS default_table
+        FROM partman.part_config pc
+        JOIN pg_class p ON p.oid = pc.parent_table::regclass
+        JOIN pg_inherits i ON i.inhparent = p.oid
+        JOIN pg_class c ON c.oid = i.inhrelid
+        JOIN pg_namespace n ON n.oid = c.relnamespace
+        WHERE pc.parent_table LIKE 'telegraf.%'
+          AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT'
+    LOOP
+        EXECUTE format('TRUNCATE TABLE %s', r.default_table);
+        PERFORM partman.create_partition_time(
+            r.parent_table, ARRAY[date_trunc('day', now())]::timestamptz[]);
+    END LOOP;
+END
+$$;
+EOSQL
+fi
+
+if $restart_needed; then
+  if $NO_RESTART; then
+    echo "WARNING: pg_cron's launcher is dead and --no-restart was given."
+    echo "         Maintenance will not run on its own until so-postgres is restarted."
+  else
+    echo "Restarting so-postgres to revive pg_cron's launcher."
+    docker restart so-postgres >/dev/null
+    for _ in $(seq 1 60); do
+      docker exec so-postgres pg_isready -U postgres -q 2>/dev/null && break
+      sleep 2
+    done
+    docker exec so-postgres pg_isready -U postgres -q \
+      || { echo "so-postgres did not come back; check 'docker logs so-postgres'." >&2; exit 1; }
+  fi
+fi
+
+echo "Running partition maintenance."
+# so_admin.telegraf_maintenance() only exists once the postgres state has landed.
+psql_tg -v ON_ERROR_STOP=1 <<'EOSQL'
+SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL
+            THEN 'true' ELSE 'false' END AS has_proc \gset
+\if :has_proc
+CALL so_admin.telegraf_maintenance();
+\else
+CALL partman.run_maintenance_proc();
+\endif
+EOSQL
+
+echo
+echo "Telegraf partition status after repair:"
+psql_tg -c "$REPORT"
+echo "The $JOB_NAME job runs hourly at :17. Confirm it fired with:"
+echo "  so-telegraf-repair --check"

salt/telegraf/etc/telegraf.conf

@@ -122,7 +122,7 @@
   create_templates = [
     '''CREATE TABLE IF NOT EXISTS {{ .table }} ({{ .columns }}) PARTITION BY RANGE ("time")''',
     '''ALTER TABLE {{ .table }} ALTER COLUMN "time" SET NOT NULL''',
-    '''SELECT partman.create_parent(p_parent_table := {{ printf "%s.%s" .table.Schema .table.Name | quoteLiteral }}, p_control := 'time', p_type := 'range', p_interval := '1 day', p_premake := 3) WHERE NOT EXISTS (SELECT 1 FROM partman.part_config WHERE parent_table = {{ printf "%s.%s" .table.Schema .table.Name | quoteLiteral }})'''
+    '''SELECT partman.create_parent(p_parent_table := {{ printf "%s.%s" .table.Schema .table.Name | quoteLiteral }}, p_control := 'time', p_type := 'range', p_interval := '1 day', p_premake := 7) WHERE NOT EXISTS (SELECT 1 FROM partman.part_config WHERE parent_table = {{ printf "%s.%s" .table.Schema .table.Name | quoteLiteral }})'''
   ]
   tag_table_create_templates = [
     '''CREATE TABLE IF NOT EXISTS {{ .table }} ({{ .columns }}, PRIMARY KEY (tag_id))'''

salt/zeek/config.sls

@@ -133,6 +133,20 @@ zeekctlcfg:
     - defaults:
         ZEEKCTL: {{ ZEEKMERGED.config.zeekctl | tojson }}
 
+# ZeekControl only acts on these five settings from its 'cron' command, so track them
+# separately from zeekctl.cfg. Changing CompressLogs or LogRotationInterval rewrites
+# zeekctl.cfg but must not trigger a cron run, so zeekctlcron watches this file instead.
+zeekctlcronsettings:
+  file.managed:
+    - name: /opt/so/state/zeek/zeekctl_cron_settings
+    - makedirs: True
+    - contents: |
+        LogExpireInterval={{ ZEEKMERGED.config.zeekctl.get('LogExpireInterval', '') }}
+        StatsLogExpireInterval={{ ZEEKMERGED.config.zeekctl.get('StatsLogExpireInterval', '') }}
+        CrashExpireInterval={{ ZEEKMERGED.config.zeekctl.get('CrashExpireInterval', '') }}
+        MinDiskSpace={{ ZEEKMERGED.config.zeekctl.get('MinDiskSpace', '') }}
+        MailHostUpDown={{ ZEEKMERGED.config.zeekctl.get('MailHostUpDown', '') }}
+
 # Sync node.cfg
 nodecfg:
   file.managed:

salt/zeek/enabled.sls

@@ -71,6 +71,25 @@ so-zeek:
       - file: zeekctlcfg
       - file: zeekbpf
 
+# LogExpireInterval, StatsLogExpireInterval, CrashExpireInterval, MinDiskSpace and
+# MailHostUpDown stay inert until 'zeekctl cron' runs, so run it once whenever one of
+# them changes. --no-watch skips ZeekControl's crashed-node watchdog, which would
+# otherwise race the container restart that the same config change triggers, and which
+# would duplicate the healthcheck beacon in salt/_beacons/zeek.py. The retry covers the
+# startup deploy still holding the ZeekControl lock. Run as the zeek user, as
+# salt/_modules/zeekctl.py does, so the stats logs and zeekctl-config.sh this writes stay
+# owned by uid 937 rather than root.
+zeekctlcron:
+  cmd.run:
+    - name: docker exec so-zeek runuser -l zeek -c '/opt/zeek/bin/zeekctl cron --no-watch'
+    - onchanges:
+      - file: zeekctlcronsettings
+    - require:
+      - docker_container: so-zeek
+    - retry:
+        attempts: 5
+        interval: 30
+
 delete_so-zeek_so-status.disabled:
   file.uncomment:
     - name: /opt/so/conf/so-status/so-status.conf

salt/zeek/soc_zeek.yaml

@@ -58,6 +58,26 @@ zeek:
       CompressLogs:
         description: This setting enables compression of Zeek logs. If you are seeing packet loss at the top of the hour in Zeek or PCAP you might need to disable this by seting it to 0. This will use more disk space but save IO and CPU.
         helpLink: zeek
+      LogExpireInterval:
+        description: Number of days to keep rotated Zeek logs in /nsm/zeek/logs, or 0 to keep them forever. Saving this setting runs "zeekctl cron", which deletes any logs already older than the value you set. Note that this is a one time cleanup rather than a rolling retention policy, so logs that age past the interval afterwards are not removed until this setting is changed again. Ongoing cleanup based on disk usage is handled separately by so-sensor-clean.
+        helpLink: zeek
+        advanced: True
+      StatsLogExpireInterval:
+        description: Number of days to keep entries in the Zeek stats log, or 0 to keep them forever. Saving this setting runs "zeekctl cron", which applies the new value once. See LogExpireInterval for details on this behavior.
+        helpLink: zeek
+        advanced: True
+      CrashExpireInterval:
+        description: Number of days to keep Zeek crash directories, or 0 to keep them forever. Saving this setting runs "zeekctl cron", which applies the new value once. See LogExpireInterval for details on this behavior.
+        helpLink: zeek
+        advanced: True
+      MinDiskSpace:
+        description: Percentage of free disk space below which ZeekControl reports a warning, or 0 to disable the check. This check only runs during "zeekctl cron", which Security Onion runs when you save this setting. It does not delete anything.
+        helpLink: zeek
+        advanced: True
+      MailHostUpDown:
+        description: Set to 1 to send mail when a Zeek node changes between the up and down states. This check only runs during "zeekctl cron", which Security Onion runs when you save this setting, so it is not a continuous monitor. Requires MailTo to be set to a working address.
+        helpLink: zeek
+        advanced: True
   policy:
     custom:
       filters: