mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-11-18 18:16:02 +01:00
soup: gate Telegraf partition warning on Postgres output, once per run (review fixes for #16398)
This commit is contained in:
6 files changed
+481
-160
No files matched your search
@@ -4,15 +4,18 @@ set -e
|
||||
# Provision Telegraf state inside the so-postgres container.
|
||||
# Usage: so-telegraf-postgres <subcommand>
|
||||
# create_db Ensure the so_telegraf database exists.
|
||||
# group_role Provision the so_telegraf group role, telegraf/partman schemas,
|
||||
# pg_partman, the so_admin maintenance routines, and the hourly
|
||||
# pg_cron maintenance job.
|
||||
# group_role Provision the so_telegraf group role, the so_maint_telegraf
|
||||
# maintenance login role, telegraf/partman schemas, pg_partman,
|
||||
# the so_admin maintenance routines, and the hourly pg_cron
|
||||
# maintenance job.
|
||||
# user Create or update a per-minion login role granted to so_telegraf.
|
||||
# Env: ROLE_USER, ROLE_PASS.
|
||||
# retention Reconcile partman retention and premake on telegraf parents.
|
||||
# Env: RETENTION_DAYS.
|
||||
# maintenance Drain default partitions and run partman maintenance.
|
||||
# check Report partition health. Non-zero if any parent is unhealthy.
|
||||
# maintenance Drain default partitions and run partman maintenance, as
|
||||
# so_maint_telegraf.
|
||||
# check Report partition health, as so_maint_telegraf. Non-zero if any
|
||||
# parent is unhealthy.
|
||||
#
|
||||
# A default partition holding rows for a day blocks creating that day's child,
|
||||
# so maintenance drains defaults before calling partman. Use so-telegraf-repair
|
||||
@@ -20,6 +23,10 @@ set -e
|
||||
|
||||
cmd="${1:?subcommand required}"
|
||||
|
||||
# Long enough for an hourly maintenance run to release its locks, short enough
|
||||
# that soup and admins are not left hanging.
|
||||
LOCK_TIMEOUT=60s
|
||||
|
||||
case "$cmd" in
|
||||
create_db)
|
||||
if ! docker exec so-postgres psql -U postgres -tAc \
|
||||
@@ -30,7 +37,9 @@ case "$cmd" in
|
||||
;;
|
||||
|
||||
group_role)
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -v lock_timeout="$LOCK_TIMEOUT" \
|
||||
-U postgres -d so_telegraf <<'EOSQL'
|
||||
SET lock_timeout = :'lock_timeout';
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'so_telegraf') THEN
|
||||
@@ -38,6 +47,17 @@ BEGIN
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'so_maint_telegraf') THEN
|
||||
CREATE ROLE so_maint_telegraf;
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
ALTER ROLE so_maint_telegraf LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD NULL;
|
||||
GRANT so_telegraf TO so_maint_telegraf;
|
||||
ALTER ROLE so_maint_telegraf SET role = so_telegraf;
|
||||
ALTER ROLE so_maint_telegraf SET search_path = pg_catalog;
|
||||
GRANT CONNECT ON DATABASE so_telegraf TO so_telegraf;
|
||||
CREATE SCHEMA IF NOT EXISTS telegraf AUTHORIZATION so_telegraf;
|
||||
GRANT USAGE, CREATE ON SCHEMA telegraf TO so_telegraf;
|
||||
@@ -58,52 +78,90 @@ ALTER DEFAULT PRIVILEGES IN SCHEMA partman
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA partman
|
||||
GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO so_telegraf;
|
||||
|
||||
-- pg_cron runs these as postgres, so they must not sit in a schema any
|
||||
-- Telegraf role can create objects in.
|
||||
-- Owned by postgres so no Telegraf role can create objects here; the routines
|
||||
-- run with the caller's privileges.
|
||||
CREATE SCHEMA IF NOT EXISTS so_admin AUTHORIZATION postgres;
|
||||
REVOKE ALL ON SCHEMA so_admin FROM PUBLIC;
|
||||
|
||||
-- Resolved through pg_inherits, never by name, so a view or a table that only
|
||||
-- carries the _default name is never read or written.
|
||||
CREATE OR REPLACE FUNCTION so_admin.telegraf_defaults()
|
||||
RETURNS TABLE (
|
||||
parent_table regclass,
|
||||
default_part regclass,
|
||||
has_rows boolean,
|
||||
default_size bigint
|
||||
)
|
||||
LANGUAGE plpgsql
|
||||
STABLE
|
||||
SECURITY INVOKER
|
||||
AS $func$
|
||||
DECLARE
|
||||
r record;
|
||||
BEGIN
|
||||
FOR r IN
|
||||
SELECT c.oid::regclass AS pt
|
||||
FROM partman.part_config pc
|
||||
JOIN pg_catalog.pg_class c ON c.oid = to_regclass(pc.parent_table)
|
||||
WHERE pc.parent_table LIKE 'telegraf.%'
|
||||
AND c.relkind = 'p'
|
||||
ORDER BY pc.parent_table
|
||||
LOOP
|
||||
parent_table := r.pt;
|
||||
|
||||
SELECT c.oid::regclass INTO default_part
|
||||
FROM pg_catalog.pg_inherits i
|
||||
JOIN pg_catalog.pg_class c ON c.oid = i.inhrelid
|
||||
WHERE i.inhparent = r.pt
|
||||
AND c.relkind = 'r' AND c.relispartition
|
||||
AND pg_catalog.pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT';
|
||||
|
||||
IF default_part IS NULL THEN
|
||||
has_rows := false;
|
||||
default_size := NULL;
|
||||
ELSE
|
||||
EXECUTE format('SELECT EXISTS (SELECT 1 FROM %s LIMIT 1)', default_part)
|
||||
INTO has_rows;
|
||||
default_size := pg_catalog.pg_total_relation_size(default_part);
|
||||
END IF;
|
||||
|
||||
RETURN NEXT;
|
||||
END LOOP;
|
||||
END;
|
||||
$func$;
|
||||
|
||||
CREATE OR REPLACE PROCEDURE so_admin.telegraf_maintenance()
|
||||
LANGUAGE plpgsql
|
||||
SECURITY INVOKER
|
||||
-- No SET clause: it would forbid the COMMITs below, so so_maint_telegraf pins search_path instead.
|
||||
AS $proc$
|
||||
DECLARE
|
||||
r record;
|
||||
v_default text;
|
||||
v_rows bigint;
|
||||
r record;
|
||||
BEGIN
|
||||
-- No per-parent EXCEPTION handler: partition_data_proc commits internally,
|
||||
-- and COMMIT is illegal while a subtransaction is active. A failing parent
|
||||
-- aborts the run and the next pass retries.
|
||||
FOR r IN
|
||||
SELECT parent_table, retention
|
||||
FROM partman.part_config
|
||||
WHERE parent_table LIKE 'telegraf.%'
|
||||
ORDER BY parent_table
|
||||
SELECT pc.parent_table, pc.retention, d.default_part, d.default_size
|
||||
FROM so_admin.telegraf_defaults() d
|
||||
JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table
|
||||
WHERE d.default_part IS NOT NULL AND d.has_rows
|
||||
ORDER BY pc.parent_table
|
||||
LOOP
|
||||
v_default := format('%I.%I',
|
||||
split_part(r.parent_table, '.', 1),
|
||||
split_part(r.parent_table, '.', 2) || '_default');
|
||||
|
||||
CONTINUE WHEN to_regclass(v_default) IS NULL;
|
||||
|
||||
EXECUTE format('SELECT count(*) FROM %s', v_default) INTO v_rows;
|
||||
CONTINUE WHEN v_rows = 0;
|
||||
|
||||
RAISE WARNING 'so_admin.telegraf_maintenance: % rows stranded in %, draining',
|
||||
v_rows, v_default;
|
||||
RAISE WARNING 'so_admin.telegraf_maintenance: % holds rows (%), draining',
|
||||
r.default_part, pg_size_pretty(r.default_size);
|
||||
|
||||
-- Cheaper to delete expired rows than to repartition and then drop them.
|
||||
IF r.retention IS NOT NULL THEN
|
||||
EXECUTE format('DELETE FROM %s WHERE "time" < now() - %L::interval',
|
||||
v_default, r.retention);
|
||||
r.default_part, r.retention);
|
||||
COMMIT;
|
||||
END IF;
|
||||
|
||||
-- Bounded so a large backlog drains across several runs.
|
||||
CALL partman.partition_data_proc(
|
||||
p_parent_table := r.parent_table,
|
||||
p_loop_count := 200,
|
||||
p_source_table := v_default
|
||||
p_loop_count := 200
|
||||
);
|
||||
COMMIT;
|
||||
END LOOP;
|
||||
@@ -112,58 +170,75 @@ BEGIN
|
||||
END;
|
||||
$proc$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION so_admin.telegraf_partition_status()
|
||||
DROP FUNCTION IF EXISTS so_admin.telegraf_partition_status();
|
||||
CREATE FUNCTION so_admin.telegraf_partition_status()
|
||||
RETURNS TABLE (
|
||||
parent_table text,
|
||||
oldest_child date,
|
||||
newest_child date,
|
||||
days_ahead int,
|
||||
retention text,
|
||||
default_rows bigint,
|
||||
default_size text
|
||||
parent_table text,
|
||||
oldest_child date,
|
||||
newest_child date,
|
||||
days_ahead int,
|
||||
retention text,
|
||||
default_has_rows boolean,
|
||||
default_size text
|
||||
)
|
||||
LANGUAGE plpgsql
|
||||
STABLE
|
||||
AS $func$
|
||||
DECLARE
|
||||
r record;
|
||||
v_default regclass;
|
||||
r record;
|
||||
BEGIN
|
||||
FOR r IN
|
||||
SELECT pc.parent_table AS pt, pc.retention AS ret
|
||||
FROM partman.part_config pc
|
||||
WHERE pc.parent_table LIKE 'telegraf.%'
|
||||
ORDER BY pc.parent_table
|
||||
SELECT d.parent_table AS pt, d.has_rows, d.default_size AS size, pc.retention AS ret
|
||||
FROM so_admin.telegraf_defaults() d
|
||||
JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table
|
||||
ORDER BY d.parent_table::text
|
||||
LOOP
|
||||
parent_table := r.pt;
|
||||
parent_table := r.pt::text;
|
||||
retention := r.ret;
|
||||
|
||||
SELECT min(d), max(d) INTO oldest_child, newest_child
|
||||
SELECT min(x), max(x) INTO oldest_child, newest_child
|
||||
FROM (
|
||||
SELECT to_date(substring(c.relname FROM '_p(\d{8})$'), 'YYYYMMDD') AS d
|
||||
FROM pg_inherits i
|
||||
JOIN pg_class c ON c.oid = i.inhrelid
|
||||
WHERE i.inhparent = r.pt::regclass
|
||||
AND pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT'
|
||||
SELECT to_date(substring(c.relname FROM '_p(\d{8})$'), 'YYYYMMDD') AS x
|
||||
FROM pg_catalog.pg_inherits i
|
||||
JOIN pg_catalog.pg_class c ON c.oid = i.inhrelid
|
||||
WHERE i.inhparent = r.pt
|
||||
AND pg_catalog.pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT'
|
||||
) s;
|
||||
|
||||
days_ahead := newest_child - current_date;
|
||||
|
||||
v_default := to_regclass(format('%I.%I',
|
||||
split_part(r.pt, '.', 1),
|
||||
split_part(r.pt, '.', 2) || '_default'));
|
||||
IF v_default IS NULL THEN
|
||||
default_rows := NULL;
|
||||
default_size := NULL;
|
||||
ELSE
|
||||
EXECUTE format('SELECT count(*) FROM %s', v_default::text) INTO default_rows;
|
||||
default_size := pg_size_pretty(pg_total_relation_size(v_default));
|
||||
END IF;
|
||||
days_ahead := newest_child - current_date;
|
||||
default_has_rows := r.has_rows;
|
||||
default_size := pg_size_pretty(r.size);
|
||||
|
||||
RETURN NEXT;
|
||||
END LOOP;
|
||||
END;
|
||||
$func$;
|
||||
|
||||
GRANT USAGE ON SCHEMA so_admin TO so_telegraf;
|
||||
GRANT EXECUTE ON FUNCTION so_admin.telegraf_defaults() TO so_telegraf;
|
||||
GRANT EXECUTE ON PROCEDURE so_admin.telegraf_maintenance() TO so_telegraf;
|
||||
GRANT EXECUTE ON FUNCTION so_admin.telegraf_partition_status() TO so_telegraf;
|
||||
|
||||
-- Partitions created by a superuser session (an older repair) belong to
|
||||
-- postgres; partman acting as so_telegraf could not drop them.
|
||||
DO $$
|
||||
DECLARE
|
||||
r record;
|
||||
BEGIN
|
||||
FOR r IN
|
||||
SELECT n.nspname, c.relname
|
||||
FROM pg_catalog.pg_class c
|
||||
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE c.relkind IN ('r', 'p')
|
||||
AND (n.nspname = 'telegraf'
|
||||
OR (n.nspname = 'partman' AND c.relname LIKE 'template\_telegraf\_%'))
|
||||
AND c.relowner <> 'so_telegraf'::regrole
|
||||
LOOP
|
||||
EXECUTE format('ALTER TABLE %I.%I OWNER TO so_telegraf', r.nspname, r.relname);
|
||||
END LOOP;
|
||||
END
|
||||
$$;
|
||||
|
||||
-- Drop the registration older releases left in so_telegraf.
|
||||
SELECT CASE
|
||||
WHEN current_setting('cron.database_name', true) IS DISTINCT FROM current_database()
|
||||
@@ -175,20 +250,69 @@ DROP EXTENSION pg_cron CASCADE;
|
||||
\endif
|
||||
EOSQL
|
||||
|
||||
stalled=false
|
||||
probe_failed=false
|
||||
if out=$(docker exec -e PGOPTIONS="-c lock_timeout=$LOCK_TIMEOUT" so-postgres \
|
||||
psql -U so_maint_telegraf -d so_telegraf -tAc \
|
||||
"SELECT coalesce(bool_or(has_rows), false) FROM so_admin.telegraf_defaults()"); then
|
||||
case "$out" in
|
||||
t) stalled=true ;;
|
||||
f) stalled=false ;;
|
||||
*) probe_failed=true ;;
|
||||
esac
|
||||
else
|
||||
probe_failed=true
|
||||
fi
|
||||
if $probe_failed; then
|
||||
echo "ERROR: could not check Telegraf default partitions; the maintenance job was not changed." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Guarded on the live GUC so applying this before the postgresql.conf change
|
||||
# has restarted the container skips rather than failing.
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d postgres <<'EOSQL'
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -v stalled="$stalled" \
|
||||
-U postgres -d postgres <<'EOSQL'
|
||||
SELECT CASE WHEN current_setting('cron.database_name', true) = current_database()
|
||||
THEN 'true' ELSE 'false' END AS cron_here \gset
|
||||
\if :cron_here
|
||||
CREATE EXTENSION IF NOT EXISTS pg_cron;
|
||||
-- cron.schedule_in_database is idempotent by jobname.
|
||||
-- A highstate reconciles the job's definition but never its active flag; only so-telegraf-repair turns it on.
|
||||
BEGIN;
|
||||
SELECT coalesce(max(jobid)::text, '') AS jobid,
|
||||
count(*) > 0 AS job_exists,
|
||||
coalesce(bool_and(active), true) AS job_active
|
||||
FROM cron.job
|
||||
WHERE jobname = 'telegraf-partman-maintenance' AND username = 'so_maint_telegraf' \gset
|
||||
SELECT cron.unschedule(jobid) FROM cron.job
|
||||
WHERE jobname = 'telegraf-partman-maintenance' AND username <> 'so_maint_telegraf';
|
||||
\if :job_exists
|
||||
SELECT cron.alter_job(:jobid,
|
||||
schedule := '17 * * * *',
|
||||
command := 'CALL so_admin.telegraf_maintenance()',
|
||||
database := 'so_telegraf');
|
||||
\else
|
||||
-- Registered inactive on a stalled grid so nothing drains the default
|
||||
-- partitions before so-telegraf-repair truncates them.
|
||||
SELECT cron.schedule_in_database(
|
||||
'telegraf-partman-maintenance',
|
||||
'17 * * * *',
|
||||
'CALL so_admin.telegraf_maintenance()',
|
||||
'so_telegraf'
|
||||
'so_telegraf',
|
||||
'so_maint_telegraf',
|
||||
NOT :'stalled'::boolean
|
||||
);
|
||||
\endif
|
||||
UPDATE cron.job SET nodename = current_setting('cron.host')
|
||||
WHERE jobname = 'telegraf-partman-maintenance'
|
||||
AND nodename IS DISTINCT FROM current_setting('cron.host');
|
||||
COMMIT;
|
||||
SELECT coalesce(bool_and(active), true) AS job_active
|
||||
FROM cron.job
|
||||
WHERE jobname = 'telegraf-partman-maintenance' AND username = 'so_maint_telegraf' \gset
|
||||
\if :job_active
|
||||
\else
|
||||
\echo 'WARNING: the Telegraf maintenance job is inactive; run ''sudo so-telegraf-repair''.'
|
||||
\endif
|
||||
\else
|
||||
\echo 'pg_cron metadata database is not `postgres` yet; skipping job registration.'
|
||||
\endif
|
||||
@@ -241,13 +365,15 @@ SET retention = :'retention_days' || ' days',
|
||||
retention_keep_table = false,
|
||||
premake = 7,
|
||||
infinite_time_partitions = true
|
||||
WHERE parent_table LIKE 'telegraf.%';
|
||||
WHERE parent_table LIKE 'telegraf.%'
|
||||
AND EXISTS (SELECT 1 FROM pg_catalog.pg_class c
|
||||
WHERE c.oid = to_regclass(parent_table) AND c.relkind = 'p');
|
||||
\endif
|
||||
EOSQL
|
||||
;;
|
||||
|
||||
maintenance)
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U so_maint_telegraf -d so_telegraf <<'EOSQL'
|
||||
SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL
|
||||
THEN 'true' ELSE 'false' END AS has_proc \gset
|
||||
\if :has_proc
|
||||
@@ -259,7 +385,7 @@ EOSQL
|
||||
;;
|
||||
|
||||
check)
|
||||
docker exec -i so-postgres psql -U postgres -d so_telegraf <<'EOSQL'
|
||||
docker exec -i so-postgres psql -U so_maint_telegraf -d so_telegraf <<'EOSQL'
|
||||
\pset border 2
|
||||
SELECT * FROM so_admin.telegraf_partition_status();
|
||||
EOSQL
|
||||
@@ -278,9 +404,9 @@ LIMIT 5;
|
||||
\echo 'pg_cron is not installed in this database.'
|
||||
\endif
|
||||
EOSQL
|
||||
unhealthy=$(docker exec so-postgres psql -U postgres -d so_telegraf -tAc \
|
||||
unhealthy=$(docker exec so-postgres psql -U so_maint_telegraf -d so_telegraf -tAc \
|
||||
"SELECT count(*) FROM so_admin.telegraf_partition_status()
|
||||
WHERE coalesce(default_rows, 0) > 0 OR coalesce(days_ahead, -1) < 1")
|
||||
WHERE default_has_rows OR coalesce(days_ahead, -1) < 1")
|
||||
if [ "${unhealthy:-1}" != "0" ]; then
|
||||
echo "so-telegraf-postgres check: $unhealthy telegraf parent(s) unhealthy" >&2
|
||||
exit 1
|
||||
|
||||
Reference in new issue
Block a user