#!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. # Point-in-time host metrics from the Telegraf Postgres backend. # Sanity-check tool for verifying metrics are landing before the grid # dashboards consume them. . /usr/sbin/so-common usage() { cat </dev/null | cut -d\| -f1 | grep -qw so_telegraf; then echo "Database so_telegraf not found. Is global.telegraf_output set to POSTGRES or BOTH?" exit 2 fi # List telegraf schemas (role-per-minion naming convention: so_telegraf_) SCHEMAS=$(so_psql -c "SELECT schema_name FROM information_schema.schemata WHERE schema_name LIKE 'so_telegraf_%' ORDER BY schema_name;") if [ -z "$SCHEMAS" ]; then echo "No minion schemas found in so_telegraf." exit 0 fi print_metric() { local schema="$1" table="$2" query="$3" # Confirm table exists in this schema before querying local exists exists=$(so_psql -c "SELECT 1 FROM information_schema.tables WHERE table_schema='${schema}' AND table_name='${table}' LIMIT 1;") [ -z "$exists" ] && return 0 so_psql -c "$query" } # Telegraf's postgresql output stores tag values either as individual columns # on the _tag table or as a single JSONB "tags" column, depending on # plugin version. Returns a SQL expression that extracts the named tag # regardless of layout. Empty string if the tag table doesn't exist. tag_expr() { local schema="$1" table="$2" tag="$3" alias="$4" local has_col has_col=$(so_psql -c " SELECT 1 FROM information_schema.columns WHERE table_schema='${schema}' AND table_name='${table}_tag' AND column_name='${tag}' LIMIT 1;") if [ -n "$has_col" ]; then echo "${alias}.${tag}" return fi local has_tags has_tags=$(so_psql -c " SELECT 1 FROM information_schema.columns WHERE table_schema='${schema}' AND table_name='${table}_tag' AND column_name='tags' LIMIT 1;") if [ -n "$has_tags" ]; then echo "(${alias}.tags->>'${tag}')" return fi echo "" } for schema in $SCHEMAS; do minion="${schema#so_telegraf_}" if [ -n "$FILTER_MINION" ]; then # Compare against the sanitized form used in schema names want=$(echo "$FILTER_MINION" | tr '.-' '_' | tr '[:upper:]' '[:lower:]') [ "$minion" != "$want" ] && continue fi echo "====================================================================" echo " Minion: $minion" echo "====================================================================" cpu_tag=$(tag_expr "$schema" "cpu" "cpu" "t") if [ -n "$cpu_tag" ]; then print_metric "$schema" "cpu" " SELECT 'cpu ' AS metric, to_char(c.time, 'YYYY-MM-DD HH24:MI:SS') AS ts, round((100 - c.usage_idle)::numeric, 1) || '% used' FROM \"${schema}\".cpu c JOIN \"${schema}\".cpu_tag t USING (tag_id) WHERE ${cpu_tag} = 'cpu-total' ORDER BY c.time DESC LIMIT 1;" fi print_metric "$schema" "mem" " SELECT 'memory ' AS metric, to_char(m.time, 'YYYY-MM-DD HH24:MI:SS') AS ts, round(m.used_percent::numeric, 1) || '% used (' || pg_size_pretty(m.used) || ' of ' || pg_size_pretty(m.total) || ')' FROM \"${schema}\".mem m ORDER BY m.time DESC LIMIT 1;" disk_path=$(tag_expr "$schema" "disk" "path" "t") if [ -n "$disk_path" ]; then print_metric "$schema" "disk" " SELECT 'disk ' || rpad(${disk_path}, 12) AS metric, to_char(d.time, 'YYYY-MM-DD HH24:MI:SS') AS ts, round(d.used_percent::numeric, 1) || '% used (' || pg_size_pretty(d.used) || ' of ' || pg_size_pretty(d.total) || ')' FROM \"${schema}\".disk d JOIN \"${schema}\".disk_tag t USING (tag_id) WHERE d.time = (SELECT max(time) FROM \"${schema}\".disk) ORDER BY ${disk_path};" fi print_metric "$schema" "system" " SELECT 'load ' AS metric, to_char(s.time, 'YYYY-MM-DD HH24:MI:SS') AS ts, s.load1 || ' / ' || s.load5 || ' / ' || s.load15 || ' (1/5/15m)' FROM \"${schema}\".system s ORDER BY s.time DESC LIMIT 1;" echo "" done