Trim Telegraf collection on Postgres-only grids and add Postgres support to Elasticsearch growth scripts

This commit is contained in:
Mike Reeves committed 2026-11-17 14:17:27 -05:00
1 parent e51706acad
commit 6d7be47a69
4 files changed
+295 -69

No files matched your search

+102
View File
@@ -656,6 +656,108 @@ set_version() {
fi
}
so_telegraf_output() {
local defaultsfile="${DEFAULT_SALT_DIR}/salt/telegraf/defaults.yaml"
local output
output=$(lookup_pillar output telegraf) || return 1
if [ -z "$output" ]; then
if [ ! -f "$defaultsfile" ]; then
echo "Could not find $defaultsfile" >&2
return 1
fi
output=$(grep -E "^ output: " "$defaultsfile" | awk -F: '{print $2}' | tr -d '[:space:]')
fi
echo "${output^^}"
}
so_telegraf_pg_check() {
local app="$1"
local databases role_ok
if ! docker_check_running postgres; then
echo "so-postgres is not running. Run $app on the manager." >&2
return 1
fi
if ! databases=$(docker exec so-postgres psql -U postgres -lqt 2>/dev/null); then
echo "Unable to list the so-postgres databases." >&2
return 1
fi
if ! echo "$databases" | cut -d\| -f1 | grep -qw so_telegraf; then
echo "Database so_telegraf not found. Is telegraf.output set to POSTGRES or BOTH?" >&2
return 1
fi
if ! role_ok=$(docker exec so-postgres psql -U postgres -tAc "SELECT rolcanlogin AND NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls) FROM pg_roles WHERE rolname = 'so_maint_telegraf'" 2>/dev/null); then
echo "Unable to query the so-postgres roles." >&2
return 1
fi
if [ -z "$role_ok" ]; then
echo "The so_maint_telegraf role is missing; run a highstate." >&2
return 1
elif [ "$role_ok" != t ]; then
echo "The so_maint_telegraf role is misconfigured; run a highstate." >&2
return 1
fi
}
so_telegraf_psql() {
local app="$1"
local sql="$2"
docker exec -e PGOPTIONS="-c application_name=${app} -c default_transaction_read_only=on -c client_min_messages=error" \
so-postgres psql -U so_maint_telegraf -d so_telegraf -v ON_ERROR_STOP=1 -At -F '|' -c "$sql"
}
# Prints one stream|current|then line per data stream, in bytes summed across hosts. current is
# each host's newest size in the past 4 hours; then is that host's newest size in the window
# [now - start, now - stop), or its current size when it has none there, so a host missing from the
# window adds no growth. Empty output means the window has no rows at all. Values over 15 digits
# (1 PB) are skipped so the sums here and in the callers stay far below the 64-bit limit; such a
# value can only come from a forged Telegraf writer.
so_es_index_size_growth() {
local app="$1"
local start="$2"
local stop="$3"
local relkind
relkind=$(so_telegraf_psql "$app" "SELECT c.relkind FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'telegraf' AND c.relname = 'elasticsearch_index_size'") || return 1
[[ "$relkind" == "r" || "$relkind" == "p" ]] || return 0
so_telegraf_psql "$app" "$(cat <<EOF
WITH sizes AS (
SELECT d.tag_id, d.time, f.key AS stream, f.value::bigint AS bytes
FROM telegraf.elasticsearch_index_size d
CROSS JOIN LATERAL pg_catalog.jsonb_each_text(d.fields) f
WHERE d.time >= now() - interval '${start}'
AND f.key ~ '^[^|"''[:space:][:cntrl:]]+$'
AND pg_catalog.strpos(f.key, pg_catalog.chr(92)) = 0
AND f.value ~ '^[0-9]{1,15}$'
),
cur AS (
SELECT DISTINCT ON (tag_id, stream) tag_id, stream, bytes
FROM sizes
WHERE time >= now() - interval '4 hours'
ORDER BY tag_id, stream, time DESC
),
past AS (
SELECT DISTINCT ON (tag_id, stream) tag_id, stream, bytes
FROM sizes
WHERE time < now() - interval '${stop}'
ORDER BY tag_id, stream, time DESC
)
SELECT c.stream, sum(c.bytes), sum(coalesce(p.bytes, c.bytes))
FROM cur c
LEFT JOIN past p USING (tag_id, stream)
WHERE EXISTS (
SELECT 1 FROM telegraf.elasticsearch_index_size
WHERE time >= now() - interval '${start}' AND time < now() - interval '${stop}'
)
GROUP BY c.stream
ORDER BY c.stream
EOF
)"
}
status () {
printf "\n=========================================================================\n$(date) | $1\n=========================================================================\n"
}
@@ -5,6 +5,10 @@
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
# Reports per data stream Elasticsearch growth over the last 24 hours, 7 days and 30 days. The data
# source follows telegraf.output: Postgres in POSTGRES mode, where history is limited to
# postgres.telegraf.retention_days and the script must run on the manager; InfluxDB otherwise.
INFLUX_URL="https://localhost:8086/api/v2"
. /usr/sbin/so-common
@@ -18,8 +22,6 @@ lookup_org_id() {
echo "$response" | jq -r ".orgs[] | select(.name == \"Security Onion\").id"
}
ORG_ID=$(lookup_org_id)
run_flux_query() {
local query=$1
request "query?org=$ORG_ID" -H 'Accept:application/csv' -H 'Content-type:application/vnd.flux' -d "$query" -XPOST 2>/dev/null
@@ -39,75 +41,137 @@ bytes_to_gb() {
fi
}
indexes_query='from(bucket: "telegraf/so_long_term")
|> range(start: -7d)
|> filter(fn: (r) => r._measurement == "elasticsearch_index_size")
|> distinct(column: "_field")
|> keep(columns: ["_field"])'
influxdb_indices_growth() {
ORG_ID=$(lookup_org_id)
indexes_result=$(run_flux_query "$indexes_query")
indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | grep -v '^_field$' | sed 's/\r$//' | sort -u)
indexes_query='from(bucket: "telegraf/so_long_term")
|> range(start: -7d)
|> filter(fn: (r) => r._measurement == "elasticsearch_index_size")
|> distinct(column: "_field")
|> keep(columns: ["_field"])'
printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)"
printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})"
indexes_result=$(run_flux_query "$indexes_query")
indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | grep -v '^_field$' | sed 's/\r$//' | sort -u)
for index in $indexes; do
[[ -z "$index" ]] && continue
current_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -4h)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
current_result=$(run_flux_query "$current_query")
current_size=$(read_csv_result "$current_result")
current_size=${current_size:-0}
printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)"
printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})"
size_24h_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -25h, stop: -23h)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
size_24h_result=$(run_flux_query "$size_24h_query")
size_24h_ago=$(read_csv_result "$size_24h_result")
size_24h_ago=${size_24h_ago:-$current_size}
for index in $indexes; do
[[ -z "$index" ]] && continue
current_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -4h)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
current_result=$(run_flux_query "$current_query")
current_size=$(read_csv_result "$current_result")
current_size=${current_size:-0}
size_7d_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -7d8h, stop: -7d)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
size_7d_result=$(run_flux_query "$size_7d_query")
size_7d_ago=$(read_csv_result "$size_7d_result")
size_7d_ago=${size_7d_ago:-$current_size}
size_24h_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -25h, stop: -23h)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
size_24h_result=$(run_flux_query "$size_24h_query")
size_24h_ago=$(read_csv_result "$size_24h_result")
size_24h_ago=${size_24h_ago:-$current_size}
size_30d_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -30d8h, stop: -30d)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
size_30d_result=$(run_flux_query "$size_30d_query")
size_30d_ago=$(read_csv_result "$size_30d_result")
size_30d_ago=${size_30d_ago:-$current_size}
size_7d_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -7d8h, stop: -7d)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
size_7d_result=$(run_flux_query "$size_7d_query")
size_7d_ago=$(read_csv_result "$size_7d_result")
size_7d_ago=${size_7d_ago:-$current_size}
# if an index was recently cleaned up by ilm it will result in a negative number for 'index growth'.
growth_24h=$(( current_size > size_24h_ago ? current_size - size_24h_ago : 0 ))
size_30d_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -30d8h, stop: -30d)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
size_30d_result=$(run_flux_query "$size_30d_query")
size_30d_ago=$(read_csv_result "$size_30d_result")
size_30d_ago=${size_30d_ago:-$current_size}
growth_7d=$(( current_size > size_7d_ago ? current_size - size_7d_ago : 0 ))
# if an index was recently cleaned up by ilm it will result in a negative number for 'index growth'.
growth_24h=$(( current_size > size_24h_ago ? current_size - size_24h_ago : 0 ))
growth_30d=$(( current_size > size_30d_ago ? current_size - size_30d_ago : 0 ))
growth_7d=$(( current_size > size_7d_ago ? current_size - size_7d_ago : 0 ))
growth_24h_gb=$(bytes_to_gb "$growth_24h")
growth_7d_gb=$(bytes_to_gb "$growth_7d")
growth_30d_gb=$(bytes_to_gb "$growth_30d")
growth_30d=$(( current_size > size_30d_ago ? current_size - size_30d_ago : 0 ))
# Only results for indices with atleast 1 metric above 0.00
if [[ "$growth_24h_gb" != "0.00" ]] || [[ "$growth_7d_gb" != "0.00" ]] || [[ "$growth_30d_gb" != "0.00" ]]; then
printf "%020.2f|%-50s %15s %15s %15s\n" \
"$growth_24h" \
"$index" \
"$growth_24h_gb" \
"$growth_7d_gb" \
"$growth_30d_gb"
fi
done | sort -t'|' -k1,1nr | cut -d'|' -f2-
growth_24h_gb=$(bytes_to_gb "$growth_24h")
growth_7d_gb=$(bytes_to_gb "$growth_7d")
growth_30d_gb=$(bytes_to_gb "$growth_30d")
# Only results for indices with atleast 1 metric above 0.00
if [[ "$growth_24h_gb" != "0.00" ]] || [[ "$growth_7d_gb" != "0.00" ]] || [[ "$growth_30d_gb" != "0.00" ]]; then
printf "%020.2f|%-50s %15s %15s %15s\n" \
"$growth_24h" \
"$index" \
"$growth_24h_gb" \
"$growth_7d_gb" \
"$growth_30d_gb"
fi
done | sort -t'|' -k1,1nr | cut -d'|' -f2-
}
postgres_indices_growth() {
local app="so-elasticsearch-indices-growth"
local -a starts=("25 hours" "7 days 8 hours" "30 days 8 hours")
local -a stops=("23 hours" "7 days" "30 days")
local -A current past window_has_data
local i rows stream current_size past_size growth growth_gb sort_key show
local -a columns
for i in 0 1 2; do
if ! rows=$(so_es_index_size_growth "$app" "${starts[i]}" "${stops[i]}"); then
echo "Failed to query telegraf.elasticsearch_index_size in so_telegraf." >&2
exit 1
fi
[[ -z "$rows" ]] && continue
window_has_data[$i]=1
while IFS='|' read -r stream current_size past_size; do
current[$stream]=$current_size
past[$i,$stream]=$past_size
done <<< "$rows"
done
printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)"
printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})"
for stream in "${!current[@]}"; do
columns=()
sort_key=0
show=false
for i in 0 1 2; do
if [[ -z "${window_has_data[$i]}" ]]; then
columns+=("n/a")
continue
fi
current_size=${current[$stream]}
past_size=${past[$i,$stream]-$current_size}
growth=$(( current_size > past_size ? current_size - past_size : 0 ))
growth_gb=$(bytes_to_gb "$growth")
[[ "$growth_gb" != "0.00" ]] && show=true
[[ $i -eq 0 ]] && sort_key=$growth
columns+=("$growth_gb")
done
if [[ "$show" == true ]]; then
printf "%020.2f|%-50s %15s %15s %15s\n" "$sort_key" "$stream" "${columns[@]}"
fi
done | sort -t'|' -k1,1nr | cut -d'|' -f2-
}
if ! TG_OUTPUT=$(so_telegraf_output); then
echo "Unable to read telegraf.output from pillar." >&2
exit 1
fi
if [[ "$TG_OUTPUT" == "POSTGRES" ]]; then
so_telegraf_pg_check so-elasticsearch-indices-growth || exit 1
postgres_indices_growth
else
influxdb_indices_growth
fi
@@ -58,6 +58,9 @@ Usage: $(basename "$0") [OPTIONS]
Estimate remaining days until Elasticsearch cluster reaches low watermark threshold.
Index growth history follows telegraf.output: in POSTGRES mode it is read from Postgres, limited to
postgres.telegraf.retention_days, and the script must run on the manager; otherwise from InfluxDB.
OPTIONS:
--json Output results in JSON format
-v, --verbose Show additional output
@@ -168,11 +171,25 @@ run_indices_growth() {
fi
}
fetch_postgres_total_bytes() {
local total
total=$(so_es_index_size_growth so-elasticsearch-retention-estimate "$1" "$2") || return 1
echo "$total" | awk -F'|' '{ sum += $3 } END { print sum + 0 }'
}
fetch_total_bytes() {
local start="$1"
local stop="$2"
local pg_start="$3"
local pg_stop="$4"
local range_line
if [ "$TG_OUTPUT" = "POSTGRES" ]; then
fetch_postgres_total_bytes "$pg_start" "$pg_stop"
return
fi
if [ -n "$stop" ]; then
range_line=" |> range(start: ${start}, stop: ${stop})"
else
@@ -208,8 +225,13 @@ fail() {
echo -e "\nDISCLAIMER: Script output is based on current data patterns, but are approximations solely intended to assist with getting a general ILM policy configured."
ORG_ID=$(lookup_org_id)
[ -n "$ORG_ID" ] || fail "Unable to resolve InfluxDB org id"
TG_OUTPUT=$(so_telegraf_output) || fail "Unable to read telegraf.output from pillar."
if [ "$TG_OUTPUT" = "POSTGRES" ]; then
pg_check_error=$(so_telegraf_pg_check so-elasticsearch-retention-estimate 2>&1) || fail "$pg_check_error"
else
ORG_ID=$(lookup_org_id)
[ -n "$ORG_ID" ] || fail "Unable to resolve InfluxDB org id"
fi
cluster_storage_size=0
indexed_storage_source="elasticsearch"
@@ -221,10 +243,11 @@ if [ -n "$cluster_storage_size_output" ]; then
fi
fi
# historical data from influxdb for growth calculation
one_day_total=$(fetch_total_bytes "-25h" "-23h")
seven_day_total=$(fetch_total_bytes "-7d8h" "-7d")
thirty_day_total=$(fetch_total_bytes "-30d8h" "-30d")
# historical index sizes for growth calculation
query_error="Failed to query telegraf.elasticsearch_index_size in so_telegraf."
one_day_total=$(fetch_total_bytes "-25h" "-23h" "25 hours" "23 hours") || fail "$query_error"
seven_day_total=$(fetch_total_bytes "-7d8h" "-7d" "7 days 8 hours" "7 days") || fail "$query_error"
thirty_day_total=$(fetch_total_bytes "-30d8h" "-30d" "30 days 8 hours" "30 days") || fail "$query_error"
# available historical windows (prefer 30d/7d when available, to avoid using recent 24h traffic spike as true daily ingest rate)
history_days=0
@@ -244,7 +267,12 @@ elif [ "$one_day_total" -gt 0 ]; then
historical_total=$one_day_total
fi
[ "$history_days" -gt 0 ] || fail "Historical InfluxDB data unavailable for growth calculation. If this a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics."
if [ "$TG_OUTPUT" = "POSTGRES" ]; then
history_unavailable="Historical Postgres data unavailable for growth calculation. Postgres keeps postgres.telegraf.retention_days days of history; if this is a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics."
else
history_unavailable="Historical InfluxDB data unavailable for growth calculation. If this a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics."
fi
[ "$history_days" -gt 0 ] || fail "$history_unavailable"
# Daily growth rate
growth_bytes=$(( cluster_storage_size - historical_total ))
@@ -729,6 +757,7 @@ if [ "$should_trigger_recommendations" = true ]; then
[ -n "$index" ] || continue
growth_24h_gb=$(echo "$line" | awk '{print $(NF-2)}')
[[ "$growth_24h_gb" =~ ^[0-9]+(\.[0-9]+)?$ ]] || growth_24h_gb=0
creation_date_display=""
retention_days=""
+31
View File
@@ -9,6 +9,7 @@
{%- set MDENGINE = GLOBALS.md_engine %}
{%- set LOGSTASH_ENABLED = LOGSTASH_MERGED.enabled %}
{%- set TG_OUT = TELEGRAFMERGED.output | upper %}
{%- set PG_ONLY = TG_OUT == 'POSTGRES' %}
{%- set PG_HOST = GLOBALS.manager_ip %}
{#- Per-minion telegraf creds live in the grid-wide telegraf/creds.sls pillar,
written by /usr/sbin/so-telegraf-cred on the manager. Each minion looks up
@@ -149,7 +150,12 @@
# Read metrics about cpu usage
[[inputs.cpu]]
## Whether to report per-cpu stats or not
{%- if PG_ONLY %}
{#- SOC and so-stats-show read only cpu-total #}
percpu = false
{%- else %}
percpu = true
{%- endif %}
## Whether to report total system cpu stats or not
totalcpu = true
## If true, collect raw CPU time metrics.
@@ -199,6 +205,11 @@
## The typical use case is for LVM volumes, to get the VG/LV name instead of
## the near-meaningless DM-0 name.
# name_templates = ["$ID_FS_LABEL","$DM_VG_NAME/$DM_LV_NAME"]
{%- if PG_ONLY %}
{#- Whole disks, md RAID included, already carry the I/O of their partitions and dm devices #}
[inputs.diskio.tagdrop]
name = ["sd*[0-9]", "vd*[0-9]", "xvd*[0-9]", "hd*[0-9]", "nvme*p[0-9]*", "mmcblk*p[0-9]*", "dm-*", "sr*", "loop*"]
{%- endif %}
# Get kernel statistics from /proc/stat
@@ -242,6 +253,12 @@
password = "{{ ES_PASS }}"
insecure_skip_verify = true
cluster_health = true
{%- if PG_ONLY %}
{#- SOC reads cluster status, indices and clusterstats_nodes ingest stats; alarms plan jvm and fs #}
cluster_health_level = "cluster"
node_stats = ["indices", "jvm", "fs"]
namedrop = ["elasticsearch_clusterstats_indices"]
{%- endif %}
{%- elif grains['role'] in ['so-searchnode'] %}
[[inputs.elasticsearch]]
servers = ["https://{{ NODEIP }}:9200"]
@@ -249,6 +266,10 @@
username = "{{ ES_USER }}"
password = "{{ ES_PASS }}"
insecure_skip_verify = true
{%- if PG_ONLY %}
{#- SOC reads indices; alarms plan jvm and fs #}
node_stats = ["indices", "jvm", "fs"]
{%- endif %}
{%- endif %}
# ## Timeout for HTTP requests to the elastic search server(s)
@@ -291,6 +312,10 @@
collect = ["pipelines"]
username = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user') }}"
password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}"
{%- if PG_ONLY %}
{#- SOC reads logstash_events only #}
namedrop = ["logstash_plugins"]
{%- endif %}
{%- endif %}
{% if grains.role in ['so-manager','so-managerhype','so-managersearch','so-standalone','so-receiver'] and GLOBALS.pipeline == "KAFKA" -%}
@@ -392,6 +417,12 @@
# # Read metrics about network interface usage
[[inputs.net]]
{%- if PG_ONLY %}
{%- from 'docker/docker.map.jinja' import DOCKERMERGED %}
{#- Only virtual interfaces; every physical NIC, bond members included, is kept for SOC and alarms #}
[inputs.net.tagdrop]
interface = {{ (['veth*', 'docker*', 'lo'] + DOCKERMERGED.networks.keys() | list) | json }}
{%- endif %}
# Scripts run every 30s||TELEGRAFMERGED.config.interval - ES index script doesn't need to run as frequently
{%- if GLOBALS.is_manager or GLOBALS.role == 'so-heavynode' %}