diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index c8ebae084..929103de5 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -656,6 +656,108 @@ set_version() { fi } +so_telegraf_output() { + local defaultsfile="${DEFAULT_SALT_DIR}/salt/telegraf/defaults.yaml" + local output + + output=$(lookup_pillar output telegraf) || return 1 + if [ -z "$output" ]; then + if [ ! -f "$defaultsfile" ]; then + echo "Could not find $defaultsfile" >&2 + return 1 + fi + output=$(grep -E "^ output: " "$defaultsfile" | awk -F: '{print $2}' | tr -d '[:space:]') + fi + echo "${output^^}" +} + +so_telegraf_pg_check() { + local app="$1" + local databases role_ok + + if ! docker_check_running postgres; then + echo "so-postgres is not running. Run $app on the manager." >&2 + return 1 + fi + if ! databases=$(docker exec so-postgres psql -U postgres -lqt 2>/dev/null); then + echo "Unable to list the so-postgres databases." >&2 + return 1 + fi + if ! echo "$databases" | cut -d\| -f1 | grep -qw so_telegraf; then + echo "Database so_telegraf not found. Is telegraf.output set to POSTGRES or BOTH?" >&2 + return 1 + fi + if ! role_ok=$(docker exec so-postgres psql -U postgres -tAc "SELECT rolcanlogin AND NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls) FROM pg_roles WHERE rolname = 'so_maint_telegraf'" 2>/dev/null); then + echo "Unable to query the so-postgres roles." >&2 + return 1 + fi + if [ -z "$role_ok" ]; then + echo "The so_maint_telegraf role is missing; run a highstate." >&2 + return 1 + elif [ "$role_ok" != t ]; then + echo "The so_maint_telegraf role is misconfigured; run a highstate." >&2 + return 1 + fi +} + +so_telegraf_psql() { + local app="$1" + local sql="$2" + + docker exec -e PGOPTIONS="-c application_name=${app} -c default_transaction_read_only=on -c client_min_messages=error" \ + so-postgres psql -U so_maint_telegraf -d so_telegraf -v ON_ERROR_STOP=1 -At -F '|' -c "$sql" +} + +# Prints one stream|current|then line per data stream, in bytes summed across hosts. current is +# each host's newest size in the past 4 hours; then is that host's newest size in the window +# [now - start, now - stop), or its current size when it has none there, so a host missing from the +# window adds no growth. Empty output means the window has no rows at all. Values over 15 digits +# (1 PB) are skipped so the sums here and in the callers stay far below the 64-bit limit; such a +# value can only come from a forged Telegraf writer. +so_es_index_size_growth() { + local app="$1" + local start="$2" + local stop="$3" + local relkind + + relkind=$(so_telegraf_psql "$app" "SELECT c.relkind FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'telegraf' AND c.relname = 'elasticsearch_index_size'") || return 1 + [[ "$relkind" == "r" || "$relkind" == "p" ]] || return 0 + + so_telegraf_psql "$app" "$(cat <= now() - interval '${start}' + AND f.key ~ '^[^|"''[:space:][:cntrl:]]+$' + AND pg_catalog.strpos(f.key, pg_catalog.chr(92)) = 0 + AND f.value ~ '^[0-9]{1,15}$' +), +cur AS ( + SELECT DISTINCT ON (tag_id, stream) tag_id, stream, bytes + FROM sizes + WHERE time >= now() - interval '4 hours' + ORDER BY tag_id, stream, time DESC +), +past AS ( + SELECT DISTINCT ON (tag_id, stream) tag_id, stream, bytes + FROM sizes + WHERE time < now() - interval '${stop}' + ORDER BY tag_id, stream, time DESC +) +SELECT c.stream, sum(c.bytes), sum(coalesce(p.bytes, c.bytes)) +FROM cur c +LEFT JOIN past p USING (tag_id, stream) +WHERE EXISTS ( + SELECT 1 FROM telegraf.elasticsearch_index_size + WHERE time >= now() - interval '${start}' AND time < now() - interval '${stop}' +) +GROUP BY c.stream +ORDER BY c.stream +EOF +)" +} + status () { printf "\n=========================================================================\n$(date) | $1\n=========================================================================\n" } diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth index 3381947eb..9acdaafcd 100644 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth @@ -5,6 +5,10 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +# Reports per data stream Elasticsearch growth over the last 24 hours, 7 days and 30 days. The data +# source follows telegraf.output: Postgres in POSTGRES mode, where history is limited to +# postgres.telegraf.retention_days and the script must run on the manager; InfluxDB otherwise. + INFLUX_URL="https://localhost:8086/api/v2" . /usr/sbin/so-common @@ -18,8 +22,6 @@ lookup_org_id() { echo "$response" | jq -r ".orgs[] | select(.name == \"Security Onion\").id" } -ORG_ID=$(lookup_org_id) - run_flux_query() { local query=$1 request "query?org=$ORG_ID" -H 'Accept:application/csv' -H 'Content-type:application/vnd.flux' -d "$query" -XPOST 2>/dev/null @@ -39,75 +41,137 @@ bytes_to_gb() { fi } -indexes_query='from(bucket: "telegraf/so_long_term") -|> range(start: -7d) -|> filter(fn: (r) => r._measurement == "elasticsearch_index_size") -|> distinct(column: "_field") -|> keep(columns: ["_field"])' +influxdb_indices_growth() { + ORG_ID=$(lookup_org_id) -indexes_result=$(run_flux_query "$indexes_query") -indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | grep -v '^_field$' | sed 's/\r$//' | sort -u) + indexes_query='from(bucket: "telegraf/so_long_term") + |> range(start: -7d) + |> filter(fn: (r) => r._measurement == "elasticsearch_index_size") + |> distinct(column: "_field") + |> keep(columns: ["_field"])' -printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)" -printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" + indexes_result=$(run_flux_query "$indexes_query") + indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | grep -v '^_field$' | sed 's/\r$//' | sort -u) -for index in $indexes; do - [[ -z "$index" ]] && continue - current_query="from(bucket: \"telegraf/so_long_term\") - |> range(start: -4h) - |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") - |> last() - |> keep(columns: [\"_value\"])" - current_result=$(run_flux_query "$current_query") - current_size=$(read_csv_result "$current_result") - current_size=${current_size:-0} + printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)" + printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" - size_24h_query="from(bucket: \"telegraf/so_long_term\") - |> range(start: -25h, stop: -23h) - |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") - |> last() - |> keep(columns: [\"_value\"])" - size_24h_result=$(run_flux_query "$size_24h_query") - size_24h_ago=$(read_csv_result "$size_24h_result") - size_24h_ago=${size_24h_ago:-$current_size} + for index in $indexes; do + [[ -z "$index" ]] && continue + current_query="from(bucket: \"telegraf/so_long_term\") + |> range(start: -4h) + |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") + |> last() + |> keep(columns: [\"_value\"])" + current_result=$(run_flux_query "$current_query") + current_size=$(read_csv_result "$current_result") + current_size=${current_size:-0} - size_7d_query="from(bucket: \"telegraf/so_long_term\") - |> range(start: -7d8h, stop: -7d) - |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") - |> last() - |> keep(columns: [\"_value\"])" - size_7d_result=$(run_flux_query "$size_7d_query") - size_7d_ago=$(read_csv_result "$size_7d_result") - size_7d_ago=${size_7d_ago:-$current_size} + size_24h_query="from(bucket: \"telegraf/so_long_term\") + |> range(start: -25h, stop: -23h) + |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") + |> last() + |> keep(columns: [\"_value\"])" + size_24h_result=$(run_flux_query "$size_24h_query") + size_24h_ago=$(read_csv_result "$size_24h_result") + size_24h_ago=${size_24h_ago:-$current_size} - size_30d_query="from(bucket: \"telegraf/so_long_term\") - |> range(start: -30d8h, stop: -30d) - |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") - |> last() - |> keep(columns: [\"_value\"])" - size_30d_result=$(run_flux_query "$size_30d_query") - size_30d_ago=$(read_csv_result "$size_30d_result") - size_30d_ago=${size_30d_ago:-$current_size} + size_7d_query="from(bucket: \"telegraf/so_long_term\") + |> range(start: -7d8h, stop: -7d) + |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") + |> last() + |> keep(columns: [\"_value\"])" + size_7d_result=$(run_flux_query "$size_7d_query") + size_7d_ago=$(read_csv_result "$size_7d_result") + size_7d_ago=${size_7d_ago:-$current_size} - # if an index was recently cleaned up by ilm it will result in a negative number for 'index growth'. - growth_24h=$(( current_size > size_24h_ago ? current_size - size_24h_ago : 0 )) + size_30d_query="from(bucket: \"telegraf/so_long_term\") + |> range(start: -30d8h, stop: -30d) + |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") + |> last() + |> keep(columns: [\"_value\"])" + size_30d_result=$(run_flux_query "$size_30d_query") + size_30d_ago=$(read_csv_result "$size_30d_result") + size_30d_ago=${size_30d_ago:-$current_size} - growth_7d=$(( current_size > size_7d_ago ? current_size - size_7d_ago : 0 )) + # if an index was recently cleaned up by ilm it will result in a negative number for 'index growth'. + growth_24h=$(( current_size > size_24h_ago ? current_size - size_24h_ago : 0 )) - growth_30d=$(( current_size > size_30d_ago ? current_size - size_30d_ago : 0 )) + growth_7d=$(( current_size > size_7d_ago ? current_size - size_7d_ago : 0 )) - growth_24h_gb=$(bytes_to_gb "$growth_24h") - growth_7d_gb=$(bytes_to_gb "$growth_7d") - growth_30d_gb=$(bytes_to_gb "$growth_30d") + growth_30d=$(( current_size > size_30d_ago ? current_size - size_30d_ago : 0 )) - # Only results for indices with atleast 1 metric above 0.00 - if [[ "$growth_24h_gb" != "0.00" ]] || [[ "$growth_7d_gb" != "0.00" ]] || [[ "$growth_30d_gb" != "0.00" ]]; then - printf "%020.2f|%-50s %15s %15s %15s\n" \ - "$growth_24h" \ - "$index" \ - "$growth_24h_gb" \ - "$growth_7d_gb" \ - "$growth_30d_gb" - fi -done | sort -t'|' -k1,1nr | cut -d'|' -f2- + growth_24h_gb=$(bytes_to_gb "$growth_24h") + growth_7d_gb=$(bytes_to_gb "$growth_7d") + growth_30d_gb=$(bytes_to_gb "$growth_30d") + # Only results for indices with atleast 1 metric above 0.00 + if [[ "$growth_24h_gb" != "0.00" ]] || [[ "$growth_7d_gb" != "0.00" ]] || [[ "$growth_30d_gb" != "0.00" ]]; then + printf "%020.2f|%-50s %15s %15s %15s\n" \ + "$growth_24h" \ + "$index" \ + "$growth_24h_gb" \ + "$growth_7d_gb" \ + "$growth_30d_gb" + fi + done | sort -t'|' -k1,1nr | cut -d'|' -f2- +} + +postgres_indices_growth() { + local app="so-elasticsearch-indices-growth" + local -a starts=("25 hours" "7 days 8 hours" "30 days 8 hours") + local -a stops=("23 hours" "7 days" "30 days") + local -A current past window_has_data + local i rows stream current_size past_size growth growth_gb sort_key show + local -a columns + + for i in 0 1 2; do + if ! rows=$(so_es_index_size_growth "$app" "${starts[i]}" "${stops[i]}"); then + echo "Failed to query telegraf.elasticsearch_index_size in so_telegraf." >&2 + exit 1 + fi + [[ -z "$rows" ]] && continue + window_has_data[$i]=1 + while IFS='|' read -r stream current_size past_size; do + current[$stream]=$current_size + past[$i,$stream]=$past_size + done <<< "$rows" + done + + printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)" + printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" + + for stream in "${!current[@]}"; do + columns=() + sort_key=0 + show=false + for i in 0 1 2; do + if [[ -z "${window_has_data[$i]}" ]]; then + columns+=("n/a") + continue + fi + current_size=${current[$stream]} + past_size=${past[$i,$stream]-$current_size} + growth=$(( current_size > past_size ? current_size - past_size : 0 )) + growth_gb=$(bytes_to_gb "$growth") + [[ "$growth_gb" != "0.00" ]] && show=true + [[ $i -eq 0 ]] && sort_key=$growth + columns+=("$growth_gb") + done + if [[ "$show" == true ]]; then + printf "%020.2f|%-50s %15s %15s %15s\n" "$sort_key" "$stream" "${columns[@]}" + fi + done | sort -t'|' -k1,1nr | cut -d'|' -f2- +} + +if ! TG_OUTPUT=$(so_telegraf_output); then + echo "Unable to read telegraf.output from pillar." >&2 + exit 1 +fi + +if [[ "$TG_OUTPUT" == "POSTGRES" ]]; then + so_telegraf_pg_check so-elasticsearch-indices-growth || exit 1 + postgres_indices_growth +else + influxdb_indices_growth +fi diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-retention-estimate b/salt/elasticsearch/tools/sbin/so-elasticsearch-retention-estimate index 1eb1e33ca..3c501434e 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-retention-estimate +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-retention-estimate @@ -58,6 +58,9 @@ Usage: $(basename "$0") [OPTIONS] Estimate remaining days until Elasticsearch cluster reaches low watermark threshold. +Index growth history follows telegraf.output: in POSTGRES mode it is read from Postgres, limited to +postgres.telegraf.retention_days, and the script must run on the manager; otherwise from InfluxDB. + OPTIONS: --json Output results in JSON format -v, --verbose Show additional output @@ -168,11 +171,25 @@ run_indices_growth() { fi } +fetch_postgres_total_bytes() { + local total + + total=$(so_es_index_size_growth so-elasticsearch-retention-estimate "$1" "$2") || return 1 + echo "$total" | awk -F'|' '{ sum += $3 } END { print sum + 0 }' +} + fetch_total_bytes() { local start="$1" local stop="$2" + local pg_start="$3" + local pg_stop="$4" local range_line + if [ "$TG_OUTPUT" = "POSTGRES" ]; then + fetch_postgres_total_bytes "$pg_start" "$pg_stop" + return + fi + if [ -n "$stop" ]; then range_line=" |> range(start: ${start}, stop: ${stop})" else @@ -208,8 +225,13 @@ fail() { echo -e "\nDISCLAIMER: Script output is based on current data patterns, but are approximations solely intended to assist with getting a general ILM policy configured." -ORG_ID=$(lookup_org_id) -[ -n "$ORG_ID" ] || fail "Unable to resolve InfluxDB org id" +TG_OUTPUT=$(so_telegraf_output) || fail "Unable to read telegraf.output from pillar." +if [ "$TG_OUTPUT" = "POSTGRES" ]; then + pg_check_error=$(so_telegraf_pg_check so-elasticsearch-retention-estimate 2>&1) || fail "$pg_check_error" +else + ORG_ID=$(lookup_org_id) + [ -n "$ORG_ID" ] || fail "Unable to resolve InfluxDB org id" +fi cluster_storage_size=0 indexed_storage_source="elasticsearch" @@ -221,10 +243,11 @@ if [ -n "$cluster_storage_size_output" ]; then fi fi -# historical data from influxdb for growth calculation -one_day_total=$(fetch_total_bytes "-25h" "-23h") -seven_day_total=$(fetch_total_bytes "-7d8h" "-7d") -thirty_day_total=$(fetch_total_bytes "-30d8h" "-30d") +# historical index sizes for growth calculation +query_error="Failed to query telegraf.elasticsearch_index_size in so_telegraf." +one_day_total=$(fetch_total_bytes "-25h" "-23h" "25 hours" "23 hours") || fail "$query_error" +seven_day_total=$(fetch_total_bytes "-7d8h" "-7d" "7 days 8 hours" "7 days") || fail "$query_error" +thirty_day_total=$(fetch_total_bytes "-30d8h" "-30d" "30 days 8 hours" "30 days") || fail "$query_error" # available historical windows (prefer 30d/7d when available, to avoid using recent 24h traffic spike as true daily ingest rate) history_days=0 @@ -244,7 +267,12 @@ elif [ "$one_day_total" -gt 0 ]; then historical_total=$one_day_total fi -[ "$history_days" -gt 0 ] || fail "Historical InfluxDB data unavailable for growth calculation. If this a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics." +if [ "$TG_OUTPUT" = "POSTGRES" ]; then + history_unavailable="Historical Postgres data unavailable for growth calculation. Postgres keeps postgres.telegraf.retention_days days of history; if this is a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics." +else + history_unavailable="Historical InfluxDB data unavailable for growth calculation. If this a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics." +fi +[ "$history_days" -gt 0 ] || fail "$history_unavailable" # Daily growth rate growth_bytes=$(( cluster_storage_size - historical_total )) @@ -729,6 +757,7 @@ if [ "$should_trigger_recommendations" = true ]; then [ -n "$index" ] || continue growth_24h_gb=$(echo "$line" | awk '{print $(NF-2)}') + [[ "$growth_24h_gb" =~ ^[0-9]+(\.[0-9]+)?$ ]] || growth_24h_gb=0 creation_date_display="" retention_days="" diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 0b2bb9bb6..94bf0af81 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -9,6 +9,7 @@ {%- set MDENGINE = GLOBALS.md_engine %} {%- set LOGSTASH_ENABLED = LOGSTASH_MERGED.enabled %} {%- set TG_OUT = TELEGRAFMERGED.output | upper %} +{%- set PG_ONLY = TG_OUT == 'POSTGRES' %} {%- set PG_HOST = GLOBALS.manager_ip %} {#- Per-minion telegraf creds live in the grid-wide telegraf/creds.sls pillar, written by /usr/sbin/so-telegraf-cred on the manager. Each minion looks up @@ -149,7 +150,12 @@ # Read metrics about cpu usage [[inputs.cpu]] ## Whether to report per-cpu stats or not +{%- if PG_ONLY %} + {#- SOC and so-stats-show read only cpu-total #} + percpu = false +{%- else %} percpu = true +{%- endif %} ## Whether to report total system cpu stats or not totalcpu = true ## If true, collect raw CPU time metrics. @@ -199,6 +205,11 @@ ## The typical use case is for LVM volumes, to get the VG/LV name instead of ## the near-meaningless DM-0 name. # name_templates = ["$ID_FS_LABEL","$DM_VG_NAME/$DM_LV_NAME"] +{%- if PG_ONLY %} + {#- Whole disks, md RAID included, already carry the I/O of their partitions and dm devices #} + [inputs.diskio.tagdrop] + name = ["sd*[0-9]", "vd*[0-9]", "xvd*[0-9]", "hd*[0-9]", "nvme*p[0-9]*", "mmcblk*p[0-9]*", "dm-*", "sr*", "loop*"] +{%- endif %} # Get kernel statistics from /proc/stat @@ -242,6 +253,12 @@ password = "{{ ES_PASS }}" insecure_skip_verify = true cluster_health = true +{%- if PG_ONLY %} + {#- SOC reads cluster status, indices and clusterstats_nodes ingest stats; alarms plan jvm and fs #} + cluster_health_level = "cluster" + node_stats = ["indices", "jvm", "fs"] + namedrop = ["elasticsearch_clusterstats_indices"] +{%- endif %} {%- elif grains['role'] in ['so-searchnode'] %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] @@ -249,6 +266,10 @@ username = "{{ ES_USER }}" password = "{{ ES_PASS }}" insecure_skip_verify = true +{%- if PG_ONLY %} + {#- SOC reads indices; alarms plan jvm and fs #} + node_stats = ["indices", "jvm", "fs"] +{%- endif %} {%- endif %} # ## Timeout for HTTP requests to the elastic search server(s) @@ -291,6 +312,10 @@ collect = ["pipelines"] username = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user') }}" password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}" +{%- if PG_ONLY %} + {#- SOC reads logstash_events only #} + namedrop = ["logstash_plugins"] +{%- endif %} {%- endif %} {% if grains.role in ['so-manager','so-managerhype','so-managersearch','so-standalone','so-receiver'] and GLOBALS.pipeline == "KAFKA" -%} @@ -392,6 +417,12 @@ # # Read metrics about network interface usage [[inputs.net]] +{%- if PG_ONLY %} +{%- from 'docker/docker.map.jinja' import DOCKERMERGED %} + {#- Only virtual interfaces; every physical NIC, bond members included, is kept for SOC and alarms #} + [inputs.net.tagdrop] + interface = {{ (['veth*', 'docker*', 'lo'] + DOCKERMERGED.networks.keys() | list) | json }} +{%- endif %} # Scripts run every 30s||TELEGRAFMERGED.config.interval - ES index script doesn't need to run as frequently {%- if GLOBALS.is_manager or GLOBALS.role == 'so-heavynode' %}