mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-11-18 18:16:02 +01:00
Compare commits
22
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6d7be47a69 | ||
|
|
e51706acad | ||
|
|
9612b15fca | ||
|
|
0f5170e17e | ||
|
|
cac027d438 | ||
|
|
dea6932342 | ||
|
|
043b2185d6 | ||
|
|
e7bb00d19d | ||
|
|
8c3611ac72 | ||
|
|
5558c54632 | ||
|
|
52f9fe3092 | ||
|
|
f492e87f20 | ||
|
|
01518ec25e | ||
|
|
5f3c1c84a4 | ||
|
|
b01f01da82 | ||
|
|
03317ba5e4 | ||
|
|
9bc3ae6b26 | ||
|
|
40a30f6908 | ||
|
|
4e0558d2d3 | ||
|
|
1fd37251c9 | ||
|
|
4a5253f688 | ||
|
|
ec220572f0 |
No files matched your search
@@ -656,6 +656,108 @@ set_version() {
|
||||
fi
|
||||
}
|
||||
|
||||
so_telegraf_output() {
|
||||
local defaultsfile="${DEFAULT_SALT_DIR}/salt/telegraf/defaults.yaml"
|
||||
local output
|
||||
|
||||
output=$(lookup_pillar output telegraf) || return 1
|
||||
if [ -z "$output" ]; then
|
||||
if [ ! -f "$defaultsfile" ]; then
|
||||
echo "Could not find $defaultsfile" >&2
|
||||
return 1
|
||||
fi
|
||||
output=$(grep -E "^ output: " "$defaultsfile" | awk -F: '{print $2}' | tr -d '[:space:]')
|
||||
fi
|
||||
echo "${output^^}"
|
||||
}
|
||||
|
||||
so_telegraf_pg_check() {
|
||||
local app="$1"
|
||||
local databases role_ok
|
||||
|
||||
if ! docker_check_running postgres; then
|
||||
echo "so-postgres is not running. Run $app on the manager." >&2
|
||||
return 1
|
||||
fi
|
||||
if ! databases=$(docker exec so-postgres psql -U postgres -lqt 2>/dev/null); then
|
||||
echo "Unable to list the so-postgres databases." >&2
|
||||
return 1
|
||||
fi
|
||||
if ! echo "$databases" | cut -d\| -f1 | grep -qw so_telegraf; then
|
||||
echo "Database so_telegraf not found. Is telegraf.output set to POSTGRES or BOTH?" >&2
|
||||
return 1
|
||||
fi
|
||||
if ! role_ok=$(docker exec so-postgres psql -U postgres -tAc "SELECT rolcanlogin AND NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls) FROM pg_roles WHERE rolname = 'so_maint_telegraf'" 2>/dev/null); then
|
||||
echo "Unable to query the so-postgres roles." >&2
|
||||
return 1
|
||||
fi
|
||||
if [ -z "$role_ok" ]; then
|
||||
echo "The so_maint_telegraf role is missing; run a highstate." >&2
|
||||
return 1
|
||||
elif [ "$role_ok" != t ]; then
|
||||
echo "The so_maint_telegraf role is misconfigured; run a highstate." >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
so_telegraf_psql() {
|
||||
local app="$1"
|
||||
local sql="$2"
|
||||
|
||||
docker exec -e PGOPTIONS="-c application_name=${app} -c default_transaction_read_only=on -c client_min_messages=error" \
|
||||
so-postgres psql -U so_maint_telegraf -d so_telegraf -v ON_ERROR_STOP=1 -At -F '|' -c "$sql"
|
||||
}
|
||||
|
||||
# Prints one stream|current|then line per data stream, in bytes summed across hosts. current is
|
||||
# each host's newest size in the past 4 hours; then is that host's newest size in the window
|
||||
# [now - start, now - stop), or its current size when it has none there, so a host missing from the
|
||||
# window adds no growth. Empty output means the window has no rows at all. Values over 15 digits
|
||||
# (1 PB) are skipped so the sums here and in the callers stay far below the 64-bit limit; such a
|
||||
# value can only come from a forged Telegraf writer.
|
||||
so_es_index_size_growth() {
|
||||
local app="$1"
|
||||
local start="$2"
|
||||
local stop="$3"
|
||||
local relkind
|
||||
|
||||
relkind=$(so_telegraf_psql "$app" "SELECT c.relkind FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'telegraf' AND c.relname = 'elasticsearch_index_size'") || return 1
|
||||
[[ "$relkind" == "r" || "$relkind" == "p" ]] || return 0
|
||||
|
||||
so_telegraf_psql "$app" "$(cat <<EOF
|
||||
WITH sizes AS (
|
||||
SELECT d.tag_id, d.time, f.key AS stream, f.value::bigint AS bytes
|
||||
FROM telegraf.elasticsearch_index_size d
|
||||
CROSS JOIN LATERAL pg_catalog.jsonb_each_text(d.fields) f
|
||||
WHERE d.time >= now() - interval '${start}'
|
||||
AND f.key ~ '^[^|"''[:space:][:cntrl:]]+$'
|
||||
AND pg_catalog.strpos(f.key, pg_catalog.chr(92)) = 0
|
||||
AND f.value ~ '^[0-9]{1,15}$'
|
||||
),
|
||||
cur AS (
|
||||
SELECT DISTINCT ON (tag_id, stream) tag_id, stream, bytes
|
||||
FROM sizes
|
||||
WHERE time >= now() - interval '4 hours'
|
||||
ORDER BY tag_id, stream, time DESC
|
||||
),
|
||||
past AS (
|
||||
SELECT DISTINCT ON (tag_id, stream) tag_id, stream, bytes
|
||||
FROM sizes
|
||||
WHERE time < now() - interval '${stop}'
|
||||
ORDER BY tag_id, stream, time DESC
|
||||
)
|
||||
SELECT c.stream, sum(c.bytes), sum(coalesce(p.bytes, c.bytes))
|
||||
FROM cur c
|
||||
LEFT JOIN past p USING (tag_id, stream)
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM telegraf.elasticsearch_index_size
|
||||
WHERE time >= now() - interval '${start}' AND time < now() - interval '${stop}'
|
||||
)
|
||||
GROUP BY c.stream
|
||||
ORDER BY c.stream
|
||||
EOF
|
||||
)"
|
||||
}
|
||||
|
||||
status () {
|
||||
printf "\n=========================================================================\n$(date) | $1\n=========================================================================\n"
|
||||
}
|
||||
|
||||
@@ -103,7 +103,6 @@ docker:
|
||||
- 80:80
|
||||
- 443:443
|
||||
- 8443:8443
|
||||
- 7788:7788
|
||||
- 7789:7789
|
||||
custom_bind_mounts: []
|
||||
extra_hosts: []
|
||||
|
||||
@@ -5,6 +5,10 @@
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
# Reports per data stream Elasticsearch growth over the last 24 hours, 7 days and 30 days. The data
|
||||
# source follows telegraf.output: Postgres in POSTGRES mode, where history is limited to
|
||||
# postgres.telegraf.retention_days and the script must run on the manager; InfluxDB otherwise.
|
||||
|
||||
INFLUX_URL="https://localhost:8086/api/v2"
|
||||
|
||||
. /usr/sbin/so-common
|
||||
@@ -18,8 +22,6 @@ lookup_org_id() {
|
||||
echo "$response" | jq -r ".orgs[] | select(.name == \"Security Onion\").id"
|
||||
}
|
||||
|
||||
ORG_ID=$(lookup_org_id)
|
||||
|
||||
run_flux_query() {
|
||||
local query=$1
|
||||
request "query?org=$ORG_ID" -H 'Accept:application/csv' -H 'Content-type:application/vnd.flux' -d "$query" -XPOST 2>/dev/null
|
||||
@@ -39,75 +41,137 @@ bytes_to_gb() {
|
||||
fi
|
||||
}
|
||||
|
||||
indexes_query='from(bucket: "telegraf/so_long_term")
|
||||
|> range(start: -7d)
|
||||
|> filter(fn: (r) => r._measurement == "elasticsearch_index_size")
|
||||
|> distinct(column: "_field")
|
||||
|> keep(columns: ["_field"])'
|
||||
influxdb_indices_growth() {
|
||||
ORG_ID=$(lookup_org_id)
|
||||
|
||||
indexes_result=$(run_flux_query "$indexes_query")
|
||||
indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | grep -v '^_field$' | sed 's/\r$//' | sort -u)
|
||||
indexes_query='from(bucket: "telegraf/so_long_term")
|
||||
|> range(start: -7d)
|
||||
|> filter(fn: (r) => r._measurement == "elasticsearch_index_size")
|
||||
|> distinct(column: "_field")
|
||||
|> keep(columns: ["_field"])'
|
||||
|
||||
printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)"
|
||||
printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})"
|
||||
indexes_result=$(run_flux_query "$indexes_query")
|
||||
indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | grep -v '^_field$' | sed 's/\r$//' | sort -u)
|
||||
|
||||
for index in $indexes; do
|
||||
[[ -z "$index" ]] && continue
|
||||
current_query="from(bucket: \"telegraf/so_long_term\")
|
||||
|> range(start: -4h)
|
||||
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|
||||
|> last()
|
||||
|> keep(columns: [\"_value\"])"
|
||||
current_result=$(run_flux_query "$current_query")
|
||||
current_size=$(read_csv_result "$current_result")
|
||||
current_size=${current_size:-0}
|
||||
printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)"
|
||||
printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})"
|
||||
|
||||
size_24h_query="from(bucket: \"telegraf/so_long_term\")
|
||||
|> range(start: -25h, stop: -23h)
|
||||
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|
||||
|> last()
|
||||
|> keep(columns: [\"_value\"])"
|
||||
size_24h_result=$(run_flux_query "$size_24h_query")
|
||||
size_24h_ago=$(read_csv_result "$size_24h_result")
|
||||
size_24h_ago=${size_24h_ago:-$current_size}
|
||||
for index in $indexes; do
|
||||
[[ -z "$index" ]] && continue
|
||||
current_query="from(bucket: \"telegraf/so_long_term\")
|
||||
|> range(start: -4h)
|
||||
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|
||||
|> last()
|
||||
|> keep(columns: [\"_value\"])"
|
||||
current_result=$(run_flux_query "$current_query")
|
||||
current_size=$(read_csv_result "$current_result")
|
||||
current_size=${current_size:-0}
|
||||
|
||||
size_7d_query="from(bucket: \"telegraf/so_long_term\")
|
||||
|> range(start: -7d8h, stop: -7d)
|
||||
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|
||||
|> last()
|
||||
|> keep(columns: [\"_value\"])"
|
||||
size_7d_result=$(run_flux_query "$size_7d_query")
|
||||
size_7d_ago=$(read_csv_result "$size_7d_result")
|
||||
size_7d_ago=${size_7d_ago:-$current_size}
|
||||
size_24h_query="from(bucket: \"telegraf/so_long_term\")
|
||||
|> range(start: -25h, stop: -23h)
|
||||
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|
||||
|> last()
|
||||
|> keep(columns: [\"_value\"])"
|
||||
size_24h_result=$(run_flux_query "$size_24h_query")
|
||||
size_24h_ago=$(read_csv_result "$size_24h_result")
|
||||
size_24h_ago=${size_24h_ago:-$current_size}
|
||||
|
||||
size_30d_query="from(bucket: \"telegraf/so_long_term\")
|
||||
|> range(start: -30d8h, stop: -30d)
|
||||
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|
||||
|> last()
|
||||
|> keep(columns: [\"_value\"])"
|
||||
size_30d_result=$(run_flux_query "$size_30d_query")
|
||||
size_30d_ago=$(read_csv_result "$size_30d_result")
|
||||
size_30d_ago=${size_30d_ago:-$current_size}
|
||||
size_7d_query="from(bucket: \"telegraf/so_long_term\")
|
||||
|> range(start: -7d8h, stop: -7d)
|
||||
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|
||||
|> last()
|
||||
|> keep(columns: [\"_value\"])"
|
||||
size_7d_result=$(run_flux_query "$size_7d_query")
|
||||
size_7d_ago=$(read_csv_result "$size_7d_result")
|
||||
size_7d_ago=${size_7d_ago:-$current_size}
|
||||
|
||||
# if an index was recently cleaned up by ilm it will result in a negative number for 'index growth'.
|
||||
growth_24h=$(( current_size > size_24h_ago ? current_size - size_24h_ago : 0 ))
|
||||
size_30d_query="from(bucket: \"telegraf/so_long_term\")
|
||||
|> range(start: -30d8h, stop: -30d)
|
||||
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|
||||
|> last()
|
||||
|> keep(columns: [\"_value\"])"
|
||||
size_30d_result=$(run_flux_query "$size_30d_query")
|
||||
size_30d_ago=$(read_csv_result "$size_30d_result")
|
||||
size_30d_ago=${size_30d_ago:-$current_size}
|
||||
|
||||
growth_7d=$(( current_size > size_7d_ago ? current_size - size_7d_ago : 0 ))
|
||||
# if an index was recently cleaned up by ilm it will result in a negative number for 'index growth'.
|
||||
growth_24h=$(( current_size > size_24h_ago ? current_size - size_24h_ago : 0 ))
|
||||
|
||||
growth_30d=$(( current_size > size_30d_ago ? current_size - size_30d_ago : 0 ))
|
||||
growth_7d=$(( current_size > size_7d_ago ? current_size - size_7d_ago : 0 ))
|
||||
|
||||
growth_24h_gb=$(bytes_to_gb "$growth_24h")
|
||||
growth_7d_gb=$(bytes_to_gb "$growth_7d")
|
||||
growth_30d_gb=$(bytes_to_gb "$growth_30d")
|
||||
growth_30d=$(( current_size > size_30d_ago ? current_size - size_30d_ago : 0 ))
|
||||
|
||||
# Only results for indices with atleast 1 metric above 0.00
|
||||
if [[ "$growth_24h_gb" != "0.00" ]] || [[ "$growth_7d_gb" != "0.00" ]] || [[ "$growth_30d_gb" != "0.00" ]]; then
|
||||
printf "%020.2f|%-50s %15s %15s %15s\n" \
|
||||
"$growth_24h" \
|
||||
"$index" \
|
||||
"$growth_24h_gb" \
|
||||
"$growth_7d_gb" \
|
||||
"$growth_30d_gb"
|
||||
fi
|
||||
done | sort -t'|' -k1,1nr | cut -d'|' -f2-
|
||||
growth_24h_gb=$(bytes_to_gb "$growth_24h")
|
||||
growth_7d_gb=$(bytes_to_gb "$growth_7d")
|
||||
growth_30d_gb=$(bytes_to_gb "$growth_30d")
|
||||
|
||||
# Only results for indices with atleast 1 metric above 0.00
|
||||
if [[ "$growth_24h_gb" != "0.00" ]] || [[ "$growth_7d_gb" != "0.00" ]] || [[ "$growth_30d_gb" != "0.00" ]]; then
|
||||
printf "%020.2f|%-50s %15s %15s %15s\n" \
|
||||
"$growth_24h" \
|
||||
"$index" \
|
||||
"$growth_24h_gb" \
|
||||
"$growth_7d_gb" \
|
||||
"$growth_30d_gb"
|
||||
fi
|
||||
done | sort -t'|' -k1,1nr | cut -d'|' -f2-
|
||||
}
|
||||
|
||||
postgres_indices_growth() {
|
||||
local app="so-elasticsearch-indices-growth"
|
||||
local -a starts=("25 hours" "7 days 8 hours" "30 days 8 hours")
|
||||
local -a stops=("23 hours" "7 days" "30 days")
|
||||
local -A current past window_has_data
|
||||
local i rows stream current_size past_size growth growth_gb sort_key show
|
||||
local -a columns
|
||||
|
||||
for i in 0 1 2; do
|
||||
if ! rows=$(so_es_index_size_growth "$app" "${starts[i]}" "${stops[i]}"); then
|
||||
echo "Failed to query telegraf.elasticsearch_index_size in so_telegraf." >&2
|
||||
exit 1
|
||||
fi
|
||||
[[ -z "$rows" ]] && continue
|
||||
window_has_data[$i]=1
|
||||
while IFS='|' read -r stream current_size past_size; do
|
||||
current[$stream]=$current_size
|
||||
past[$i,$stream]=$past_size
|
||||
done <<< "$rows"
|
||||
done
|
||||
|
||||
printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)"
|
||||
printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})"
|
||||
|
||||
for stream in "${!current[@]}"; do
|
||||
columns=()
|
||||
sort_key=0
|
||||
show=false
|
||||
for i in 0 1 2; do
|
||||
if [[ -z "${window_has_data[$i]}" ]]; then
|
||||
columns+=("n/a")
|
||||
continue
|
||||
fi
|
||||
current_size=${current[$stream]}
|
||||
past_size=${past[$i,$stream]-$current_size}
|
||||
growth=$(( current_size > past_size ? current_size - past_size : 0 ))
|
||||
growth_gb=$(bytes_to_gb "$growth")
|
||||
[[ "$growth_gb" != "0.00" ]] && show=true
|
||||
[[ $i -eq 0 ]] && sort_key=$growth
|
||||
columns+=("$growth_gb")
|
||||
done
|
||||
if [[ "$show" == true ]]; then
|
||||
printf "%020.2f|%-50s %15s %15s %15s\n" "$sort_key" "$stream" "${columns[@]}"
|
||||
fi
|
||||
done | sort -t'|' -k1,1nr | cut -d'|' -f2-
|
||||
}
|
||||
|
||||
if ! TG_OUTPUT=$(so_telegraf_output); then
|
||||
echo "Unable to read telegraf.output from pillar." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$TG_OUTPUT" == "POSTGRES" ]]; then
|
||||
so_telegraf_pg_check so-elasticsearch-indices-growth || exit 1
|
||||
postgres_indices_growth
|
||||
else
|
||||
influxdb_indices_growth
|
||||
fi
|
||||
@@ -58,6 +58,9 @@ Usage: $(basename "$0") [OPTIONS]
|
||||
|
||||
Estimate remaining days until Elasticsearch cluster reaches low watermark threshold.
|
||||
|
||||
Index growth history follows telegraf.output: in POSTGRES mode it is read from Postgres, limited to
|
||||
postgres.telegraf.retention_days, and the script must run on the manager; otherwise from InfluxDB.
|
||||
|
||||
OPTIONS:
|
||||
--json Output results in JSON format
|
||||
-v, --verbose Show additional output
|
||||
@@ -168,11 +171,25 @@ run_indices_growth() {
|
||||
fi
|
||||
}
|
||||
|
||||
fetch_postgres_total_bytes() {
|
||||
local total
|
||||
|
||||
total=$(so_es_index_size_growth so-elasticsearch-retention-estimate "$1" "$2") || return 1
|
||||
echo "$total" | awk -F'|' '{ sum += $3 } END { print sum + 0 }'
|
||||
}
|
||||
|
||||
fetch_total_bytes() {
|
||||
local start="$1"
|
||||
local stop="$2"
|
||||
local pg_start="$3"
|
||||
local pg_stop="$4"
|
||||
local range_line
|
||||
|
||||
if [ "$TG_OUTPUT" = "POSTGRES" ]; then
|
||||
fetch_postgres_total_bytes "$pg_start" "$pg_stop"
|
||||
return
|
||||
fi
|
||||
|
||||
if [ -n "$stop" ]; then
|
||||
range_line=" |> range(start: ${start}, stop: ${stop})"
|
||||
else
|
||||
@@ -208,8 +225,13 @@ fail() {
|
||||
|
||||
echo -e "\nDISCLAIMER: Script output is based on current data patterns, but are approximations solely intended to assist with getting a general ILM policy configured."
|
||||
|
||||
ORG_ID=$(lookup_org_id)
|
||||
[ -n "$ORG_ID" ] || fail "Unable to resolve InfluxDB org id"
|
||||
TG_OUTPUT=$(so_telegraf_output) || fail "Unable to read telegraf.output from pillar."
|
||||
if [ "$TG_OUTPUT" = "POSTGRES" ]; then
|
||||
pg_check_error=$(so_telegraf_pg_check so-elasticsearch-retention-estimate 2>&1) || fail "$pg_check_error"
|
||||
else
|
||||
ORG_ID=$(lookup_org_id)
|
||||
[ -n "$ORG_ID" ] || fail "Unable to resolve InfluxDB org id"
|
||||
fi
|
||||
|
||||
cluster_storage_size=0
|
||||
indexed_storage_source="elasticsearch"
|
||||
@@ -221,10 +243,11 @@ if [ -n "$cluster_storage_size_output" ]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# historical data from influxdb for growth calculation
|
||||
one_day_total=$(fetch_total_bytes "-25h" "-23h")
|
||||
seven_day_total=$(fetch_total_bytes "-7d8h" "-7d")
|
||||
thirty_day_total=$(fetch_total_bytes "-30d8h" "-30d")
|
||||
# historical index sizes for growth calculation
|
||||
query_error="Failed to query telegraf.elasticsearch_index_size in so_telegraf."
|
||||
one_day_total=$(fetch_total_bytes "-25h" "-23h" "25 hours" "23 hours") || fail "$query_error"
|
||||
seven_day_total=$(fetch_total_bytes "-7d8h" "-7d" "7 days 8 hours" "7 days") || fail "$query_error"
|
||||
thirty_day_total=$(fetch_total_bytes "-30d8h" "-30d" "30 days 8 hours" "30 days") || fail "$query_error"
|
||||
|
||||
# available historical windows (prefer 30d/7d when available, to avoid using recent 24h traffic spike as true daily ingest rate)
|
||||
history_days=0
|
||||
@@ -244,7 +267,12 @@ elif [ "$one_day_total" -gt 0 ]; then
|
||||
historical_total=$one_day_total
|
||||
fi
|
||||
|
||||
[ "$history_days" -gt 0 ] || fail "Historical InfluxDB data unavailable for growth calculation. If this a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics."
|
||||
if [ "$TG_OUTPUT" = "POSTGRES" ]; then
|
||||
history_unavailable="Historical Postgres data unavailable for growth calculation. Postgres keeps postgres.telegraf.retention_days days of history; if this is a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics."
|
||||
else
|
||||
history_unavailable="Historical InfluxDB data unavailable for growth calculation. If this a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics."
|
||||
fi
|
||||
[ "$history_days" -gt 0 ] || fail "$history_unavailable"
|
||||
|
||||
# Daily growth rate
|
||||
growth_bytes=$(( cluster_storage_size - historical_total ))
|
||||
@@ -729,6 +757,7 @@ if [ "$should_trigger_recommendations" = true ]; then
|
||||
[ -n "$index" ] || continue
|
||||
|
||||
growth_24h_gb=$(echo "$line" | awk '{print $(NF-2)}')
|
||||
[[ "$growth_24h_gb" =~ ^[0-9]+(\.[0-9]+)?$ ]] || growth_24h_gb=0
|
||||
|
||||
creation_date_display=""
|
||||
retention_days=""
|
||||
|
||||
@@ -118,10 +118,6 @@ firewall:
|
||||
tcp:
|
||||
- 5601
|
||||
udp: []
|
||||
localrules:
|
||||
tcp:
|
||||
- 7788
|
||||
udp: []
|
||||
nginx:
|
||||
tcp:
|
||||
- 80
|
||||
@@ -200,7 +196,6 @@ firewall:
|
||||
- postgres
|
||||
- elasticsearch_rest
|
||||
- elasticsearch_node
|
||||
- localrules
|
||||
sensor:
|
||||
portgroups:
|
||||
- beats_5044
|
||||
@@ -391,7 +386,6 @@ firewall:
|
||||
- elastic_agent_control
|
||||
- elastic_agent_data
|
||||
- elastic_agent_update
|
||||
- localrules
|
||||
- sensoroni
|
||||
fleet:
|
||||
portgroups:
|
||||
@@ -611,7 +605,6 @@ firewall:
|
||||
- elastic_agent_control
|
||||
- elastic_agent_data
|
||||
- elastic_agent_update
|
||||
- localrules
|
||||
- sensoroni
|
||||
fleet:
|
||||
portgroups:
|
||||
@@ -829,7 +822,6 @@ firewall:
|
||||
- elastic_agent_control
|
||||
- elastic_agent_data
|
||||
- elastic_agent_update
|
||||
- localrules
|
||||
- sensoroni
|
||||
fleet:
|
||||
portgroups:
|
||||
@@ -1058,7 +1050,6 @@ firewall:
|
||||
- elastic_agent_update
|
||||
- endgame
|
||||
- strelka_frontend
|
||||
- localrules
|
||||
fleet:
|
||||
portgroups:
|
||||
- elasticsearch_rest
|
||||
@@ -1518,7 +1509,6 @@ firewall:
|
||||
- elasticsearch_rest
|
||||
- elasticsearch_node
|
||||
- elastic_agent_control
|
||||
- localrules
|
||||
sensor:
|
||||
portgroups:
|
||||
- beats_5044
|
||||
|
||||
@@ -138,9 +138,6 @@ firewall:
|
||||
kibana:
|
||||
tcp: *tcpsettings
|
||||
udp: *udpsettings
|
||||
localrules:
|
||||
tcp: *tcpsettings
|
||||
udp: *udpsettings
|
||||
nginx:
|
||||
tcp: *tcpsettings
|
||||
udp: *udpsettings
|
||||
|
||||
@@ -20,6 +20,7 @@ def showUsage(args):
|
||||
print(' append - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
|
||||
print(' appendlistobject - Append an object to a yaml list key. Requires KEY and JSON_OBJECT args.', file=sys.stderr)
|
||||
print(' removelistitem - Remove a list item from a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
|
||||
print(' purgelistitem - Remove every occurrence of a list item from all lists nested under a yaml key, if it exists. Requires KEY and LISTITEM args.', file=sys.stderr)
|
||||
print(' replacelistobject - Replace a list object based on a condition. Requires KEY, CONDITION_FIELD, CONDITION_VALUE, and JSON_OBJECT args.', file=sys.stderr)
|
||||
print(' add - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr)
|
||||
print(' get [-r] - Displays (to stdout) the value stored in the given key. Requires KEY arg. Use -r for raw output without YAML formatting.', file=sys.stderr)
|
||||
@@ -199,6 +200,33 @@ def removelistitem(args):
|
||||
return 0
|
||||
|
||||
|
||||
def purgeListItem(content, listItem):
|
||||
if isinstance(content, dict):
|
||||
for value in content.values():
|
||||
purgeListItem(value, listItem)
|
||||
elif isinstance(content, list):
|
||||
content[:] = [item for item in content if item != listItem]
|
||||
for item in content:
|
||||
purgeListItem(item, listItem)
|
||||
|
||||
|
||||
def purgelistitem(args):
|
||||
if len(args) != 3:
|
||||
print('Missing filename, key arg, or list item to purge', file=sys.stderr)
|
||||
showUsage(None)
|
||||
return 1
|
||||
|
||||
filename = args[0]
|
||||
key = args[1]
|
||||
listItem = args[2]
|
||||
|
||||
content = loadYaml(filename)
|
||||
purgeListItem(getKeyValue(content, key), convertType(listItem))
|
||||
writeYaml(filename, content)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def replaceListObjectByCondition(content, key, conditionField, conditionValue, newObject):
|
||||
pieces = key.split(".", 1)
|
||||
if len(pieces) > 1:
|
||||
@@ -291,6 +319,8 @@ def add(args):
|
||||
|
||||
|
||||
def removeKey(content, key):
|
||||
if not isinstance(content, dict):
|
||||
return
|
||||
pieces = key.split(".", 1)
|
||||
if len(pieces) > 1:
|
||||
if pieces[0] in content:
|
||||
@@ -334,6 +364,8 @@ def replace(args):
|
||||
|
||||
|
||||
def getKeyValue(content, key):
|
||||
if not isinstance(content, dict):
|
||||
return None
|
||||
pieces = key.split(".", 1)
|
||||
if len(pieces) > 1 and pieces[0] in content:
|
||||
return getKeyValue(content[pieces[0]], pieces[1])
|
||||
@@ -385,6 +417,7 @@ def main():
|
||||
"append": append,
|
||||
"appendlistobject": appendlistobject,
|
||||
"removelistitem": removelistitem,
|
||||
"purgelistitem": purgelistitem,
|
||||
"replacelistobject": replacelistobject,
|
||||
"get": get,
|
||||
"remove": remove,
|
||||
|
||||
@@ -128,6 +128,22 @@ class TestRemove(unittest.TestCase):
|
||||
sysmock.assert_called_once_with(1)
|
||||
self.assertIn("Missing filename or key arg\n", mock_stderr.getvalue())
|
||||
|
||||
def test_remove_null_parent(self):
|
||||
filename = "/tmp/so-yaml_test-remove.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("key1: null\n")
|
||||
file.close()
|
||||
|
||||
result = soyaml.remove([filename, "key1.child1.deep1"])
|
||||
self.assertEqual(result, 0)
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1: null\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_append_missing_arg(self):
|
||||
with patch('sys.exit', new=MagicMock()) as sysmock:
|
||||
with patch('sys.stderr', new=StringIO()) as mock_stderr:
|
||||
@@ -565,6 +581,19 @@ class TestRemove(unittest.TestCase):
|
||||
self.assertEqual(result, 2)
|
||||
self.assertEqual("", mock_stdout.getvalue())
|
||||
|
||||
def test_get_null_parent(self):
|
||||
with patch('sys.stdout', new=StringIO()) as mock_stdout:
|
||||
with patch('sys.stderr', new=StringIO()) as mock_stderr:
|
||||
filename = "/tmp/so-yaml_test-get.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("key1: null\n")
|
||||
file.close()
|
||||
|
||||
result = soyaml.get([filename, "key1.child1"])
|
||||
self.assertEqual(result, 2)
|
||||
self.assertEqual("", mock_stdout.getvalue())
|
||||
self.assertIn("Key 'key1.child1' not found by so-yaml.py", mock_stderr.getvalue())
|
||||
|
||||
def test_get_empty_file(self):
|
||||
with patch('sys.stdout', new=StringIO()) as mock_stdout:
|
||||
with patch('sys.stderr', new=StringIO()) as mock_stderr:
|
||||
@@ -709,6 +738,255 @@ class TestRemoveListItem(unittest.TestCase):
|
||||
self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())
|
||||
|
||||
|
||||
class TestPurgeListItem(unittest.TestCase):
|
||||
|
||||
def test_purgelistitem_missing_arg(self):
|
||||
with patch('sys.exit', new=MagicMock()) as sysmock:
|
||||
with patch('sys.stderr', new=StringIO()) as mock_stderr:
|
||||
sys.argv = ["cmd", "help"]
|
||||
soyaml.purgelistitem(["file", "key"])
|
||||
sysmock.assert_called()
|
||||
self.assertIn("Missing filename, key arg, or list item to purge", mock_stderr.getvalue())
|
||||
|
||||
def test_purgelistitem(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: [a,b,c], key2: false}")
|
||||
file.close()
|
||||
|
||||
soyaml.purgelistitem([filename, "key1", "b"])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n- a\n- c\nkey2: false\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_nested_lists(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: { child1: [a,b,c], child2: { deep1: [b,d], deep2: 45 } }, key2: [b,e]}")
|
||||
file.close()
|
||||
|
||||
soyaml.purgelistitem([filename, "key1", "b"])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n child1:\n - a\n - c\n child2:\n deep1:\n - d\n deep2: 45\nkey2:\n- b\n- e\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_duplicates(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: [b,a,b,c,b]}")
|
||||
file.close()
|
||||
|
||||
soyaml.purgelistitem([filename, "key1", "b"])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n- a\n- c\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_list_of_objects(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: [{name: item1, values: [a,b]}, b, {name: item2, values: [b,c]}]}")
|
||||
file.close()
|
||||
|
||||
soyaml.purgelistitem([filename, "key1", "b"])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n- name: item1\n values:\n - a\n- name: item2\n values:\n - c\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_typed(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: [1, '1', 2]}")
|
||||
file.close()
|
||||
|
||||
soyaml.purgelistitem([filename, "key1", "1"])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n- '1'\n- 2\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_item_not_in_list(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: { child1: [a,b,c] }}")
|
||||
file.close()
|
||||
|
||||
soyaml.purgelistitem([filename, "key1", "d"])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n child1:\n - a\n - b\n - c\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_key_noexist(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: [a,b,c]}")
|
||||
file.close()
|
||||
|
||||
result = soyaml.purgelistitem([filename, "key2.child1", "b"])
|
||||
self.assertEqual(result, 0)
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n- a\n- b\n- c\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_key_noexist_deep(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: { child1: [a,b,c] }}")
|
||||
file.close()
|
||||
|
||||
result = soyaml.purgelistitem([filename, "key1.child2.deep1", "b"])
|
||||
self.assertEqual(result, 0)
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n child1:\n - a\n - b\n - c\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_scalar_and_null_values(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: { child1: b, child2: null, child3: [b] }}")
|
||||
file.close()
|
||||
|
||||
soyaml.purgelistitem([filename, "key1", "b"])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n child1: b\n child2: null\n child3: []\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_lists_of_lists(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: [[a,b], [b,[b,c]], b]}")
|
||||
file.close()
|
||||
|
||||
soyaml.purgelistitem([filename, "key1", "b"])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n- - a\n- - - c\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_file_object(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: [{name: x}, a, {name: y}, {name: x}]}")
|
||||
file.close()
|
||||
|
||||
itemfile = "/tmp/so-yaml_test-purgelistitem-item.yaml"
|
||||
file = open(itemfile, "w")
|
||||
file.write("name: x\n")
|
||||
file.close()
|
||||
|
||||
soyaml.purgelistitem([filename, "key1", "file:" + itemfile])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n- a\n- name: y\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_main(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: { child1: [a,b,c] }, key2: [b]}")
|
||||
file.close()
|
||||
|
||||
with patch('sys.exit', new=MagicMock()) as sysmock:
|
||||
sys.argv = ["cmd", "purgelistitem", filename, "key1", "b"]
|
||||
soyaml.main()
|
||||
sysmock.assert_called_once_with(0)
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n child1:\n - a\n - c\nkey2:\n- b\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_with_remove(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: { child1: { deep1: [a] }, child2: [b] }, key2: { group1: { items: [x,child1,y] }, group2: { items: [child1] } }}")
|
||||
file.close()
|
||||
|
||||
soyaml.remove([filename, "key1.child1"])
|
||||
soyaml.purgelistitem([filename, "key2", "child1"])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n child2:\n - b\nkey2:\n group1:\n items:\n - x\n - y\n group2:\n items: []\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_null_parent(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("key1: null\n")
|
||||
file.close()
|
||||
|
||||
result = soyaml.purgelistitem([filename, "key1.child1", "b"])
|
||||
self.assertEqual(result, 0)
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1: null\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_non_dict_parent(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("key1:\n- a\n- b\n")
|
||||
file.close()
|
||||
|
||||
result = soyaml.purgelistitem([filename, "key1.child1", "b"])
|
||||
self.assertEqual(result, 0)
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n- a\n- b\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
|
||||
class TestAppendListObject(unittest.TestCase):
|
||||
|
||||
def test_appendlistobject_missing_arg(self):
|
||||
|
||||
@@ -5,7 +5,6 @@
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
|
||||
. /usr/sbin/so-common
|
||||
. /usr/sbin/so-image-common
|
||||
|
||||
@@ -17,7 +16,9 @@ INSTALLEDVERSION=$(cat /etc/soversion)
|
||||
# not finish its post-upgrade steps; its contents are the resume point. It is read
|
||||
# here before preupgrade_changes mutates INSTALLEDVERSION and before any highstate
|
||||
# stamps /etc/soversion from the pillar.
|
||||
|
||||
POSTVERSION_FILE=/etc/sopostversion
|
||||
|
||||
if [ -f "$POSTVERSION_FILE" ]; then
|
||||
POSTVERSION=$(cat "$POSTVERSION_FILE")
|
||||
else
|
||||
@@ -527,6 +528,7 @@ preupgrade_changes() {
|
||||
[[ "$INSTALLEDVERSION" == "3.1.0" ]] && up_to_3.2.0
|
||||
[[ "$INSTALLEDVERSION" == "3.2.0" ]] && up_to_3.3.0
|
||||
[[ "$INSTALLEDVERSION" == "3.3.0" ]] && up_to_3.4.0
|
||||
[[ "$INSTALLEDVERSION" == "3.4.0" ]] && up_to_3.5.0
|
||||
true
|
||||
}
|
||||
|
||||
@@ -546,6 +548,7 @@ postupgrade_changes() {
|
||||
[[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0
|
||||
[[ "$POSTVERSION" == "3.2.0" ]] && post_to_3.3.0
|
||||
[[ "$POSTVERSION" == "3.3.0" ]] && post_to_3.4.0
|
||||
[[ "$POSTVERSION" == "3.4.0" ]] && post_to_3.5.0
|
||||
# All applicable post-upgrade steps completed; clear the resume marker.
|
||||
rm -f "$POSTVERSION_FILE"
|
||||
true
|
||||
@@ -1065,17 +1068,31 @@ telegraf_repair() {
|
||||
# when there is something to repair, so everyone else is left alone.
|
||||
local repair=/usr/sbin/so-telegraf-repair
|
||||
[[ -x "$repair" ]] || return 0
|
||||
docker ps --format '{{.Names}}' | grep -qx so-postgres || return 0
|
||||
local state
|
||||
state=$(docker inspect -f '{{.State.Running}} {{.State.Restarting}}' so-postgres 2>/dev/null) || return 0
|
||||
if [[ "$state" != "true false" ]]; then
|
||||
echo " so-postgres is not running; skipping the Telegraf partition check."
|
||||
FINAL_MESSAGE_QUEUE+=("WARNING: so-postgres is not running, so Telegraf partitions were not checked. If /nsm is full, follow \"Telegraf Partition Repair\" at:")
|
||||
FINAL_MESSAGE_QUEUE+=(" - https://securityonion.net/docs/postgresql")
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "Checking Telegraf metric partitions."
|
||||
local status=0
|
||||
"$repair" --check >> "$SOUP_LOG" 2>&1 || status=$?
|
||||
local out status=0
|
||||
out=$("$repair" --check 2>&1) || status=$?
|
||||
echo "$out" >> "$SOUP_LOG"
|
||||
case "$status" in
|
||||
0) echo " Telegraf partitions are healthy; nothing to repair." ;;
|
||||
1) echo " Repairing stalled Telegraf partitions."
|
||||
"$repair" --yes \
|
||||
|| echo " warning: so-telegraf-repair failed; run it manually" >&2 ;;
|
||||
*) echo " Skipping; Telegraf is not storing metrics in Postgres on this host." ;;
|
||||
if ! "$repair" --yes; then
|
||||
echo " warning: so-telegraf-repair failed; run it manually" >&2
|
||||
FINAL_MESSAGE_QUEUE+=("WARNING: Telegraf partition repair failed; Postgres metrics will keep filling /nsm until it is repaired:")
|
||||
FINAL_MESSAGE_QUEUE+=(" - sudo so-telegraf-repair")
|
||||
fi ;;
|
||||
2) echo " Skipping: $(tail -n 1 <<<"$out")" ;;
|
||||
*) echo " warning: could not check Telegraf partitions" >&2
|
||||
FINAL_MESSAGE_QUEUE+=("WARNING: Telegraf partitions could not be checked during the upgrade; run:")
|
||||
FINAL_MESSAGE_QUEUE+=(" - sudo so-telegraf-repair --check") ;;
|
||||
esac
|
||||
}
|
||||
|
||||
@@ -1202,6 +1219,32 @@ add_detection_runtime_fields() {
|
||||
}
|
||||
### 3.4.0 End ###
|
||||
|
||||
### 3.5.0 Scripts ###
|
||||
up_to_3.5.0() {
|
||||
remove_localrules_portgroup
|
||||
|
||||
INSTALLEDVERSION=3.5.0
|
||||
}
|
||||
|
||||
remove_localrules_portgroup() {
|
||||
local pillar_file
|
||||
|
||||
for pillar_file in /opt/so/saltstack/local/pillar/firewall/{soc,adv}_firewall.sls; do
|
||||
grep -qs localrules "$pillar_file" || continue
|
||||
so-yaml.py get "$pillar_file" firewall 2>/dev/null | grep -q '\<localrules\>' || continue
|
||||
echo "Removing the localrules portgroup from $pillar_file."
|
||||
so-yaml.py remove "$pillar_file" firewall.portgroups.localrules
|
||||
so-yaml.py purgelistitem "$pillar_file" firewall.role localrules
|
||||
done
|
||||
}
|
||||
|
||||
post_to_3.5.0() {
|
||||
telegraf_repair
|
||||
|
||||
set_postversion 3.5.0
|
||||
}
|
||||
### 3.5.0 End ###
|
||||
|
||||
|
||||
repo_sync() {
|
||||
echo "Sync the local repo."
|
||||
|
||||
+13
-17
@@ -10,29 +10,25 @@
|
||||
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
|
||||
{% from 'nginx/map.jinja' import NGINXMERGED %}
|
||||
|
||||
{% if GLOBALS.role != 'so-fleet' %}
|
||||
{% set container_config = 'so-nginx' %}
|
||||
{% else %}
|
||||
{# if this is an so-fleet node then we want to use the port bindings, custom bind mounts defined for fleet #}
|
||||
{% set container_config = 'so-nginx-fleet-node' %}
|
||||
{% endif %}
|
||||
|
||||
include:
|
||||
- ca.trust
|
||||
- nginx.ssl
|
||||
- nginx.config
|
||||
- nginx.sostatus
|
||||
|
||||
{% if GLOBALS.role != 'so-fleet' %}
|
||||
{% set container_config = 'so-nginx' %}
|
||||
make-rule-dir-nginx:
|
||||
fleet_beats_dir_nginx:
|
||||
file.directory:
|
||||
- name: /nsm/rules
|
||||
- user: socore
|
||||
- group: socore
|
||||
- recurse:
|
||||
- user
|
||||
- group
|
||||
- name: /nsm/elastic-fleet/artifacts/beats
|
||||
- makedirs: True
|
||||
- show_changes: False
|
||||
|
||||
{% else %}
|
||||
{# if this is an so-fleet node then we want to use the port bindings, custom bind mounts defined for fleet #}
|
||||
{% set container_config = 'so-nginx-fleet-node' %}
|
||||
{% endif %}
|
||||
|
||||
so-nginx:
|
||||
docker_container.running:
|
||||
- image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-nginx:{{ GLOBALS.so_version }}
|
||||
@@ -54,9 +50,9 @@ so-nginx:
|
||||
- /opt/so/log/nginx/:/var/log/nginx:rw
|
||||
- /opt/so/tmp/nginx/:/var/lib/nginx:rw
|
||||
- /opt/so/tmp/nginx/:/run:rw
|
||||
- /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages
|
||||
- /nsm/elastic-fleet/artifacts/:/opt/socore/html/artifacts
|
||||
- /nsm/elastic-fleet/artifacts/beats/:/opt/socore/elastic-artifacts/artifacts/beats:ro
|
||||
{% if GLOBALS.is_manager %}
|
||||
- /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages
|
||||
- /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro
|
||||
- /etc/pki/managerssl.key:/etc/pki/nginx/server.key:ro
|
||||
# ATT&CK Navigator binds
|
||||
@@ -64,7 +60,6 @@ so-nginx:
|
||||
- /opt/so/conf/navigator/config.json:/opt/socore/html/navigator/assets/config.json:ro
|
||||
- /nsm/repo:/opt/socore/html/repo:ro
|
||||
- /nsm/kernelrepo:/opt/socore/html/kernelrepo:ro
|
||||
- /nsm/rules:/nsm/rules:ro
|
||||
{% if NGINXMERGED.external_suricata %}
|
||||
- /opt/so/rules/nids/suri:/surirules:ro
|
||||
{% endif %}
|
||||
@@ -106,6 +101,7 @@ so-nginx:
|
||||
{% endif %}
|
||||
- require:
|
||||
- file: nginxconf
|
||||
- file: fleet_beats_dir_nginx
|
||||
{% if GLOBALS.is_manager %}
|
||||
{% if NGINXMERGED.ssl.replace_cert %}
|
||||
- file: managerssl_key
|
||||
|
||||
+16
-50
@@ -4,6 +4,8 @@
|
||||
{%- set role = grains.id.split('_') | last %}
|
||||
{%- set influxpass = salt['pillar.get']('secrets:influx_pass') %}
|
||||
{%- set influxauth = ('so:' + influxpass) | base64_encode %}
|
||||
{%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %}
|
||||
{%- set full_server_name = all_names | unique | join(' ') %}
|
||||
|
||||
worker_processes auto;
|
||||
error_log /var/log/nginx/error.log;
|
||||
@@ -39,52 +41,31 @@ http {
|
||||
|
||||
include /etc/nginx/conf.d/*.conf;
|
||||
|
||||
{%- if role in ['fleet'] %}
|
||||
|
||||
server {
|
||||
listen 8443;
|
||||
server_name {{ GLOBALS.hostname }};
|
||||
root /opt/socore/html;
|
||||
location /artifacts/ {
|
||||
try_files $uri =206;
|
||||
proxy_read_timeout 90;
|
||||
proxy_connect_timeout 90;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header Proxy "";
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
}
|
||||
|
||||
{%- endif %}
|
||||
|
||||
{%- if GLOBALS.is_manager %}
|
||||
{%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %}
|
||||
{%- set full_server_name = all_names | unique | join(' ') %}
|
||||
|
||||
server {
|
||||
listen 80 default_server;
|
||||
server_name _;
|
||||
return 307 https://{{ GLOBALS.url_base }}$request_uri;
|
||||
}
|
||||
|
||||
{%- endif %}
|
||||
{%- if GLOBALS.is_manager or role == 'fleet' %}
|
||||
server {
|
||||
listen 8443;
|
||||
{%- if GLOBALS.is_manager %}
|
||||
server_name {{ full_server_name }};
|
||||
root /opt/socore/html;
|
||||
location /artifacts/ {
|
||||
try_files $uri =206;
|
||||
proxy_read_timeout 90;
|
||||
proxy_connect_timeout 90;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header Proxy "";
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
{%- else %}
|
||||
server_name {{ GLOBALS.hostname }};
|
||||
{%- endif %}
|
||||
root /opt/socore/elastic-artifacts;
|
||||
location /artifacts/beats/ {
|
||||
try_files $uri =404;
|
||||
}
|
||||
location / {
|
||||
return 404;
|
||||
}
|
||||
}
|
||||
|
||||
{%- endif %}
|
||||
{%- if GLOBALS.is_manager %}
|
||||
server {
|
||||
listen 443 ssl default_server;
|
||||
http2 on;
|
||||
@@ -111,21 +92,6 @@ http {
|
||||
{%- endif %}
|
||||
|
||||
{%- if GLOBALS.is_manager %}
|
||||
|
||||
server {
|
||||
listen 7788;
|
||||
server_name {{ full_server_name }};
|
||||
root /nsm/rules;
|
||||
location / {
|
||||
allow all;
|
||||
sendfile on;
|
||||
sendfile_max_chunk 1m;
|
||||
autoindex on;
|
||||
autoindex_exact_size off;
|
||||
autoindex_format html;
|
||||
autoindex_localtime on;
|
||||
}
|
||||
}
|
||||
{%- if NGINXMERGED.external_suricata %}
|
||||
server {
|
||||
listen 7789 ssl;
|
||||
|
||||
@@ -17,3 +17,4 @@ postgres:
|
||||
log_min_messages: 'warning'
|
||||
shared_preload_libraries: pg_cron
|
||||
cron.database_name: postgres
|
||||
cron.host: /run/postgresql
|
||||
@@ -87,3 +87,9 @@ postgres:
|
||||
global: True
|
||||
advanced: True
|
||||
helpLink: postgres
|
||||
cron.host:
|
||||
description: Where pg_cron connects to run scheduled jobs such as Telegraf partition maintenance. Must be the container's PostgreSQL socket directory, /run/postgresql, because pg_cron connects without a password and only local socket connections are trusted. Any other value makes every scheduled job fail. Changing it restarts PostgreSQL.
|
||||
readonly: True
|
||||
global: True
|
||||
advanced: True
|
||||
helpLink: postgres
|
||||
@@ -52,7 +52,7 @@ if [ -n "$FILTER_HOST" ] && ! [[ "$FILTER_HOST" =~ $HOST_RE ]]; then
|
||||
fi
|
||||
|
||||
so_psql() {
|
||||
docker exec so-postgres psql -U postgres -d so_telegraf -At -F $'\t' "$@"
|
||||
docker exec -e PGOPTIONS='-c application_name=so-stats-show' so-postgres psql -U so_maint_telegraf -d so_telegraf -At -F $'\t' "$@"
|
||||
}
|
||||
|
||||
if ! docker exec so-postgres psql -U postgres -lqt 2>/dev/null | cut -d\| -f1 | grep -qw so_telegraf; then
|
||||
@@ -60,6 +60,15 @@ if ! docker exec so-postgres psql -U postgres -lqt 2>/dev/null | cut -d\| -f1 |
|
||||
exit 2
|
||||
fi
|
||||
|
||||
role_ok=$(docker exec so-postgres psql -U postgres -tAc "SELECT rolcanlogin AND NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls) FROM pg_roles WHERE rolname = 'so_maint_telegraf'" 2>/dev/null)
|
||||
if [ -z "$role_ok" ]; then
|
||||
echo "The so_maint_telegraf role is missing; run a highstate."
|
||||
exit 1
|
||||
elif [ "$role_ok" != t ]; then
|
||||
echo "The so_maint_telegraf role is misconfigured; run a highstate."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
table_exists() {
|
||||
local table="$1"
|
||||
[ -n "$(so_psql -c "SELECT 1 FROM information_schema.tables WHERE table_schema='${SCHEMA}' AND table_name='${table}' LIMIT 1;")" ]
|
||||
|
||||
@@ -4,15 +4,18 @@ set -e
|
||||
# Provision Telegraf state inside the so-postgres container.
|
||||
# Usage: so-telegraf-postgres <subcommand>
|
||||
# create_db Ensure the so_telegraf database exists.
|
||||
# group_role Provision the so_telegraf group role, telegraf/partman schemas,
|
||||
# pg_partman, the so_admin maintenance routines, and the hourly
|
||||
# pg_cron maintenance job.
|
||||
# group_role Provision the so_telegraf group role, the so_maint_telegraf
|
||||
# maintenance login role, telegraf/partman schemas, pg_partman,
|
||||
# the so_admin maintenance routines, and the hourly pg_cron
|
||||
# maintenance job.
|
||||
# user Create or update a per-minion login role granted to so_telegraf.
|
||||
# Env: ROLE_USER, ROLE_PASS.
|
||||
# retention Reconcile partman retention and premake on telegraf parents.
|
||||
# Env: RETENTION_DAYS.
|
||||
# maintenance Drain default partitions and run partman maintenance.
|
||||
# check Report partition health. Non-zero if any parent is unhealthy.
|
||||
# maintenance Drain default partitions and run partman maintenance, as
|
||||
# so_maint_telegraf.
|
||||
# check Report partition health, as so_maint_telegraf. Non-zero if any
|
||||
# parent is unhealthy.
|
||||
#
|
||||
# A default partition holding rows for a day blocks creating that day's child,
|
||||
# so maintenance drains defaults before calling partman. Use so-telegraf-repair
|
||||
@@ -20,6 +23,10 @@ set -e
|
||||
|
||||
cmd="${1:?subcommand required}"
|
||||
|
||||
# Long enough for an hourly maintenance run to release its locks, short enough
|
||||
# that soup and admins are not left hanging.
|
||||
LOCK_TIMEOUT=60s
|
||||
|
||||
case "$cmd" in
|
||||
create_db)
|
||||
if ! docker exec so-postgres psql -U postgres -tAc \
|
||||
@@ -30,7 +37,9 @@ case "$cmd" in
|
||||
;;
|
||||
|
||||
group_role)
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -v lock_timeout="$LOCK_TIMEOUT" \
|
||||
-U postgres -d so_telegraf <<'EOSQL'
|
||||
SET lock_timeout = :'lock_timeout';
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'so_telegraf') THEN
|
||||
@@ -38,6 +47,17 @@ BEGIN
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'so_maint_telegraf') THEN
|
||||
CREATE ROLE so_maint_telegraf;
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
ALTER ROLE so_maint_telegraf LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD NULL;
|
||||
GRANT so_telegraf TO so_maint_telegraf;
|
||||
ALTER ROLE so_maint_telegraf SET role = so_telegraf;
|
||||
ALTER ROLE so_maint_telegraf SET search_path = pg_catalog;
|
||||
GRANT CONNECT ON DATABASE so_telegraf TO so_telegraf;
|
||||
CREATE SCHEMA IF NOT EXISTS telegraf AUTHORIZATION so_telegraf;
|
||||
GRANT USAGE, CREATE ON SCHEMA telegraf TO so_telegraf;
|
||||
@@ -58,52 +78,90 @@ ALTER DEFAULT PRIVILEGES IN SCHEMA partman
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA partman
|
||||
GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO so_telegraf;
|
||||
|
||||
-- pg_cron runs these as postgres, so they must not sit in a schema any
|
||||
-- Telegraf role can create objects in.
|
||||
-- Owned by postgres so no Telegraf role can create objects here; the routines
|
||||
-- run with the caller's privileges.
|
||||
CREATE SCHEMA IF NOT EXISTS so_admin AUTHORIZATION postgres;
|
||||
REVOKE ALL ON SCHEMA so_admin FROM PUBLIC;
|
||||
|
||||
-- Resolved through pg_inherits, never by name, so a view or a table that only
|
||||
-- carries the _default name is never read or written.
|
||||
CREATE OR REPLACE FUNCTION so_admin.telegraf_defaults()
|
||||
RETURNS TABLE (
|
||||
parent_table regclass,
|
||||
default_part regclass,
|
||||
has_rows boolean,
|
||||
default_size bigint
|
||||
)
|
||||
LANGUAGE plpgsql
|
||||
STABLE
|
||||
SECURITY INVOKER
|
||||
AS $func$
|
||||
DECLARE
|
||||
r record;
|
||||
BEGIN
|
||||
FOR r IN
|
||||
SELECT c.oid::regclass AS pt
|
||||
FROM partman.part_config pc
|
||||
JOIN pg_catalog.pg_class c ON c.oid = to_regclass(pc.parent_table)
|
||||
WHERE pc.parent_table LIKE 'telegraf.%'
|
||||
AND c.relkind = 'p'
|
||||
ORDER BY pc.parent_table
|
||||
LOOP
|
||||
parent_table := r.pt;
|
||||
|
||||
SELECT c.oid::regclass INTO default_part
|
||||
FROM pg_catalog.pg_inherits i
|
||||
JOIN pg_catalog.pg_class c ON c.oid = i.inhrelid
|
||||
WHERE i.inhparent = r.pt
|
||||
AND c.relkind = 'r' AND c.relispartition
|
||||
AND pg_catalog.pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT';
|
||||
|
||||
IF default_part IS NULL THEN
|
||||
has_rows := false;
|
||||
default_size := NULL;
|
||||
ELSE
|
||||
EXECUTE format('SELECT EXISTS (SELECT 1 FROM %s LIMIT 1)', default_part)
|
||||
INTO has_rows;
|
||||
default_size := pg_catalog.pg_total_relation_size(default_part);
|
||||
END IF;
|
||||
|
||||
RETURN NEXT;
|
||||
END LOOP;
|
||||
END;
|
||||
$func$;
|
||||
|
||||
CREATE OR REPLACE PROCEDURE so_admin.telegraf_maintenance()
|
||||
LANGUAGE plpgsql
|
||||
SECURITY INVOKER
|
||||
-- No SET clause: it would forbid the COMMITs below, so so_maint_telegraf pins search_path instead.
|
||||
AS $proc$
|
||||
DECLARE
|
||||
r record;
|
||||
v_default text;
|
||||
v_rows bigint;
|
||||
r record;
|
||||
BEGIN
|
||||
-- No per-parent EXCEPTION handler: partition_data_proc commits internally,
|
||||
-- and COMMIT is illegal while a subtransaction is active. A failing parent
|
||||
-- aborts the run and the next pass retries.
|
||||
FOR r IN
|
||||
SELECT parent_table, retention
|
||||
FROM partman.part_config
|
||||
WHERE parent_table LIKE 'telegraf.%'
|
||||
ORDER BY parent_table
|
||||
SELECT pc.parent_table, pc.retention, d.default_part, d.default_size
|
||||
FROM so_admin.telegraf_defaults() d
|
||||
JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table
|
||||
WHERE d.default_part IS NOT NULL AND d.has_rows
|
||||
ORDER BY pc.parent_table
|
||||
LOOP
|
||||
v_default := format('%I.%I',
|
||||
split_part(r.parent_table, '.', 1),
|
||||
split_part(r.parent_table, '.', 2) || '_default');
|
||||
|
||||
CONTINUE WHEN to_regclass(v_default) IS NULL;
|
||||
|
||||
EXECUTE format('SELECT count(*) FROM %s', v_default) INTO v_rows;
|
||||
CONTINUE WHEN v_rows = 0;
|
||||
|
||||
RAISE WARNING 'so_admin.telegraf_maintenance: % rows stranded in %, draining',
|
||||
v_rows, v_default;
|
||||
RAISE WARNING 'so_admin.telegraf_maintenance: % holds rows (%), draining',
|
||||
r.default_part, pg_size_pretty(r.default_size);
|
||||
|
||||
-- Cheaper to delete expired rows than to repartition and then drop them.
|
||||
IF r.retention IS NOT NULL THEN
|
||||
EXECUTE format('DELETE FROM %s WHERE "time" < now() - %L::interval',
|
||||
v_default, r.retention);
|
||||
r.default_part, r.retention);
|
||||
COMMIT;
|
||||
END IF;
|
||||
|
||||
-- Bounded so a large backlog drains across several runs.
|
||||
CALL partman.partition_data_proc(
|
||||
p_parent_table := r.parent_table,
|
||||
p_loop_count := 200,
|
||||
p_source_table := v_default
|
||||
p_loop_count := 200
|
||||
);
|
||||
COMMIT;
|
||||
END LOOP;
|
||||
@@ -112,58 +170,75 @@ BEGIN
|
||||
END;
|
||||
$proc$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION so_admin.telegraf_partition_status()
|
||||
DROP FUNCTION IF EXISTS so_admin.telegraf_partition_status();
|
||||
CREATE FUNCTION so_admin.telegraf_partition_status()
|
||||
RETURNS TABLE (
|
||||
parent_table text,
|
||||
oldest_child date,
|
||||
newest_child date,
|
||||
days_ahead int,
|
||||
retention text,
|
||||
default_rows bigint,
|
||||
default_size text
|
||||
parent_table text,
|
||||
oldest_child date,
|
||||
newest_child date,
|
||||
days_ahead int,
|
||||
retention text,
|
||||
default_has_rows boolean,
|
||||
default_size text
|
||||
)
|
||||
LANGUAGE plpgsql
|
||||
STABLE
|
||||
AS $func$
|
||||
DECLARE
|
||||
r record;
|
||||
v_default regclass;
|
||||
r record;
|
||||
BEGIN
|
||||
FOR r IN
|
||||
SELECT pc.parent_table AS pt, pc.retention AS ret
|
||||
FROM partman.part_config pc
|
||||
WHERE pc.parent_table LIKE 'telegraf.%'
|
||||
ORDER BY pc.parent_table
|
||||
SELECT d.parent_table AS pt, d.has_rows, d.default_size AS size, pc.retention AS ret
|
||||
FROM so_admin.telegraf_defaults() d
|
||||
JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table
|
||||
ORDER BY d.parent_table::text
|
||||
LOOP
|
||||
parent_table := r.pt;
|
||||
parent_table := r.pt::text;
|
||||
retention := r.ret;
|
||||
|
||||
SELECT min(d), max(d) INTO oldest_child, newest_child
|
||||
SELECT min(x), max(x) INTO oldest_child, newest_child
|
||||
FROM (
|
||||
SELECT to_date(substring(c.relname FROM '_p(\d{8})$'), 'YYYYMMDD') AS d
|
||||
FROM pg_inherits i
|
||||
JOIN pg_class c ON c.oid = i.inhrelid
|
||||
WHERE i.inhparent = r.pt::regclass
|
||||
AND pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT'
|
||||
SELECT to_date(substring(c.relname FROM '_p(\d{8})$'), 'YYYYMMDD') AS x
|
||||
FROM pg_catalog.pg_inherits i
|
||||
JOIN pg_catalog.pg_class c ON c.oid = i.inhrelid
|
||||
WHERE i.inhparent = r.pt
|
||||
AND pg_catalog.pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT'
|
||||
) s;
|
||||
|
||||
days_ahead := newest_child - current_date;
|
||||
|
||||
v_default := to_regclass(format('%I.%I',
|
||||
split_part(r.pt, '.', 1),
|
||||
split_part(r.pt, '.', 2) || '_default'));
|
||||
IF v_default IS NULL THEN
|
||||
default_rows := NULL;
|
||||
default_size := NULL;
|
||||
ELSE
|
||||
EXECUTE format('SELECT count(*) FROM %s', v_default::text) INTO default_rows;
|
||||
default_size := pg_size_pretty(pg_total_relation_size(v_default));
|
||||
END IF;
|
||||
days_ahead := newest_child - current_date;
|
||||
default_has_rows := r.has_rows;
|
||||
default_size := pg_size_pretty(r.size);
|
||||
|
||||
RETURN NEXT;
|
||||
END LOOP;
|
||||
END;
|
||||
$func$;
|
||||
|
||||
GRANT USAGE ON SCHEMA so_admin TO so_telegraf;
|
||||
GRANT EXECUTE ON FUNCTION so_admin.telegraf_defaults() TO so_telegraf;
|
||||
GRANT EXECUTE ON PROCEDURE so_admin.telegraf_maintenance() TO so_telegraf;
|
||||
GRANT EXECUTE ON FUNCTION so_admin.telegraf_partition_status() TO so_telegraf;
|
||||
|
||||
-- Partitions created by a superuser session (an older repair) belong to
|
||||
-- postgres; partman acting as so_telegraf could not drop them.
|
||||
DO $$
|
||||
DECLARE
|
||||
r record;
|
||||
BEGIN
|
||||
FOR r IN
|
||||
SELECT n.nspname, c.relname
|
||||
FROM pg_catalog.pg_class c
|
||||
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE c.relkind IN ('r', 'p')
|
||||
AND (n.nspname = 'telegraf'
|
||||
OR (n.nspname = 'partman' AND c.relname LIKE 'template\_telegraf\_%'))
|
||||
AND c.relowner <> 'so_telegraf'::regrole
|
||||
LOOP
|
||||
EXECUTE format('ALTER TABLE %I.%I OWNER TO so_telegraf', r.nspname, r.relname);
|
||||
END LOOP;
|
||||
END
|
||||
$$;
|
||||
|
||||
-- Drop the registration older releases left in so_telegraf.
|
||||
SELECT CASE
|
||||
WHEN current_setting('cron.database_name', true) IS DISTINCT FROM current_database()
|
||||
@@ -175,20 +250,69 @@ DROP EXTENSION pg_cron CASCADE;
|
||||
\endif
|
||||
EOSQL
|
||||
|
||||
stalled=false
|
||||
probe_failed=false
|
||||
if out=$(docker exec -e PGOPTIONS="-c lock_timeout=$LOCK_TIMEOUT" so-postgres \
|
||||
psql -U so_maint_telegraf -d so_telegraf -tAc \
|
||||
"SELECT coalesce(bool_or(has_rows), false) FROM so_admin.telegraf_defaults()"); then
|
||||
case "$out" in
|
||||
t) stalled=true ;;
|
||||
f) stalled=false ;;
|
||||
*) probe_failed=true ;;
|
||||
esac
|
||||
else
|
||||
probe_failed=true
|
||||
fi
|
||||
if $probe_failed; then
|
||||
echo "ERROR: could not check Telegraf default partitions; the maintenance job was not changed." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Guarded on the live GUC so applying this before the postgresql.conf change
|
||||
# has restarted the container skips rather than failing.
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d postgres <<'EOSQL'
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -v stalled="$stalled" \
|
||||
-U postgres -d postgres <<'EOSQL'
|
||||
SELECT CASE WHEN current_setting('cron.database_name', true) = current_database()
|
||||
THEN 'true' ELSE 'false' END AS cron_here \gset
|
||||
\if :cron_here
|
||||
CREATE EXTENSION IF NOT EXISTS pg_cron;
|
||||
-- cron.schedule_in_database is idempotent by jobname.
|
||||
-- A highstate reconciles the job's definition but never its active flag; only so-telegraf-repair turns it on.
|
||||
BEGIN;
|
||||
SELECT coalesce(max(jobid)::text, '') AS jobid,
|
||||
count(*) > 0 AS job_exists,
|
||||
coalesce(bool_and(active), true) AS job_active
|
||||
FROM cron.job
|
||||
WHERE jobname = 'telegraf-partman-maintenance' AND username = 'so_maint_telegraf' \gset
|
||||
SELECT cron.unschedule(jobid) FROM cron.job
|
||||
WHERE jobname = 'telegraf-partman-maintenance' AND username <> 'so_maint_telegraf';
|
||||
\if :job_exists
|
||||
SELECT cron.alter_job(:jobid,
|
||||
schedule := '17 * * * *',
|
||||
command := 'CALL so_admin.telegraf_maintenance()',
|
||||
database := 'so_telegraf');
|
||||
\else
|
||||
-- Registered inactive on a stalled grid so nothing drains the default
|
||||
-- partitions before so-telegraf-repair truncates them.
|
||||
SELECT cron.schedule_in_database(
|
||||
'telegraf-partman-maintenance',
|
||||
'17 * * * *',
|
||||
'CALL so_admin.telegraf_maintenance()',
|
||||
'so_telegraf'
|
||||
'so_telegraf',
|
||||
'so_maint_telegraf',
|
||||
NOT :'stalled'::boolean
|
||||
);
|
||||
\endif
|
||||
UPDATE cron.job SET nodename = current_setting('cron.host')
|
||||
WHERE jobname = 'telegraf-partman-maintenance'
|
||||
AND nodename IS DISTINCT FROM current_setting('cron.host');
|
||||
COMMIT;
|
||||
SELECT coalesce(bool_and(active), true) AS job_active
|
||||
FROM cron.job
|
||||
WHERE jobname = 'telegraf-partman-maintenance' AND username = 'so_maint_telegraf' \gset
|
||||
\if :job_active
|
||||
\else
|
||||
\echo 'WARNING: the Telegraf maintenance job is inactive; run ''sudo so-telegraf-repair''.'
|
||||
\endif
|
||||
\else
|
||||
\echo 'pg_cron metadata database is not `postgres` yet; skipping job registration.'
|
||||
\endif
|
||||
@@ -241,13 +365,15 @@ SET retention = :'retention_days' || ' days',
|
||||
retention_keep_table = false,
|
||||
premake = 7,
|
||||
infinite_time_partitions = true
|
||||
WHERE parent_table LIKE 'telegraf.%';
|
||||
WHERE parent_table LIKE 'telegraf.%'
|
||||
AND EXISTS (SELECT 1 FROM pg_catalog.pg_class c
|
||||
WHERE c.oid = to_regclass(parent_table) AND c.relkind = 'p');
|
||||
\endif
|
||||
EOSQL
|
||||
;;
|
||||
|
||||
maintenance)
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U so_maint_telegraf -d so_telegraf <<'EOSQL'
|
||||
SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL
|
||||
THEN 'true' ELSE 'false' END AS has_proc \gset
|
||||
\if :has_proc
|
||||
@@ -259,7 +385,7 @@ EOSQL
|
||||
;;
|
||||
|
||||
check)
|
||||
docker exec -i so-postgres psql -U postgres -d so_telegraf <<'EOSQL'
|
||||
docker exec -i so-postgres psql -U so_maint_telegraf -d so_telegraf <<'EOSQL'
|
||||
\pset border 2
|
||||
SELECT * FROM so_admin.telegraf_partition_status();
|
||||
EOSQL
|
||||
@@ -278,9 +404,9 @@ LIMIT 5;
|
||||
\echo 'pg_cron is not installed in this database.'
|
||||
\endif
|
||||
EOSQL
|
||||
unhealthy=$(docker exec so-postgres psql -U postgres -d so_telegraf -tAc \
|
||||
unhealthy=$(docker exec so-postgres psql -U so_maint_telegraf -d so_telegraf -tAc \
|
||||
"SELECT count(*) FROM so_admin.telegraf_partition_status()
|
||||
WHERE coalesce(default_rows, 0) > 0 OR coalesce(days_ahead, -1) < 1")
|
||||
WHERE default_has_rows OR coalesce(days_ahead, -1) < 1")
|
||||
if [ "${unhealthy:-1}" != "0" ]; then
|
||||
echo "so-telegraf-postgres check: $unhealthy telegraf parent(s) unhealthy" >&2
|
||||
exit 1
|
||||
|
||||
@@ -11,26 +11,38 @@
|
||||
# maintenance once. Healthy grids are reported and left alone.
|
||||
#
|
||||
# Usage: so-telegraf-repair [--check] [--yes] [--no-restart]
|
||||
# --check Report health and change nothing.
|
||||
# --check Report health and change nothing. Also reports a missing
|
||||
# so_maint_telegraf role and a maintenance job that is missing
|
||||
# or inactive.
|
||||
# --yes Skip the confirmation prompt (for soup and other automation).
|
||||
# --no-restart Never restart so-postgres, even if pg_cron's launcher is dead.
|
||||
#
|
||||
# Repair mode (without --check) registers the so_maint_telegraf role and the
|
||||
# maintenance job if they are missing, pauses the job while it repairs, and
|
||||
# always leaves it active afterwards. Data work runs as so_maint_telegraf.
|
||||
#
|
||||
# Exit status:
|
||||
# 0 healthy, or repair completed
|
||||
# 1 repair is needed (--check only)
|
||||
# 2 cannot run here: so-postgres, so_telegraf or pg_partman is missing
|
||||
# 1 repair is needed (--check), or the repair failed after it started
|
||||
# 2 skipped: Telegraf output is not Postgres, or so-postgres, so_telegraf or pg_partman is missing
|
||||
# 3 could not check, for example a connection error
|
||||
|
||||
set -e
|
||||
set -eE
|
||||
|
||||
# Matches p_premake in telegraf.conf's create_parent template.
|
||||
PREMAKE=7
|
||||
JOB_NAME=telegraf-partman-maintenance
|
||||
MAINT_ROLE=so_maint_telegraf
|
||||
# Long enough for an hourly maintenance run to release its locks, short enough
|
||||
# that soup and admins are not left hanging.
|
||||
LOCK_TIMEOUT=60s
|
||||
PG_OPTS="-c lock_timeout=$LOCK_TIMEOUT -c application_name=so-telegraf-repair"
|
||||
|
||||
CHECK_ONLY=false
|
||||
ASSUME_YES=false
|
||||
NO_RESTART=false
|
||||
|
||||
usage() { sed -n '/^# Usage:/,/^# 2 /p' "$0" | sed 's/^# \?//'; }
|
||||
usage() { sed -n '/^# Usage:/,/^# 3 /p' "$0" | sed 's/^# \?//'; }
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
@@ -38,75 +50,170 @@ while [[ $# -gt 0 ]]; do
|
||||
--yes|-y) ASSUME_YES=true ;;
|
||||
--no-restart) NO_RESTART=true ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) echo "Unknown option: $1" >&2; usage >&2; exit 2 ;;
|
||||
*) echo "Unknown option: $1" >&2; usage >&2; exit 3 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
skip() { echo "$*"; exit 2; }
|
||||
# Strips C0 (except tab and newline), DEL and C1 characters, so names and
|
||||
# messages written with Telegraf credentials can't drive the terminal.
|
||||
sanitise() { LC_ALL=C tr -d '\000-\010\013-\037\177' | LC_ALL=C sed 's/\xc2[\x80-\x9f]//g'; }
|
||||
|
||||
psql_tg() { docker exec -i so-postgres psql -U postgres -d so_telegraf "$@"; }
|
||||
psql_pg() { docker exec -i so-postgres psql -U postgres -d postgres "$@"; }
|
||||
ERR_FILE=$(mktemp)
|
||||
ERR_STATUS=3
|
||||
ERR_PREFIX="Could not check Telegraf partitions"
|
||||
job_id=
|
||||
restore_active=
|
||||
|
||||
on_err() {
|
||||
# Command substitutions inherit this trap; only the top level reports.
|
||||
[[ $BASH_SUBSHELL -eq 0 ]] || exit "$ERR_STATUS"
|
||||
local line
|
||||
line=$(tail -n 1 "$ERR_FILE" | sanitise)
|
||||
echo "$ERR_PREFIX: ${line:-command failed}" >&2
|
||||
exit "$ERR_STATUS"
|
||||
}
|
||||
|
||||
on_exit() {
|
||||
if [[ -n "$restore_active" ]]; then
|
||||
psql_cron -tAc "SELECT cron.alter_job($job_id, active := $restore_active)" >/dev/null || true
|
||||
fi
|
||||
rm -f "$ERR_FILE"
|
||||
}
|
||||
|
||||
trap on_err ERR
|
||||
trap on_exit EXIT
|
||||
|
||||
finish() { trap - ERR; exit "$1"; }
|
||||
skip() { echo "$*"; finish 2; }
|
||||
|
||||
# Keeps the last stderr line for on_err while still showing all of it.
|
||||
capture() {
|
||||
local rc=0
|
||||
: >"$ERR_FILE"
|
||||
"$@" 2>"$ERR_FILE" || rc=$?
|
||||
sanitise <"$ERR_FILE" >&2
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
psql_tg() {
|
||||
capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U "$MAINT_ROLE" -d so_telegraf "$@"
|
||||
}
|
||||
psql_pg() {
|
||||
capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U postgres -d postgres "$@"
|
||||
}
|
||||
psql_cron() {
|
||||
capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U postgres -d "$cron_db" "$@"
|
||||
}
|
||||
|
||||
# A part_config row counts only while its parent is a partitioned table.
|
||||
TELEGRAF_PARENTS="parent_table LIKE 'telegraf.%'
|
||||
AND EXISTS (SELECT 1 FROM pg_class c
|
||||
WHERE c.oid = to_regclass(parent_table) AND c.relkind = 'p')"
|
||||
|
||||
# query_to_xml so the per-table row counts need no helper function installed.
|
||||
REPORT="
|
||||
WITH parents AS (
|
||||
SELECT pc.parent_table,
|
||||
pc.premake,
|
||||
split_part(pc.parent_table, '.', 1) AS sch,
|
||||
split_part(pc.parent_table, '.', 2) AS tbl
|
||||
FROM partman.part_config pc
|
||||
WHERE pc.parent_table LIKE 'telegraf.%'
|
||||
), children AS (
|
||||
SELECT p.parent_table,
|
||||
max(to_date(substring(c.relname FROM '_p(\d{8})\$'), 'YYYYMMDD')) AS newest_child
|
||||
FROM parents p
|
||||
JOIN pg_class pt ON pt.oid = p.parent_table::regclass
|
||||
JOIN pg_inherits i ON i.inhparent = pt.oid
|
||||
JOIN pg_class c ON c.oid = i.inhrelid
|
||||
WHERE pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT'
|
||||
GROUP BY p.parent_table
|
||||
), defaults AS (
|
||||
SELECT p.parent_table,
|
||||
p.premake,
|
||||
format('%I.%I', p.sch, p.tbl || '_default') AS default_table,
|
||||
to_regclass(format('%I.%I', p.sch, p.tbl || '_default')) AS default_oid
|
||||
FROM parents p
|
||||
)
|
||||
SELECT d.parent_table,
|
||||
c.newest_child,
|
||||
(c.newest_child - current_date) AS days_ahead,
|
||||
d.premake,
|
||||
CASE WHEN d.default_oid IS NULL THEN NULL ELSE
|
||||
(xpath('/row/cnt/text()',
|
||||
query_to_xml(format('SELECT count(*) AS cnt FROM %s', d.default_table),
|
||||
false, true, '')))[1]::text::bigint
|
||||
END AS default_rows,
|
||||
CASE WHEN d.default_oid IS NULL THEN NULL
|
||||
ELSE pg_size_pretty(pg_total_relation_size(d.default_oid)) END AS default_size
|
||||
FROM defaults d
|
||||
LEFT JOIN children c ON c.parent_table = d.parent_table
|
||||
ORDER BY 1
|
||||
pc.premake,
|
||||
d.has_rows AS default_has_rows,
|
||||
pg_size_pretty(d.default_size) AS default_size
|
||||
FROM so_admin.telegraf_defaults() d
|
||||
JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table
|
||||
JOIN pg_class p ON p.oid = d.parent_table AND p.relkind = 'p'
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT max(to_date(substring(ch.relname FROM '_p(\d{8})\$'), 'YYYYMMDD')) AS newest_child
|
||||
FROM pg_inherits i
|
||||
JOIN pg_class ch ON ch.oid = i.inhrelid
|
||||
WHERE i.inhparent = d.parent_table
|
||||
AND pg_get_expr(ch.relpartbound, ch.oid) <> 'DEFAULT'
|
||||
) c ON true
|
||||
ORDER BY d.parent_table::text
|
||||
"
|
||||
|
||||
docker ps --format '{{.Names}}' | grep -qx so-postgres \
|
||||
print_report() {
|
||||
local report
|
||||
report=$(psql_tg -c "$REPORT")
|
||||
sanitise <<<"$report"
|
||||
}
|
||||
|
||||
has_cron() {
|
||||
[[ -n "$cron_db" ]] || return 0
|
||||
psql_cron -tAc "SELECT 1 FROM pg_extension WHERE extname = 'pg_cron'"
|
||||
}
|
||||
|
||||
lookup_job() {
|
||||
local cron
|
||||
cron=$(has_cron)
|
||||
[[ -n "$cron" ]] || return 0
|
||||
psql_cron -tA -F '|' -c \
|
||||
"SELECT jobid, active FROM cron.job
|
||||
WHERE jobname = '$JOB_NAME' AND username = '$MAINT_ROLE'
|
||||
ORDER BY jobid LIMIT 1"
|
||||
}
|
||||
|
||||
output=$(capture salt-call pillar.get telegraf:output --out=newline_values_only)
|
||||
output=$(tr -d '[:space:]' <<<"$output" | tr '[:lower:]' '[:upper:]')
|
||||
output=${output:-INFLUXDB}
|
||||
case "$output" in
|
||||
POSTGRES|BOTH) ;;
|
||||
*) skip "Telegraf output is $(sanitise <<<"$output"); not checking Postgres partitions." ;;
|
||||
esac
|
||||
|
||||
containers=$(capture docker ps --format '{{.Names}}')
|
||||
grep -qx so-postgres <<<"$containers" \
|
||||
|| skip "so-postgres is not running; nothing to repair."
|
||||
docker exec so-postgres psql -U postgres -tAc \
|
||||
"SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1 \
|
||||
has_db=$(psql_pg -tAc "SELECT 1 FROM pg_database WHERE datname = 'so_telegraf'")
|
||||
[[ -n "$has_db" ]] \
|
||||
|| skip "The so_telegraf database does not exist; Telegraf is not writing to Postgres."
|
||||
psql_tg -tAc "SELECT 1 FROM pg_extension WHERE extname='pg_partman'" | grep -q 1 \
|
||||
|
||||
# so_telegraf before the postgres state lands, postgres after.
|
||||
cron_db=$(psql_pg -tAc "SELECT current_setting('cron.database_name', true)")
|
||||
cron_db=$(tr -d '[:space:]' <<<"$cron_db")
|
||||
|
||||
role_ok=$(psql_pg -tAc \
|
||||
"SELECT rolcanlogin AND NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls)
|
||||
FROM pg_roles WHERE rolname = '$MAINT_ROLE'")
|
||||
job=
|
||||
[[ "$cron_db" == postgres ]] && job=$(lookup_job)
|
||||
|
||||
if [[ "$role_ok" != t ]] && $CHECK_ONLY; then
|
||||
echo "Repair is needed:"
|
||||
if [[ -z "$role_ok" ]]; then
|
||||
echo " * maintenance role $MAINT_ROLE is missing"
|
||||
else
|
||||
echo " * maintenance role $MAINT_ROLE is misconfigured"
|
||||
fi
|
||||
echo
|
||||
echo "Re-run without --check to repair."
|
||||
finish 1
|
||||
fi
|
||||
|
||||
if ! $CHECK_ONLY && [[ "$role_ok" != t || ( "$cron_db" == postgres && -z "$job" ) ]]; then
|
||||
echo "Registering the maintenance role and job."
|
||||
if ! /usr/sbin/so-telegraf-postgres group_role; then
|
||||
echo "Could not register the maintenance job; run a highstate and try again." >&2
|
||||
finish 3
|
||||
fi
|
||||
[[ "$cron_db" == postgres ]] && job=$(lookup_job)
|
||||
fi
|
||||
|
||||
has_partman=$(psql_tg -tAc "SELECT 1 FROM pg_extension WHERE extname = 'pg_partman'")
|
||||
[[ -n "$has_partman" ]] \
|
||||
|| skip "pg_partman is not installed in so_telegraf; nothing to repair."
|
||||
|
||||
parents=$(psql_tg -tAc \
|
||||
"SELECT count(*) FROM partman.part_config WHERE parent_table LIKE 'telegraf.%'")
|
||||
stranded=$(psql_tg -tAc "SELECT coalesce(sum(default_rows), 0) FROM ( $REPORT ) t")
|
||||
behind=$(psql_tg -tAc \
|
||||
"SELECT count(*) FROM ( $REPORT ) t WHERE coalesce(days_ahead, -1) < 1")
|
||||
"SELECT count(*) FROM partman.part_config WHERE $TELEGRAF_PARENTS")
|
||||
counts=$(psql_tg -tAc \
|
||||
"SELECT count(*) FILTER (WHERE default_has_rows) || '|' ||
|
||||
count(*) FILTER (WHERE coalesce(days_ahead, -1) < 1)
|
||||
FROM ( $REPORT ) t")
|
||||
IFS='|' read -r stranded behind <<<"$counts"
|
||||
# premake < 7, or infinite_time_partitions off: without the latter partman
|
||||
# refuses to premake forward across the gap the stall left behind.
|
||||
misconfigured=$(psql_tg -tAc \
|
||||
"SELECT count(*) FROM partman.part_config
|
||||
WHERE parent_table LIKE 'telegraf.%'
|
||||
WHERE $TELEGRAF_PARENTS
|
||||
AND (premake < $PREMAKE OR NOT infinite_time_partitions)")
|
||||
|
||||
# Both columns are matched because which one carries the launcher's name varies
|
||||
@@ -115,16 +222,24 @@ launcher=$(psql_pg -tAc \
|
||||
"SELECT count(*) FROM pg_stat_activity
|
||||
WHERE backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%'")
|
||||
|
||||
# so_telegraf before the postgres state lands, postgres after.
|
||||
cron_db=$(docker exec so-postgres psql -U postgres -tAc \
|
||||
"SELECT current_setting('cron.database_name', true)" | tr -d '[:space:]')
|
||||
last_run=never
|
||||
if [[ -n "$cron_db" ]]; then
|
||||
last_run=$(docker exec so-postgres psql -U postgres -d "$cron_db" -tAc \
|
||||
"SELECT coalesce(max(d.start_time)::text, 'never')
|
||||
cron_installed=$(has_cron)
|
||||
if [[ -n "$cron_installed" ]]; then
|
||||
run=$(psql_cron -tAc \
|
||||
"SELECT d.status || ' at ' || d.start_time
|
||||
FROM cron.job_run_details d JOIN cron.job j USING (jobid)
|
||||
WHERE j.jobname = '$JOB_NAME'" 2>/dev/null | tr -d '[:space:]' || echo unknown)
|
||||
[[ -n "$last_run" ]] || last_run=never
|
||||
WHERE j.jobname = '$JOB_NAME'
|
||||
ORDER BY d.start_time DESC LIMIT 1")
|
||||
if [[ -n "$run" ]]; then
|
||||
message=$(psql_cron -tAc \
|
||||
"SELECT coalesce(d.return_message, '')
|
||||
FROM cron.job_run_details d JOIN cron.job j USING (jobid)
|
||||
WHERE j.jobname = '$JOB_NAME'
|
||||
ORDER BY d.start_time DESC LIMIT 1")
|
||||
message=$(sanitise <<<"$message")
|
||||
message=${message//$'\n'/ }
|
||||
last_run="$(sanitise <<<"$run"): ${message:0:200}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# A grid that has never written a metric has nothing to recover, and an empty
|
||||
@@ -132,43 +247,67 @@ fi
|
||||
restart_needed=false
|
||||
[[ "$launcher" -eq 0 && "$parents" -gt 0 && -n "$cron_db" ]] && restart_needed=true
|
||||
|
||||
# group_role registers the job only in the postgres database.
|
||||
job_inactive=false
|
||||
if [[ "$cron_db" == postgres ]]; then
|
||||
[[ -z "$job" || "${job##*|}" == f ]] && job_inactive=true
|
||||
fi
|
||||
|
||||
repair_needed=false
|
||||
[[ "$stranded" -gt 0 ]] && repair_needed=true
|
||||
[[ "$behind" -gt 0 ]] && repair_needed=true
|
||||
[[ "$misconfigured" -gt 0 ]] && repair_needed=true
|
||||
$restart_needed && repair_needed=true
|
||||
$job_inactive && repair_needed=true
|
||||
|
||||
echo "Telegraf partition status:"
|
||||
psql_tg -c "$REPORT"
|
||||
echo "Rows stranded in default partitions: $stranded"
|
||||
echo "pg_cron metadata database: ${cron_db:-unset}"
|
||||
print_report
|
||||
echo "Default partitions with rows: $stranded"
|
||||
echo "pg_cron metadata database: $(sanitise <<<"${cron_db:-unset}")"
|
||||
[[ "$cron_db" == postgres ]] \
|
||||
|| echo "pg_cron metadata database is $(sanitise <<<"${cron_db:-unset}"); not checking the maintenance job."
|
||||
echo "pg_cron launcher running: $([[ "$launcher" -gt 0 ]] && echo yes || echo no)"
|
||||
echo "Last $JOB_NAME run: $last_run"
|
||||
echo
|
||||
|
||||
if ! $repair_needed; then
|
||||
echo "Telegraf partitions are healthy. Nothing to do."
|
||||
exit 0
|
||||
finish 0
|
||||
fi
|
||||
|
||||
if $CHECK_ONLY; then
|
||||
echo "Repair is needed:"
|
||||
[[ "$stranded" -gt 0 ]] && echo " * $stranded row(s) stranded in default partitions"
|
||||
[[ "$stranded" -gt 0 ]] && echo " * $stranded default partition(s) hold rows"
|
||||
[[ "$behind" -gt 0 ]] && echo " * $behind parent(s) with no partition for the current window"
|
||||
[[ "$misconfigured" -gt 0 ]] && echo " * $misconfigured parent(s) with stale partman settings"
|
||||
$restart_needed && echo " * pg_cron's launcher is dead; maintenance is not running at all"
|
||||
$job_inactive && echo " * the maintenance job is missing or inactive"
|
||||
echo
|
||||
echo "Re-run without --check to repair."
|
||||
exit 1
|
||||
finish 1
|
||||
fi
|
||||
|
||||
if [[ "$stranded" -gt 0 ]] && ! $ASSUME_YES; then
|
||||
echo "This will permanently discard the $stranded stranded row(s) above."
|
||||
echo "This will permanently discard the rows in the $stranded default partition(s) above."
|
||||
$restart_needed && ! $NO_RESTART && \
|
||||
echo "so-postgres will also be restarted, which briefly interrupts SOC."
|
||||
[[ -t 0 ]] || { echo "Not a terminal; re-run with --yes to confirm." >&2; exit 2; }
|
||||
[[ -t 0 ]] || { echo "Not a terminal; re-run with --yes to confirm." >&2; finish 1; }
|
||||
read -r -p "Continue? [y/N] " answer
|
||||
[[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 0; }
|
||||
[[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; finish 0; }
|
||||
fi
|
||||
|
||||
ERR_STATUS=1
|
||||
ERR_PREFIX="Telegraf partition repair failed"
|
||||
|
||||
if [[ "$cron_db" == postgres ]]; then
|
||||
if [[ -z "$job" ]]; then
|
||||
echo "Could not register the maintenance job; run a highstate and try again." >&2
|
||||
finish 3
|
||||
fi
|
||||
job_id=${job%%|*}
|
||||
[[ "${job##*|}" == t ]] && restore_active=true || restore_active=false
|
||||
echo "Pausing the $JOB_NAME job during the repair."
|
||||
psql_cron -tAc "SELECT cron.alter_job($job_id, active := false)" >/dev/null
|
||||
fi
|
||||
|
||||
if [[ "$misconfigured" -gt 0 ]]; then
|
||||
@@ -178,7 +317,7 @@ if [[ "$misconfigured" -gt 0 ]]; then
|
||||
"UPDATE partman.part_config
|
||||
SET premake = GREATEST(premake, $PREMAKE),
|
||||
infinite_time_partitions = true
|
||||
WHERE parent_table LIKE 'telegraf.%'"
|
||||
WHERE $TELEGRAF_PARENTS"
|
||||
fi
|
||||
|
||||
if [[ "$stranded" -gt 0 ]]; then
|
||||
@@ -186,27 +325,24 @@ if [[ "$stranded" -gt 0 ]]; then
|
||||
# One transaction: Telegraf is still writing, so a default emptied without
|
||||
# its partition in place is refilled before maintenance can attach one.
|
||||
psql_tg -v ON_ERROR_STOP=1 <<'EOSQL'
|
||||
BEGIN;
|
||||
DO $$
|
||||
DECLARE
|
||||
r record;
|
||||
BEGIN
|
||||
FOR r IN
|
||||
SELECT pc.parent_table,
|
||||
format('%I.%I', n.nspname, c.relname) AS default_table
|
||||
FROM partman.part_config pc
|
||||
JOIN pg_class p ON p.oid = pc.parent_table::regclass
|
||||
JOIN pg_inherits i ON i.inhparent = p.oid
|
||||
JOIN pg_class c ON c.oid = i.inhrelid
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE pc.parent_table LIKE 'telegraf.%'
|
||||
AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT'
|
||||
SELECT pc.parent_table, d.default_part
|
||||
FROM so_admin.telegraf_defaults() d
|
||||
JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table
|
||||
WHERE d.default_part IS NOT NULL
|
||||
LOOP
|
||||
EXECUTE format('TRUNCATE TABLE %s', r.default_table);
|
||||
EXECUTE format('TRUNCATE TABLE %s', r.default_part);
|
||||
PERFORM partman.create_partition_time(
|
||||
r.parent_table, ARRAY[date_trunc('day', now())]::timestamptz[]);
|
||||
END LOOP;
|
||||
END
|
||||
$$;
|
||||
COMMIT;
|
||||
EOSQL
|
||||
fi
|
||||
|
||||
@@ -222,7 +358,7 @@ if $restart_needed; then
|
||||
sleep 2
|
||||
done
|
||||
docker exec so-postgres pg_isready -U postgres -q \
|
||||
|| { echo "so-postgres did not come back; check 'docker logs so-postgres'." >&2; exit 1; }
|
||||
|| { echo "so-postgres did not come back; check 'docker logs so-postgres'." >&2; finish 1; }
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -238,8 +374,14 @@ CALL partman.run_maintenance_proc();
|
||||
\endif
|
||||
EOSQL
|
||||
|
||||
if [[ -n "$job_id" ]]; then
|
||||
psql_cron -tAc "SELECT cron.alter_job($job_id, active := true)" >/dev/null
|
||||
restore_active=
|
||||
echo "The $JOB_NAME job is active."
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "Telegraf partition status after repair:"
|
||||
psql_tg -c "$REPORT"
|
||||
print_report
|
||||
echo "The $JOB_NAME job runs hourly at :17. Confirm it fired with:"
|
||||
echo " so-telegraf-repair --check"
|
||||
@@ -251,6 +251,16 @@ socsensoronirepos:
|
||||
- mode: 775
|
||||
- makedirs: True
|
||||
|
||||
nsm_rules_dir:
|
||||
file.directory:
|
||||
- name: /nsm/rules
|
||||
- user: socore
|
||||
- group: socore
|
||||
- recurse:
|
||||
- user
|
||||
- group
|
||||
- show_changes: False
|
||||
|
||||
create_custom_local_yara_repo_template:
|
||||
git.present:
|
||||
- name: /nsm/rules/custom-local-repos/local-yara
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
{%- set MDENGINE = GLOBALS.md_engine %}
|
||||
{%- set LOGSTASH_ENABLED = LOGSTASH_MERGED.enabled %}
|
||||
{%- set TG_OUT = TELEGRAFMERGED.output | upper %}
|
||||
{%- set PG_ONLY = TG_OUT == 'POSTGRES' %}
|
||||
{%- set PG_HOST = GLOBALS.manager_ip %}
|
||||
{#- Per-minion telegraf creds live in the grid-wide telegraf/creds.sls pillar,
|
||||
written by /usr/sbin/so-telegraf-cred on the manager. Each minion looks up
|
||||
@@ -138,6 +139,10 @@
|
||||
# AGGREGATOR PLUGINS #
|
||||
###############################################################################
|
||||
|
||||
[[aggregators.merge]]
|
||||
drop_original = true
|
||||
namepass = ["system", "swap", "kafka_partition"]
|
||||
|
||||
###############################################################################
|
||||
# INPUT PLUGINS #
|
||||
###############################################################################
|
||||
@@ -145,7 +150,12 @@
|
||||
# Read metrics about cpu usage
|
||||
[[inputs.cpu]]
|
||||
## Whether to report per-cpu stats or not
|
||||
{%- if PG_ONLY %}
|
||||
{#- SOC and so-stats-show read only cpu-total #}
|
||||
percpu = false
|
||||
{%- else %}
|
||||
percpu = true
|
||||
{%- endif %}
|
||||
## Whether to report total system cpu stats or not
|
||||
totalcpu = true
|
||||
## If true, collect raw CPU time metrics.
|
||||
@@ -195,6 +205,11 @@
|
||||
## The typical use case is for LVM volumes, to get the VG/LV name instead of
|
||||
## the near-meaningless DM-0 name.
|
||||
# name_templates = ["$ID_FS_LABEL","$DM_VG_NAME/$DM_LV_NAME"]
|
||||
{%- if PG_ONLY %}
|
||||
{#- Whole disks, md RAID included, already carry the I/O of their partitions and dm devices #}
|
||||
[inputs.diskio.tagdrop]
|
||||
name = ["sd*[0-9]", "vd*[0-9]", "xvd*[0-9]", "hd*[0-9]", "nvme*p[0-9]*", "mmcblk*p[0-9]*", "dm-*", "sr*", "loop*"]
|
||||
{%- endif %}
|
||||
|
||||
|
||||
# Get kernel statistics from /proc/stat
|
||||
@@ -238,6 +253,12 @@
|
||||
password = "{{ ES_PASS }}"
|
||||
insecure_skip_verify = true
|
||||
cluster_health = true
|
||||
{%- if PG_ONLY %}
|
||||
{#- SOC reads cluster status, indices and clusterstats_nodes ingest stats; alarms plan jvm and fs #}
|
||||
cluster_health_level = "cluster"
|
||||
node_stats = ["indices", "jvm", "fs"]
|
||||
namedrop = ["elasticsearch_clusterstats_indices"]
|
||||
{%- endif %}
|
||||
{%- elif grains['role'] in ['so-searchnode'] %}
|
||||
[[inputs.elasticsearch]]
|
||||
servers = ["https://{{ NODEIP }}:9200"]
|
||||
@@ -245,6 +266,10 @@
|
||||
username = "{{ ES_USER }}"
|
||||
password = "{{ ES_PASS }}"
|
||||
insecure_skip_verify = true
|
||||
{%- if PG_ONLY %}
|
||||
{#- SOC reads indices; alarms plan jvm and fs #}
|
||||
node_stats = ["indices", "jvm", "fs"]
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
|
||||
# ## Timeout for HTTP requests to the elastic search server(s)
|
||||
@@ -287,9 +312,13 @@
|
||||
collect = ["pipelines"]
|
||||
username = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user') }}"
|
||||
password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}"
|
||||
{%- if PG_ONLY %}
|
||||
{#- SOC reads logstash_events only #}
|
||||
namedrop = ["logstash_plugins"]
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
|
||||
{% if grains.role in ['so-manager','so-managersearch','so-standalone','so-receiver'] and GLOBALS.pipeline == "KAFKA" -%}
|
||||
{% if grains.role in ['so-manager','so-managerhype','so-managersearch','so-standalone','so-receiver'] and GLOBALS.pipeline == "KAFKA" -%}
|
||||
[[inputs.jolokia2_agent]]
|
||||
name_prefix= "kafka_"
|
||||
urls = ["http://{{ NODEIP }}:8778/jolokia"]
|
||||
@@ -346,6 +375,26 @@
|
||||
interval = "60s"
|
||||
{%- endif %}
|
||||
|
||||
{#- These report state that changes slowly; collecting them every interval only stores duplicate rows #}
|
||||
{%- set SLOWSCRIPTS = [] %}
|
||||
{%- for script in ['agentstatus.sh', 'os.sh', 'features.sh', 'raid.sh', 'zeekloss.sh'] %}
|
||||
{%- if script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %}
|
||||
{%- do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove(script) %}
|
||||
{%- do SLOWSCRIPTS.append(script) %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if SLOWSCRIPTS | length > 0 %}
|
||||
[[inputs.exec]]
|
||||
commands = [
|
||||
{%- for script in SLOWSCRIPTS %}
|
||||
["/scripts/{{script}}"]{% if not loop.last %},{% endif %}
|
||||
{%- endfor %}
|
||||
]
|
||||
data_format = "influx"
|
||||
timeout = "15s"
|
||||
interval = "5m"
|
||||
{%- endif %}
|
||||
|
||||
{%- if TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] | length > 0 %}
|
||||
[[inputs.exec]]
|
||||
commands = [
|
||||
@@ -358,12 +407,6 @@
|
||||
timeout = "15s"
|
||||
{%- endif %}
|
||||
|
||||
{%- if salt['pillar.get']('healthcheck:enabled', False) %}
|
||||
[[inputs.file]]
|
||||
files = ["/host/nsm/zeek/logs/zeek_restart.log"]
|
||||
data_format = "influx"
|
||||
{%- endif %}
|
||||
|
||||
[[inputs.file]]
|
||||
files = ["/etc/telegraf/node_config.json"]
|
||||
name_override = "node_config"
|
||||
@@ -374,6 +417,12 @@
|
||||
|
||||
# # Read metrics about network interface usage
|
||||
[[inputs.net]]
|
||||
{%- if PG_ONLY %}
|
||||
{%- from 'docker/docker.map.jinja' import DOCKERMERGED %}
|
||||
{#- Only virtual interfaces; every physical NIC, bond members included, is kept for SOC and alarms #}
|
||||
[inputs.net.tagdrop]
|
||||
interface = {{ (['veth*', 'docker*', 'lo'] + DOCKERMERGED.networks.keys() | list) | json }}
|
||||
{%- endif %}
|
||||
|
||||
# Scripts run every 30s||TELEGRAFMERGED.config.interval - ES index script doesn't need to run as frequently
|
||||
{%- if GLOBALS.is_manager or GLOBALS.role == 'so-heavynode' %}
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
|
||||
{% if GLOBALS.pipeline != 'REDIS' %}
|
||||
{# When global pipeline is not REDIS remove redis.sh script. KAFKA metrics are collected via jolokia agent. Config in telegraf.conf #}
|
||||
{% if GLOBALS.role in ['so-standalone', 'so-manager', 'so-managersearch', 'so-receiver', 'so-heavynode'] %}
|
||||
{% if GLOBALS.role in ['so-standalone', 'so-manager', 'so-managerhype', 'so-managersearch', 'so-receiver', 'so-heavynode'] %}
|
||||
{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('redis.sh') %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
@@ -10,7 +10,7 @@
|
||||
# if this script isn't already running
|
||||
if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
|
||||
|
||||
FILES=$(ls -1x /host/nsm/strelka/unprocessed | wc -l)
|
||||
FILES=$(find /host/nsm/strelka/unprocessed -maxdepth 1 -type f | wc -l)
|
||||
echo "faffiles files=$FILES"
|
||||
|
||||
fi
|
||||
|
||||
@@ -12,8 +12,9 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
|
||||
|
||||
CONTAINERSLOG=/var/log/somon/containers.log
|
||||
# the collector rewrites this every minute; report nothing rather than repeating a stale
|
||||
# file as if it were current, in case a run was skipped or the collector is wedged
|
||||
MAXAGE=150
|
||||
# file as if it were current, in case a run was skipped or the collector is wedged.
|
||||
# Kept under two collector cycles so a 60s telegraf interval can't emit the same snapshot twice.
|
||||
MAXAGE=90
|
||||
|
||||
if [ -r "$CONTAINERSLOG" ]; then
|
||||
AGE=$(( $(date +%s) - $(stat -c %Y "$CONTAINERSLOG") ))
|
||||
|
||||
@@ -11,27 +11,40 @@
|
||||
if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
|
||||
|
||||
PREVCOUNTFILE='/tmp/eps.txt'
|
||||
EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.events.in')"
|
||||
STATS="$(curl -s localhost:9600/_node/stats)"
|
||||
|
||||
if [ ! -z "$EVENTCOUNTCURRENT" ]; then
|
||||
|
||||
if [ -f "$PREVCOUNTFILE" ]; then
|
||||
EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE`
|
||||
else
|
||||
echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE
|
||||
# the division by 30 is because the agent interval is 30 seconds
|
||||
EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30))
|
||||
if [ "$EVENTS" -lt 0 ]; then
|
||||
EVENTS=0
|
||||
fi
|
||||
|
||||
echo "consumptioneps eps=${EVENTS%%.*}"
|
||||
if echo "$STATS" | jq -e '.pipelines.search' > /dev/null 2>&1; then
|
||||
STAGE=search
|
||||
COUNT="$(echo "$STATS" | jq '.pipelines.search.events.out')"
|
||||
elif echo "$STATS" | jq -e '.pipelines.receiver' > /dev/null 2>&1; then
|
||||
STAGE=receiver
|
||||
COUNT="$(echo "$STATS" | jq '.pipelines.receiver.events.in')"
|
||||
else
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ! [[ "$COUNT" =~ ^[0-9]+$ ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
NOW=$(date +%s)
|
||||
read -r PREVEPOCH PREVCOUNT 2>/dev/null < "$PREVCOUNTFILE"
|
||||
echo "${NOW} ${COUNT}" > "$PREVCOUNTFILE"
|
||||
|
||||
if ! [[ "$PREVEPOCH" =~ ^[0-9]+$ && "$PREVCOUNT" =~ ^[0-9]+$ ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
ELAPSED=$((NOW - PREVEPOCH))
|
||||
DELTA=$((COUNT - PREVCOUNT))
|
||||
if [ "$ELAPSED" -le 0 ] || [ "$DELTA" -lt 0 ]; then
|
||||
EPS=0
|
||||
else
|
||||
EPS=$((DELTA / ELAPSED))
|
||||
fi
|
||||
|
||||
echo "consumptioneps,stage=${STAGE} eps=${EPS}"
|
||||
|
||||
fi
|
||||
|
||||
exit 0
|
||||
@@ -10,8 +10,7 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
|
||||
FPS_ENABLED=$(cat /var/log/sostatus/fps_enabled)
|
||||
LKS_ENABLED=$(cat /var/log/sostatus/lks_enabled)
|
||||
|
||||
echo "features fps=$FPS_ENABLED"
|
||||
echo "features lks=$LKS_ENABLED"
|
||||
echo "features fps=$FPS_ENABLED,lks=$LKS_ENABLED"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
Reference in new issue
Block a user