Compare commits

...
Author SHA1 Message Date
Mike Reeves 6d7be47a69 Trim Telegraf collection on Postgres-only grids and add Postgres support to Elasticsearch growth scripts 2026-11-17 14:17:27 -05:00
Josh Patterson e51706acad Merge pull request #16399 from Security-Onion-Solutions/fix/auto-apply-push-result-unknown
FIX: queue auto state apply pushes
2026-11-17 10:53:39 -05:00
Jorge Reyes 9612b15fca Merge pull request #16400 from Security-Onion-Solutions/reyesj2-patch-2
remove unused localrules config
2026-11-16 14:58:56 -06:00
reyesj2 0f5170e17e soup missing closing } post merge resolution; 2026-11-16 14:19:22 -06:00
reyesj2 cac027d438 fix merge 2026-11-16 13:52:05 -06:00
Jorge Reyes dea6932342 Merge branch '3/dev' into reyesj2-patch-2 2026-11-16 13:50:14 -06:00
Mike Reeves 043b2185d6 Merge pull request #16398 from Security-Onion-Solutions/foundry/telegraf-partition-11tqi
Postgres: Telegraf partition maintenance over the socket, with grid-test fixes
2026-11-16 14:47:37 -05:00
reyesj2 e7bb00d19d move /nsm/rules ownership into soc/config.sls. nginx no longer serves /nsm/rules, but so-soc still mounts it read-write 2026-11-16 13:47:25 -06:00
reyesj2 8c3611ac72 only rewrite pillar if it has localrules reference(s) 2026-11-16 13:46:10 -06:00
reyesj2 5558c54632 so-yaml: treat non-mapping parents as missing keys
getKeyValue and removeKey now return early when a parent of the key is
null or not a mapping, so get, remove and purgelistitem report the key as
missing instead of crashing on a hand-edited pillar such as
`firewall: null`.
2026-11-16 13:44:58 -06:00
Mike Reeves 52f9fe3092 Remove blank line in soup script 2026-11-16 13:52:14 -05:00
Mike Reeves f492e87f20 Add check for post-upgrade version file 2026-11-16 12:12:11 -05:00
Mike Reeves 01518ec25e Fix formatting issues in soup script 2026-11-16 11:14:06 -05:00
Jorge Reyes 5f3c1c84a4 Merge pull request #16396 from Security-Onion-Solutions/reyesj2-patch-3
elastic artifact repo only needs /beats/ dir
2026-11-16 09:22:23 -06:00
Mike Reeves b01f01da82 Fix formatting issues in soup script 2026-11-13 16:07:55 -05:00
Mike Reeves 03317ba5e4 Postgres: Telegraf partition maintenance over the socket, with grid-test fixes 2026-11-12 20:42:29 -05:00
reyesj2 9bc3ae6b26 further tighten elastic artifact registry 2026-11-12 15:49:34 -06:00
Jorge Reyes 40a30f6908 Merge pull request #16389 from Security-Onion-Solutions/reyesj2-patch-1
update elastic-agent artifact repo directory
2026-11-11 14:32:59 -06:00
reyesj2 4e0558d2d3 remove unused localrules config 2026-11-10 17:30:01 -06:00
Mike Reeves 1fd37251c9 Merge pull request #16390 from Security-Onion-Solutions/foundry/mwaq014injcipm
Telegraf: fix EPS double counting and duplicate metric rows
2026-11-10 14:53:30 -05:00
Mike Reeves 4a5253f688 Telegraf: fix EPS double counting and duplicate metric rows 2026-11-10 08:26:19 -05:00
reyesj2 ec220572f0 dedup artifact registry config & split ea artifacts out of /opt/socore/html/ into dedicated /opt/socore/elastic-artifacts/artifacts/ 2026-11-09 12:32:06 -06:00
23 changed files with 1195 additions and 342 deletions

No files matched your search

+102
View File
@@ -656,6 +656,108 @@ set_version() {
fi
}
so_telegraf_output() {
local defaultsfile="${DEFAULT_SALT_DIR}/salt/telegraf/defaults.yaml"
local output
output=$(lookup_pillar output telegraf) || return 1
if [ -z "$output" ]; then
if [ ! -f "$defaultsfile" ]; then
echo "Could not find $defaultsfile" >&2
return 1
fi
output=$(grep -E "^ output: " "$defaultsfile" | awk -F: '{print $2}' | tr -d '[:space:]')
fi
echo "${output^^}"
}
so_telegraf_pg_check() {
local app="$1"
local databases role_ok
if ! docker_check_running postgres; then
echo "so-postgres is not running. Run $app on the manager." >&2
return 1
fi
if ! databases=$(docker exec so-postgres psql -U postgres -lqt 2>/dev/null); then
echo "Unable to list the so-postgres databases." >&2
return 1
fi
if ! echo "$databases" | cut -d\| -f1 | grep -qw so_telegraf; then
echo "Database so_telegraf not found. Is telegraf.output set to POSTGRES or BOTH?" >&2
return 1
fi
if ! role_ok=$(docker exec so-postgres psql -U postgres -tAc "SELECT rolcanlogin AND NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls) FROM pg_roles WHERE rolname = 'so_maint_telegraf'" 2>/dev/null); then
echo "Unable to query the so-postgres roles." >&2
return 1
fi
if [ -z "$role_ok" ]; then
echo "The so_maint_telegraf role is missing; run a highstate." >&2
return 1
elif [ "$role_ok" != t ]; then
echo "The so_maint_telegraf role is misconfigured; run a highstate." >&2
return 1
fi
}
so_telegraf_psql() {
local app="$1"
local sql="$2"
docker exec -e PGOPTIONS="-c application_name=${app} -c default_transaction_read_only=on -c client_min_messages=error" \
so-postgres psql -U so_maint_telegraf -d so_telegraf -v ON_ERROR_STOP=1 -At -F '|' -c "$sql"
}
# Prints one stream|current|then line per data stream, in bytes summed across hosts. current is
# each host's newest size in the past 4 hours; then is that host's newest size in the window
# [now - start, now - stop), or its current size when it has none there, so a host missing from the
# window adds no growth. Empty output means the window has no rows at all. Values over 15 digits
# (1 PB) are skipped so the sums here and in the callers stay far below the 64-bit limit; such a
# value can only come from a forged Telegraf writer.
so_es_index_size_growth() {
local app="$1"
local start="$2"
local stop="$3"
local relkind
relkind=$(so_telegraf_psql "$app" "SELECT c.relkind FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'telegraf' AND c.relname = 'elasticsearch_index_size'") || return 1
[[ "$relkind" == "r" || "$relkind" == "p" ]] || return 0
so_telegraf_psql "$app" "$(cat <<EOF
WITH sizes AS (
SELECT d.tag_id, d.time, f.key AS stream, f.value::bigint AS bytes
FROM telegraf.elasticsearch_index_size d
CROSS JOIN LATERAL pg_catalog.jsonb_each_text(d.fields) f
WHERE d.time >= now() - interval '${start}'
AND f.key ~ '^[^|"''[:space:][:cntrl:]]+$'
AND pg_catalog.strpos(f.key, pg_catalog.chr(92)) = 0
AND f.value ~ '^[0-9]{1,15}$'
),
cur AS (
SELECT DISTINCT ON (tag_id, stream) tag_id, stream, bytes
FROM sizes
WHERE time >= now() - interval '4 hours'
ORDER BY tag_id, stream, time DESC
),
past AS (
SELECT DISTINCT ON (tag_id, stream) tag_id, stream, bytes
FROM sizes
WHERE time < now() - interval '${stop}'
ORDER BY tag_id, stream, time DESC
)
SELECT c.stream, sum(c.bytes), sum(coalesce(p.bytes, c.bytes))
FROM cur c
LEFT JOIN past p USING (tag_id, stream)
WHERE EXISTS (
SELECT 1 FROM telegraf.elasticsearch_index_size
WHERE time >= now() - interval '${start}' AND time < now() - interval '${stop}'
)
GROUP BY c.stream
ORDER BY c.stream
EOF
)"
}
status () {
printf "\n=========================================================================\n$(date) | $1\n=========================================================================\n"
}
-1
View File
@@ -103,7 +103,6 @@ docker:
- 80:80
- 443:443
- 8443:8443
- 7788:7788
- 7789:7789
custom_bind_mounts: []
extra_hosts: []
@@ -5,6 +5,10 @@
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
# Reports per data stream Elasticsearch growth over the last 24 hours, 7 days and 30 days. The data
# source follows telegraf.output: Postgres in POSTGRES mode, where history is limited to
# postgres.telegraf.retention_days and the script must run on the manager; InfluxDB otherwise.
INFLUX_URL="https://localhost:8086/api/v2"
. /usr/sbin/so-common
@@ -18,8 +22,6 @@ lookup_org_id() {
echo "$response" | jq -r ".orgs[] | select(.name == \"Security Onion\").id"
}
ORG_ID=$(lookup_org_id)
run_flux_query() {
local query=$1
request "query?org=$ORG_ID" -H 'Accept:application/csv' -H 'Content-type:application/vnd.flux' -d "$query" -XPOST 2>/dev/null
@@ -39,75 +41,137 @@ bytes_to_gb() {
fi
}
indexes_query='from(bucket: "telegraf/so_long_term")
|> range(start: -7d)
|> filter(fn: (r) => r._measurement == "elasticsearch_index_size")
|> distinct(column: "_field")
|> keep(columns: ["_field"])'
influxdb_indices_growth() {
ORG_ID=$(lookup_org_id)
indexes_result=$(run_flux_query "$indexes_query")
indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | grep -v '^_field$' | sed 's/\r$//' | sort -u)
indexes_query='from(bucket: "telegraf/so_long_term")
|> range(start: -7d)
|> filter(fn: (r) => r._measurement == "elasticsearch_index_size")
|> distinct(column: "_field")
|> keep(columns: ["_field"])'
printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)"
printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})"
indexes_result=$(run_flux_query "$indexes_query")
indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | grep -v '^_field$' | sed 's/\r$//' | sort -u)
for index in $indexes; do
[[ -z "$index" ]] && continue
current_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -4h)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
current_result=$(run_flux_query "$current_query")
current_size=$(read_csv_result "$current_result")
current_size=${current_size:-0}
printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)"
printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})"
size_24h_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -25h, stop: -23h)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
size_24h_result=$(run_flux_query "$size_24h_query")
size_24h_ago=$(read_csv_result "$size_24h_result")
size_24h_ago=${size_24h_ago:-$current_size}
for index in $indexes; do
[[ -z "$index" ]] && continue
current_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -4h)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
current_result=$(run_flux_query "$current_query")
current_size=$(read_csv_result "$current_result")
current_size=${current_size:-0}
size_7d_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -7d8h, stop: -7d)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
size_7d_result=$(run_flux_query "$size_7d_query")
size_7d_ago=$(read_csv_result "$size_7d_result")
size_7d_ago=${size_7d_ago:-$current_size}
size_24h_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -25h, stop: -23h)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
size_24h_result=$(run_flux_query "$size_24h_query")
size_24h_ago=$(read_csv_result "$size_24h_result")
size_24h_ago=${size_24h_ago:-$current_size}
size_30d_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -30d8h, stop: -30d)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
size_30d_result=$(run_flux_query "$size_30d_query")
size_30d_ago=$(read_csv_result "$size_30d_result")
size_30d_ago=${size_30d_ago:-$current_size}
size_7d_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -7d8h, stop: -7d)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
size_7d_result=$(run_flux_query "$size_7d_query")
size_7d_ago=$(read_csv_result "$size_7d_result")
size_7d_ago=${size_7d_ago:-$current_size}
# if an index was recently cleaned up by ilm it will result in a negative number for 'index growth'.
growth_24h=$(( current_size > size_24h_ago ? current_size - size_24h_ago : 0 ))
size_30d_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -30d8h, stop: -30d)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
size_30d_result=$(run_flux_query "$size_30d_query")
size_30d_ago=$(read_csv_result "$size_30d_result")
size_30d_ago=${size_30d_ago:-$current_size}
growth_7d=$(( current_size > size_7d_ago ? current_size - size_7d_ago : 0 ))
# if an index was recently cleaned up by ilm it will result in a negative number for 'index growth'.
growth_24h=$(( current_size > size_24h_ago ? current_size - size_24h_ago : 0 ))
growth_30d=$(( current_size > size_30d_ago ? current_size - size_30d_ago : 0 ))
growth_7d=$(( current_size > size_7d_ago ? current_size - size_7d_ago : 0 ))
growth_24h_gb=$(bytes_to_gb "$growth_24h")
growth_7d_gb=$(bytes_to_gb "$growth_7d")
growth_30d_gb=$(bytes_to_gb "$growth_30d")
growth_30d=$(( current_size > size_30d_ago ? current_size - size_30d_ago : 0 ))
# Only results for indices with atleast 1 metric above 0.00
if [[ "$growth_24h_gb" != "0.00" ]] || [[ "$growth_7d_gb" != "0.00" ]] || [[ "$growth_30d_gb" != "0.00" ]]; then
printf "%020.2f|%-50s %15s %15s %15s\n" \
"$growth_24h" \
"$index" \
"$growth_24h_gb" \
"$growth_7d_gb" \
"$growth_30d_gb"
fi
done | sort -t'|' -k1,1nr | cut -d'|' -f2-
growth_24h_gb=$(bytes_to_gb "$growth_24h")
growth_7d_gb=$(bytes_to_gb "$growth_7d")
growth_30d_gb=$(bytes_to_gb "$growth_30d")
# Only results for indices with atleast 1 metric above 0.00
if [[ "$growth_24h_gb" != "0.00" ]] || [[ "$growth_7d_gb" != "0.00" ]] || [[ "$growth_30d_gb" != "0.00" ]]; then
printf "%020.2f|%-50s %15s %15s %15s\n" \
"$growth_24h" \
"$index" \
"$growth_24h_gb" \
"$growth_7d_gb" \
"$growth_30d_gb"
fi
done | sort -t'|' -k1,1nr | cut -d'|' -f2-
}
postgres_indices_growth() {
local app="so-elasticsearch-indices-growth"
local -a starts=("25 hours" "7 days 8 hours" "30 days 8 hours")
local -a stops=("23 hours" "7 days" "30 days")
local -A current past window_has_data
local i rows stream current_size past_size growth growth_gb sort_key show
local -a columns
for i in 0 1 2; do
if ! rows=$(so_es_index_size_growth "$app" "${starts[i]}" "${stops[i]}"); then
echo "Failed to query telegraf.elasticsearch_index_size in so_telegraf." >&2
exit 1
fi
[[ -z "$rows" ]] && continue
window_has_data[$i]=1
while IFS='|' read -r stream current_size past_size; do
current[$stream]=$current_size
past[$i,$stream]=$past_size
done <<< "$rows"
done
printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)"
printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})"
for stream in "${!current[@]}"; do
columns=()
sort_key=0
show=false
for i in 0 1 2; do
if [[ -z "${window_has_data[$i]}" ]]; then
columns+=("n/a")
continue
fi
current_size=${current[$stream]}
past_size=${past[$i,$stream]-$current_size}
growth=$(( current_size > past_size ? current_size - past_size : 0 ))
growth_gb=$(bytes_to_gb "$growth")
[[ "$growth_gb" != "0.00" ]] && show=true
[[ $i -eq 0 ]] && sort_key=$growth
columns+=("$growth_gb")
done
if [[ "$show" == true ]]; then
printf "%020.2f|%-50s %15s %15s %15s\n" "$sort_key" "$stream" "${columns[@]}"
fi
done | sort -t'|' -k1,1nr | cut -d'|' -f2-
}
if ! TG_OUTPUT=$(so_telegraf_output); then
echo "Unable to read telegraf.output from pillar." >&2
exit 1
fi
if [[ "$TG_OUTPUT" == "POSTGRES" ]]; then
so_telegraf_pg_check so-elasticsearch-indices-growth || exit 1
postgres_indices_growth
else
influxdb_indices_growth
fi
@@ -58,6 +58,9 @@ Usage: $(basename "$0") [OPTIONS]
Estimate remaining days until Elasticsearch cluster reaches low watermark threshold.
Index growth history follows telegraf.output: in POSTGRES mode it is read from Postgres, limited to
postgres.telegraf.retention_days, and the script must run on the manager; otherwise from InfluxDB.
OPTIONS:
--json Output results in JSON format
-v, --verbose Show additional output
@@ -168,11 +171,25 @@ run_indices_growth() {
fi
}
fetch_postgres_total_bytes() {
local total
total=$(so_es_index_size_growth so-elasticsearch-retention-estimate "$1" "$2") || return 1
echo "$total" | awk -F'|' '{ sum += $3 } END { print sum + 0 }'
}
fetch_total_bytes() {
local start="$1"
local stop="$2"
local pg_start="$3"
local pg_stop="$4"
local range_line
if [ "$TG_OUTPUT" = "POSTGRES" ]; then
fetch_postgres_total_bytes "$pg_start" "$pg_stop"
return
fi
if [ -n "$stop" ]; then
range_line=" |> range(start: ${start}, stop: ${stop})"
else
@@ -208,8 +225,13 @@ fail() {
echo -e "\nDISCLAIMER: Script output is based on current data patterns, but are approximations solely intended to assist with getting a general ILM policy configured."
ORG_ID=$(lookup_org_id)
[ -n "$ORG_ID" ] || fail "Unable to resolve InfluxDB org id"
TG_OUTPUT=$(so_telegraf_output) || fail "Unable to read telegraf.output from pillar."
if [ "$TG_OUTPUT" = "POSTGRES" ]; then
pg_check_error=$(so_telegraf_pg_check so-elasticsearch-retention-estimate 2>&1) || fail "$pg_check_error"
else
ORG_ID=$(lookup_org_id)
[ -n "$ORG_ID" ] || fail "Unable to resolve InfluxDB org id"
fi
cluster_storage_size=0
indexed_storage_source="elasticsearch"
@@ -221,10 +243,11 @@ if [ -n "$cluster_storage_size_output" ]; then
fi
fi
# historical data from influxdb for growth calculation
one_day_total=$(fetch_total_bytes "-25h" "-23h")
seven_day_total=$(fetch_total_bytes "-7d8h" "-7d")
thirty_day_total=$(fetch_total_bytes "-30d8h" "-30d")
# historical index sizes for growth calculation
query_error="Failed to query telegraf.elasticsearch_index_size in so_telegraf."
one_day_total=$(fetch_total_bytes "-25h" "-23h" "25 hours" "23 hours") || fail "$query_error"
seven_day_total=$(fetch_total_bytes "-7d8h" "-7d" "7 days 8 hours" "7 days") || fail "$query_error"
thirty_day_total=$(fetch_total_bytes "-30d8h" "-30d" "30 days 8 hours" "30 days") || fail "$query_error"
# available historical windows (prefer 30d/7d when available, to avoid using recent 24h traffic spike as true daily ingest rate)
history_days=0
@@ -244,7 +267,12 @@ elif [ "$one_day_total" -gt 0 ]; then
historical_total=$one_day_total
fi
[ "$history_days" -gt 0 ] || fail "Historical InfluxDB data unavailable for growth calculation. If this a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics."
if [ "$TG_OUTPUT" = "POSTGRES" ]; then
history_unavailable="Historical Postgres data unavailable for growth calculation. Postgres keeps postgres.telegraf.retention_days days of history; if this is a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics."
else
history_unavailable="Historical InfluxDB data unavailable for growth calculation. If this a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics."
fi
[ "$history_days" -gt 0 ] || fail "$history_unavailable"
# Daily growth rate
growth_bytes=$(( cluster_storage_size - historical_total ))
@@ -729,6 +757,7 @@ if [ "$should_trigger_recommendations" = true ]; then
[ -n "$index" ] || continue
growth_24h_gb=$(echo "$line" | awk '{print $(NF-2)}')
[[ "$growth_24h_gb" =~ ^[0-9]+(\.[0-9]+)?$ ]] || growth_24h_gb=0
creation_date_display=""
retention_days=""
-10
View File
@@ -118,10 +118,6 @@ firewall:
tcp:
- 5601
udp: []
localrules:
tcp:
- 7788
udp: []
nginx:
tcp:
- 80
@@ -200,7 +196,6 @@ firewall:
- postgres
- elasticsearch_rest
- elasticsearch_node
- localrules
sensor:
portgroups:
- beats_5044
@@ -391,7 +386,6 @@ firewall:
- elastic_agent_control
- elastic_agent_data
- elastic_agent_update
- localrules
- sensoroni
fleet:
portgroups:
@@ -611,7 +605,6 @@ firewall:
- elastic_agent_control
- elastic_agent_data
- elastic_agent_update
- localrules
- sensoroni
fleet:
portgroups:
@@ -829,7 +822,6 @@ firewall:
- elastic_agent_control
- elastic_agent_data
- elastic_agent_update
- localrules
- sensoroni
fleet:
portgroups:
@@ -1058,7 +1050,6 @@ firewall:
- elastic_agent_update
- endgame
- strelka_frontend
- localrules
fleet:
portgroups:
- elasticsearch_rest
@@ -1518,7 +1509,6 @@ firewall:
- elasticsearch_rest
- elasticsearch_node
- elastic_agent_control
- localrules
sensor:
portgroups:
- beats_5044
-3
View File
@@ -138,9 +138,6 @@ firewall:
kibana:
tcp: *tcpsettings
udp: *udpsettings
localrules:
tcp: *tcpsettings
udp: *udpsettings
nginx:
tcp: *tcpsettings
udp: *udpsettings
+33
View File
@@ -20,6 +20,7 @@ def showUsage(args):
print(' append - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
print(' appendlistobject - Append an object to a yaml list key. Requires KEY and JSON_OBJECT args.', file=sys.stderr)
print(' removelistitem - Remove a list item from a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
print(' purgelistitem - Remove every occurrence of a list item from all lists nested under a yaml key, if it exists. Requires KEY and LISTITEM args.', file=sys.stderr)
print(' replacelistobject - Replace a list object based on a condition. Requires KEY, CONDITION_FIELD, CONDITION_VALUE, and JSON_OBJECT args.', file=sys.stderr)
print(' add - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr)
print(' get [-r] - Displays (to stdout) the value stored in the given key. Requires KEY arg. Use -r for raw output without YAML formatting.', file=sys.stderr)
@@ -199,6 +200,33 @@ def removelistitem(args):
return 0
def purgeListItem(content, listItem):
if isinstance(content, dict):
for value in content.values():
purgeListItem(value, listItem)
elif isinstance(content, list):
content[:] = [item for item in content if item != listItem]
for item in content:
purgeListItem(item, listItem)
def purgelistitem(args):
if len(args) != 3:
print('Missing filename, key arg, or list item to purge', file=sys.stderr)
showUsage(None)
return 1
filename = args[0]
key = args[1]
listItem = args[2]
content = loadYaml(filename)
purgeListItem(getKeyValue(content, key), convertType(listItem))
writeYaml(filename, content)
return 0
def replaceListObjectByCondition(content, key, conditionField, conditionValue, newObject):
pieces = key.split(".", 1)
if len(pieces) > 1:
@@ -291,6 +319,8 @@ def add(args):
def removeKey(content, key):
if not isinstance(content, dict):
return
pieces = key.split(".", 1)
if len(pieces) > 1:
if pieces[0] in content:
@@ -334,6 +364,8 @@ def replace(args):
def getKeyValue(content, key):
if not isinstance(content, dict):
return None
pieces = key.split(".", 1)
if len(pieces) > 1 and pieces[0] in content:
return getKeyValue(content[pieces[0]], pieces[1])
@@ -385,6 +417,7 @@ def main():
"append": append,
"appendlistobject": appendlistobject,
"removelistitem": removelistitem,
"purgelistitem": purgelistitem,
"replacelistobject": replacelistobject,
"get": get,
"remove": remove,
+278
View File
@@ -128,6 +128,22 @@ class TestRemove(unittest.TestCase):
sysmock.assert_called_once_with(1)
self.assertIn("Missing filename or key arg\n", mock_stderr.getvalue())
def test_remove_null_parent(self):
filename = "/tmp/so-yaml_test-remove.yaml"
file = open(filename, "w")
file.write("key1: null\n")
file.close()
result = soyaml.remove([filename, "key1.child1.deep1"])
self.assertEqual(result, 0)
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1: null\n"
self.assertEqual(actual, expected)
def test_append_missing_arg(self):
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -565,6 +581,19 @@ class TestRemove(unittest.TestCase):
self.assertEqual(result, 2)
self.assertEqual("", mock_stdout.getvalue())
def test_get_null_parent(self):
with patch('sys.stdout', new=StringIO()) as mock_stdout:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
filename = "/tmp/so-yaml_test-get.yaml"
file = open(filename, "w")
file.write("key1: null\n")
file.close()
result = soyaml.get([filename, "key1.child1"])
self.assertEqual(result, 2)
self.assertEqual("", mock_stdout.getvalue())
self.assertIn("Key 'key1.child1' not found by so-yaml.py", mock_stderr.getvalue())
def test_get_empty_file(self):
with patch('sys.stdout', new=StringIO()) as mock_stdout:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -709,6 +738,255 @@ class TestRemoveListItem(unittest.TestCase):
self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())
class TestPurgeListItem(unittest.TestCase):
def test_purgelistitem_missing_arg(self):
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
sys.argv = ["cmd", "help"]
soyaml.purgelistitem(["file", "key"])
sysmock.assert_called()
self.assertIn("Missing filename, key arg, or list item to purge", mock_stderr.getvalue())
def test_purgelistitem(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: [a,b,c], key2: false}")
file.close()
soyaml.purgelistitem([filename, "key1", "b"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- a\n- c\nkey2: false\n"
self.assertEqual(actual, expected)
def test_purgelistitem_nested_lists(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: { child1: [a,b,c], child2: { deep1: [b,d], deep2: 45 } }, key2: [b,e]}")
file.close()
soyaml.purgelistitem([filename, "key1", "b"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n child1:\n - a\n - c\n child2:\n deep1:\n - d\n deep2: 45\nkey2:\n- b\n- e\n"
self.assertEqual(actual, expected)
def test_purgelistitem_duplicates(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: [b,a,b,c,b]}")
file.close()
soyaml.purgelistitem([filename, "key1", "b"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- a\n- c\n"
self.assertEqual(actual, expected)
def test_purgelistitem_list_of_objects(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: [{name: item1, values: [a,b]}, b, {name: item2, values: [b,c]}]}")
file.close()
soyaml.purgelistitem([filename, "key1", "b"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- name: item1\n values:\n - a\n- name: item2\n values:\n - c\n"
self.assertEqual(actual, expected)
def test_purgelistitem_typed(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: [1, '1', 2]}")
file.close()
soyaml.purgelistitem([filename, "key1", "1"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- '1'\n- 2\n"
self.assertEqual(actual, expected)
def test_purgelistitem_item_not_in_list(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: { child1: [a,b,c] }}")
file.close()
soyaml.purgelistitem([filename, "key1", "d"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n child1:\n - a\n - b\n - c\n"
self.assertEqual(actual, expected)
def test_purgelistitem_key_noexist(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: [a,b,c]}")
file.close()
result = soyaml.purgelistitem([filename, "key2.child1", "b"])
self.assertEqual(result, 0)
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- a\n- b\n- c\n"
self.assertEqual(actual, expected)
def test_purgelistitem_key_noexist_deep(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: { child1: [a,b,c] }}")
file.close()
result = soyaml.purgelistitem([filename, "key1.child2.deep1", "b"])
self.assertEqual(result, 0)
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n child1:\n - a\n - b\n - c\n"
self.assertEqual(actual, expected)
def test_purgelistitem_scalar_and_null_values(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: { child1: b, child2: null, child3: [b] }}")
file.close()
soyaml.purgelistitem([filename, "key1", "b"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n child1: b\n child2: null\n child3: []\n"
self.assertEqual(actual, expected)
def test_purgelistitem_lists_of_lists(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: [[a,b], [b,[b,c]], b]}")
file.close()
soyaml.purgelistitem([filename, "key1", "b"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- - a\n- - - c\n"
self.assertEqual(actual, expected)
def test_purgelistitem_file_object(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: [{name: x}, a, {name: y}, {name: x}]}")
file.close()
itemfile = "/tmp/so-yaml_test-purgelistitem-item.yaml"
file = open(itemfile, "w")
file.write("name: x\n")
file.close()
soyaml.purgelistitem([filename, "key1", "file:" + itemfile])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- a\n- name: y\n"
self.assertEqual(actual, expected)
def test_purgelistitem_main(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: { child1: [a,b,c] }, key2: [b]}")
file.close()
with patch('sys.exit', new=MagicMock()) as sysmock:
sys.argv = ["cmd", "purgelistitem", filename, "key1", "b"]
soyaml.main()
sysmock.assert_called_once_with(0)
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n child1:\n - a\n - c\nkey2:\n- b\n"
self.assertEqual(actual, expected)
def test_purgelistitem_with_remove(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: { child1: { deep1: [a] }, child2: [b] }, key2: { group1: { items: [x,child1,y] }, group2: { items: [child1] } }}")
file.close()
soyaml.remove([filename, "key1.child1"])
soyaml.purgelistitem([filename, "key2", "child1"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n child2:\n - b\nkey2:\n group1:\n items:\n - x\n - y\n group2:\n items: []\n"
self.assertEqual(actual, expected)
def test_purgelistitem_null_parent(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("key1: null\n")
file.close()
result = soyaml.purgelistitem([filename, "key1.child1", "b"])
self.assertEqual(result, 0)
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1: null\n"
self.assertEqual(actual, expected)
def test_purgelistitem_non_dict_parent(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("key1:\n- a\n- b\n")
file.close()
result = soyaml.purgelistitem([filename, "key1.child1", "b"])
self.assertEqual(result, 0)
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- a\n- b\n"
self.assertEqual(actual, expected)
class TestAppendListObject(unittest.TestCase):
def test_appendlistobject_missing_arg(self):
+50 -7
View File
@@ -5,7 +5,6 @@
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
. /usr/sbin/so-common
. /usr/sbin/so-image-common
@@ -17,7 +16,9 @@ INSTALLEDVERSION=$(cat /etc/soversion)
# not finish its post-upgrade steps; its contents are the resume point. It is read
# here before preupgrade_changes mutates INSTALLEDVERSION and before any highstate
# stamps /etc/soversion from the pillar.
POSTVERSION_FILE=/etc/sopostversion
if [ -f "$POSTVERSION_FILE" ]; then
POSTVERSION=$(cat "$POSTVERSION_FILE")
else
@@ -527,6 +528,7 @@ preupgrade_changes() {
[[ "$INSTALLEDVERSION" == "3.1.0" ]] && up_to_3.2.0
[[ "$INSTALLEDVERSION" == "3.2.0" ]] && up_to_3.3.0
[[ "$INSTALLEDVERSION" == "3.3.0" ]] && up_to_3.4.0
[[ "$INSTALLEDVERSION" == "3.4.0" ]] && up_to_3.5.0
true
}
@@ -546,6 +548,7 @@ postupgrade_changes() {
[[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0
[[ "$POSTVERSION" == "3.2.0" ]] && post_to_3.3.0
[[ "$POSTVERSION" == "3.3.0" ]] && post_to_3.4.0
[[ "$POSTVERSION" == "3.4.0" ]] && post_to_3.5.0
# All applicable post-upgrade steps completed; clear the resume marker.
rm -f "$POSTVERSION_FILE"
true
@@ -1065,17 +1068,31 @@ telegraf_repair() {
# when there is something to repair, so everyone else is left alone.
local repair=/usr/sbin/so-telegraf-repair
[[ -x "$repair" ]] || return 0
docker ps --format '{{.Names}}' | grep -qx so-postgres || return 0
local state
state=$(docker inspect -f '{{.State.Running}} {{.State.Restarting}}' so-postgres 2>/dev/null) || return 0
if [[ "$state" != "true false" ]]; then
echo " so-postgres is not running; skipping the Telegraf partition check."
FINAL_MESSAGE_QUEUE+=("WARNING: so-postgres is not running, so Telegraf partitions were not checked. If /nsm is full, follow \"Telegraf Partition Repair\" at:")
FINAL_MESSAGE_QUEUE+=(" - https://securityonion.net/docs/postgresql")
return 0
fi
echo "Checking Telegraf metric partitions."
local status=0
"$repair" --check >> "$SOUP_LOG" 2>&1 || status=$?
local out status=0
out=$("$repair" --check 2>&1) || status=$?
echo "$out" >> "$SOUP_LOG"
case "$status" in
0) echo " Telegraf partitions are healthy; nothing to repair." ;;
1) echo " Repairing stalled Telegraf partitions."
"$repair" --yes \
|| echo " warning: so-telegraf-repair failed; run it manually" >&2 ;;
*) echo " Skipping; Telegraf is not storing metrics in Postgres on this host." ;;
if ! "$repair" --yes; then
echo " warning: so-telegraf-repair failed; run it manually" >&2
FINAL_MESSAGE_QUEUE+=("WARNING: Telegraf partition repair failed; Postgres metrics will keep filling /nsm until it is repaired:")
FINAL_MESSAGE_QUEUE+=(" - sudo so-telegraf-repair")
fi ;;
2) echo " Skipping: $(tail -n 1 <<<"$out")" ;;
*) echo " warning: could not check Telegraf partitions" >&2
FINAL_MESSAGE_QUEUE+=("WARNING: Telegraf partitions could not be checked during the upgrade; run:")
FINAL_MESSAGE_QUEUE+=(" - sudo so-telegraf-repair --check") ;;
esac
}
@@ -1202,6 +1219,32 @@ add_detection_runtime_fields() {
}
### 3.4.0 End ###
### 3.5.0 Scripts ###
up_to_3.5.0() {
remove_localrules_portgroup
INSTALLEDVERSION=3.5.0
}
remove_localrules_portgroup() {
local pillar_file
for pillar_file in /opt/so/saltstack/local/pillar/firewall/{soc,adv}_firewall.sls; do
grep -qs localrules "$pillar_file" || continue
so-yaml.py get "$pillar_file" firewall 2>/dev/null | grep -q '\<localrules\>' || continue
echo "Removing the localrules portgroup from $pillar_file."
so-yaml.py remove "$pillar_file" firewall.portgroups.localrules
so-yaml.py purgelistitem "$pillar_file" firewall.role localrules
done
}
post_to_3.5.0() {
telegraf_repair
set_postversion 3.5.0
}
### 3.5.0 End ###
repo_sync() {
echo "Sync the local repo."
+13 -17
View File
@@ -10,29 +10,25 @@
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'nginx/map.jinja' import NGINXMERGED %}
{% if GLOBALS.role != 'so-fleet' %}
{% set container_config = 'so-nginx' %}
{% else %}
{# if this is an so-fleet node then we want to use the port bindings, custom bind mounts defined for fleet #}
{% set container_config = 'so-nginx-fleet-node' %}
{% endif %}
include:
- ca.trust
- nginx.ssl
- nginx.config
- nginx.sostatus
{% if GLOBALS.role != 'so-fleet' %}
{% set container_config = 'so-nginx' %}
make-rule-dir-nginx:
fleet_beats_dir_nginx:
file.directory:
- name: /nsm/rules
- user: socore
- group: socore
- recurse:
- user
- group
- name: /nsm/elastic-fleet/artifacts/beats
- makedirs: True
- show_changes: False
{% else %}
{# if this is an so-fleet node then we want to use the port bindings, custom bind mounts defined for fleet #}
{% set container_config = 'so-nginx-fleet-node' %}
{% endif %}
so-nginx:
docker_container.running:
- image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-nginx:{{ GLOBALS.so_version }}
@@ -54,9 +50,9 @@ so-nginx:
- /opt/so/log/nginx/:/var/log/nginx:rw
- /opt/so/tmp/nginx/:/var/lib/nginx:rw
- /opt/so/tmp/nginx/:/run:rw
- /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages
- /nsm/elastic-fleet/artifacts/:/opt/socore/html/artifacts
- /nsm/elastic-fleet/artifacts/beats/:/opt/socore/elastic-artifacts/artifacts/beats:ro
{% if GLOBALS.is_manager %}
- /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages
- /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro
- /etc/pki/managerssl.key:/etc/pki/nginx/server.key:ro
# ATT&CK Navigator binds
@@ -64,7 +60,6 @@ so-nginx:
- /opt/so/conf/navigator/config.json:/opt/socore/html/navigator/assets/config.json:ro
- /nsm/repo:/opt/socore/html/repo:ro
- /nsm/kernelrepo:/opt/socore/html/kernelrepo:ro
- /nsm/rules:/nsm/rules:ro
{% if NGINXMERGED.external_suricata %}
- /opt/so/rules/nids/suri:/surirules:ro
{% endif %}
@@ -106,6 +101,7 @@ so-nginx:
{% endif %}
- require:
- file: nginxconf
- file: fleet_beats_dir_nginx
{% if GLOBALS.is_manager %}
{% if NGINXMERGED.ssl.replace_cert %}
- file: managerssl_key
+16 -50
View File
@@ -4,6 +4,8 @@
{%- set role = grains.id.split('_') | last %}
{%- set influxpass = salt['pillar.get']('secrets:influx_pass') %}
{%- set influxauth = ('so:' + influxpass) | base64_encode %}
{%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %}
{%- set full_server_name = all_names | unique | join(' ') %}
worker_processes auto;
error_log /var/log/nginx/error.log;
@@ -39,52 +41,31 @@ http {
include /etc/nginx/conf.d/*.conf;
{%- if role in ['fleet'] %}
server {
listen 8443;
server_name {{ GLOBALS.hostname }};
root /opt/socore/html;
location /artifacts/ {
try_files $uri =206;
proxy_read_timeout 90;
proxy_connect_timeout 90;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Proxy "";
proxy_set_header X-Forwarded-Proto $scheme;
}
}
{%- endif %}
{%- if GLOBALS.is_manager %}
{%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %}
{%- set full_server_name = all_names | unique | join(' ') %}
server {
listen 80 default_server;
server_name _;
return 307 https://{{ GLOBALS.url_base }}$request_uri;
}
{%- endif %}
{%- if GLOBALS.is_manager or role == 'fleet' %}
server {
listen 8443;
{%- if GLOBALS.is_manager %}
server_name {{ full_server_name }};
root /opt/socore/html;
location /artifacts/ {
try_files $uri =206;
proxy_read_timeout 90;
proxy_connect_timeout 90;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Proxy "";
proxy_set_header X-Forwarded-Proto $scheme;
{%- else %}
server_name {{ GLOBALS.hostname }};
{%- endif %}
root /opt/socore/elastic-artifacts;
location /artifacts/beats/ {
try_files $uri =404;
}
location / {
return 404;
}
}
{%- endif %}
{%- if GLOBALS.is_manager %}
server {
listen 443 ssl default_server;
http2 on;
@@ -111,21 +92,6 @@ http {
{%- endif %}
{%- if GLOBALS.is_manager %}
server {
listen 7788;
server_name {{ full_server_name }};
root /nsm/rules;
location / {
allow all;
sendfile on;
sendfile_max_chunk 1m;
autoindex on;
autoindex_exact_size off;
autoindex_format html;
autoindex_localtime on;
}
}
{%- if NGINXMERGED.external_suricata %}
server {
listen 7789 ssl;
+1
View File
@@ -17,3 +17,4 @@ postgres:
log_min_messages: 'warning'
shared_preload_libraries: pg_cron
cron.database_name: postgres
cron.host: /run/postgresql
+6
View File
@@ -87,3 +87,9 @@ postgres:
global: True
advanced: True
helpLink: postgres
cron.host:
description: Where pg_cron connects to run scheduled jobs such as Telegraf partition maintenance. Must be the container's PostgreSQL socket directory, /run/postgresql, because pg_cron connects without a password and only local socket connections are trusted. Any other value makes every scheduled job fail. Changing it restarts PostgreSQL.
readonly: True
global: True
advanced: True
helpLink: postgres
+10 -1
View File
@@ -52,7 +52,7 @@ if [ -n "$FILTER_HOST" ] && ! [[ "$FILTER_HOST" =~ $HOST_RE ]]; then
fi
so_psql() {
docker exec so-postgres psql -U postgres -d so_telegraf -At -F $'\t' "$@"
docker exec -e PGOPTIONS='-c application_name=so-stats-show' so-postgres psql -U so_maint_telegraf -d so_telegraf -At -F $'\t' "$@"
}
if ! docker exec so-postgres psql -U postgres -lqt 2>/dev/null | cut -d\| -f1 | grep -qw so_telegraf; then
@@ -60,6 +60,15 @@ if ! docker exec so-postgres psql -U postgres -lqt 2>/dev/null | cut -d\| -f1 |
exit 2
fi
role_ok=$(docker exec so-postgres psql -U postgres -tAc "SELECT rolcanlogin AND NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls) FROM pg_roles WHERE rolname = 'so_maint_telegraf'" 2>/dev/null)
if [ -z "$role_ok" ]; then
echo "The so_maint_telegraf role is missing; run a highstate."
exit 1
elif [ "$role_ok" != t ]; then
echo "The so_maint_telegraf role is misconfigured; run a highstate."
exit 1
fi
table_exists() {
local table="$1"
[ -n "$(so_psql -c "SELECT 1 FROM information_schema.tables WHERE table_schema='${SCHEMA}' AND table_name='${table}' LIMIT 1;")" ]
+196 -70
View File
@@ -4,15 +4,18 @@ set -e
# Provision Telegraf state inside the so-postgres container.
# Usage: so-telegraf-postgres <subcommand>
# create_db Ensure the so_telegraf database exists.
# group_role Provision the so_telegraf group role, telegraf/partman schemas,
# pg_partman, the so_admin maintenance routines, and the hourly
# pg_cron maintenance job.
# group_role Provision the so_telegraf group role, the so_maint_telegraf
# maintenance login role, telegraf/partman schemas, pg_partman,
# the so_admin maintenance routines, and the hourly pg_cron
# maintenance job.
# user Create or update a per-minion login role granted to so_telegraf.
# Env: ROLE_USER, ROLE_PASS.
# retention Reconcile partman retention and premake on telegraf parents.
# Env: RETENTION_DAYS.
# maintenance Drain default partitions and run partman maintenance.
# check Report partition health. Non-zero if any parent is unhealthy.
# maintenance Drain default partitions and run partman maintenance, as
# so_maint_telegraf.
# check Report partition health, as so_maint_telegraf. Non-zero if any
# parent is unhealthy.
#
# A default partition holding rows for a day blocks creating that day's child,
# so maintenance drains defaults before calling partman. Use so-telegraf-repair
@@ -20,6 +23,10 @@ set -e
cmd="${1:?subcommand required}"
# Long enough for an hourly maintenance run to release its locks, short enough
# that soup and admins are not left hanging.
LOCK_TIMEOUT=60s
case "$cmd" in
create_db)
if ! docker exec so-postgres psql -U postgres -tAc \
@@ -30,7 +37,9 @@ case "$cmd" in
;;
group_role)
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -v lock_timeout="$LOCK_TIMEOUT" \
-U postgres -d so_telegraf <<'EOSQL'
SET lock_timeout = :'lock_timeout';
DO $$
BEGIN
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'so_telegraf') THEN
@@ -38,6 +47,17 @@ BEGIN
END IF;
END
$$;
DO $$
BEGIN
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'so_maint_telegraf') THEN
CREATE ROLE so_maint_telegraf;
END IF;
END
$$;
ALTER ROLE so_maint_telegraf LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD NULL;
GRANT so_telegraf TO so_maint_telegraf;
ALTER ROLE so_maint_telegraf SET role = so_telegraf;
ALTER ROLE so_maint_telegraf SET search_path = pg_catalog;
GRANT CONNECT ON DATABASE so_telegraf TO so_telegraf;
CREATE SCHEMA IF NOT EXISTS telegraf AUTHORIZATION so_telegraf;
GRANT USAGE, CREATE ON SCHEMA telegraf TO so_telegraf;
@@ -58,52 +78,90 @@ ALTER DEFAULT PRIVILEGES IN SCHEMA partman
ALTER DEFAULT PRIVILEGES IN SCHEMA partman
GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO so_telegraf;
-- pg_cron runs these as postgres, so they must not sit in a schema any
-- Telegraf role can create objects in.
-- Owned by postgres so no Telegraf role can create objects here; the routines
-- run with the caller's privileges.
CREATE SCHEMA IF NOT EXISTS so_admin AUTHORIZATION postgres;
REVOKE ALL ON SCHEMA so_admin FROM PUBLIC;
-- Resolved through pg_inherits, never by name, so a view or a table that only
-- carries the _default name is never read or written.
CREATE OR REPLACE FUNCTION so_admin.telegraf_defaults()
RETURNS TABLE (
parent_table regclass,
default_part regclass,
has_rows boolean,
default_size bigint
)
LANGUAGE plpgsql
STABLE
SECURITY INVOKER
AS $func$
DECLARE
r record;
BEGIN
FOR r IN
SELECT c.oid::regclass AS pt
FROM partman.part_config pc
JOIN pg_catalog.pg_class c ON c.oid = to_regclass(pc.parent_table)
WHERE pc.parent_table LIKE 'telegraf.%'
AND c.relkind = 'p'
ORDER BY pc.parent_table
LOOP
parent_table := r.pt;
SELECT c.oid::regclass INTO default_part
FROM pg_catalog.pg_inherits i
JOIN pg_catalog.pg_class c ON c.oid = i.inhrelid
WHERE i.inhparent = r.pt
AND c.relkind = 'r' AND c.relispartition
AND pg_catalog.pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT';
IF default_part IS NULL THEN
has_rows := false;
default_size := NULL;
ELSE
EXECUTE format('SELECT EXISTS (SELECT 1 FROM %s LIMIT 1)', default_part)
INTO has_rows;
default_size := pg_catalog.pg_total_relation_size(default_part);
END IF;
RETURN NEXT;
END LOOP;
END;
$func$;
CREATE OR REPLACE PROCEDURE so_admin.telegraf_maintenance()
LANGUAGE plpgsql
SECURITY INVOKER
-- No SET clause: it would forbid the COMMITs below, so so_maint_telegraf pins search_path instead.
AS $proc$
DECLARE
r record;
v_default text;
v_rows bigint;
r record;
BEGIN
-- No per-parent EXCEPTION handler: partition_data_proc commits internally,
-- and COMMIT is illegal while a subtransaction is active. A failing parent
-- aborts the run and the next pass retries.
FOR r IN
SELECT parent_table, retention
FROM partman.part_config
WHERE parent_table LIKE 'telegraf.%'
ORDER BY parent_table
SELECT pc.parent_table, pc.retention, d.default_part, d.default_size
FROM so_admin.telegraf_defaults() d
JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table
WHERE d.default_part IS NOT NULL AND d.has_rows
ORDER BY pc.parent_table
LOOP
v_default := format('%I.%I',
split_part(r.parent_table, '.', 1),
split_part(r.parent_table, '.', 2) || '_default');
CONTINUE WHEN to_regclass(v_default) IS NULL;
EXECUTE format('SELECT count(*) FROM %s', v_default) INTO v_rows;
CONTINUE WHEN v_rows = 0;
RAISE WARNING 'so_admin.telegraf_maintenance: % rows stranded in %, draining',
v_rows, v_default;
RAISE WARNING 'so_admin.telegraf_maintenance: % holds rows (%), draining',
r.default_part, pg_size_pretty(r.default_size);
-- Cheaper to delete expired rows than to repartition and then drop them.
IF r.retention IS NOT NULL THEN
EXECUTE format('DELETE FROM %s WHERE "time" < now() - %L::interval',
v_default, r.retention);
r.default_part, r.retention);
COMMIT;
END IF;
-- Bounded so a large backlog drains across several runs.
CALL partman.partition_data_proc(
p_parent_table := r.parent_table,
p_loop_count := 200,
p_source_table := v_default
p_loop_count := 200
);
COMMIT;
END LOOP;
@@ -112,58 +170,75 @@ BEGIN
END;
$proc$;
CREATE OR REPLACE FUNCTION so_admin.telegraf_partition_status()
DROP FUNCTION IF EXISTS so_admin.telegraf_partition_status();
CREATE FUNCTION so_admin.telegraf_partition_status()
RETURNS TABLE (
parent_table text,
oldest_child date,
newest_child date,
days_ahead int,
retention text,
default_rows bigint,
default_size text
parent_table text,
oldest_child date,
newest_child date,
days_ahead int,
retention text,
default_has_rows boolean,
default_size text
)
LANGUAGE plpgsql
STABLE
AS $func$
DECLARE
r record;
v_default regclass;
r record;
BEGIN
FOR r IN
SELECT pc.parent_table AS pt, pc.retention AS ret
FROM partman.part_config pc
WHERE pc.parent_table LIKE 'telegraf.%'
ORDER BY pc.parent_table
SELECT d.parent_table AS pt, d.has_rows, d.default_size AS size, pc.retention AS ret
FROM so_admin.telegraf_defaults() d
JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table
ORDER BY d.parent_table::text
LOOP
parent_table := r.pt;
parent_table := r.pt::text;
retention := r.ret;
SELECT min(d), max(d) INTO oldest_child, newest_child
SELECT min(x), max(x) INTO oldest_child, newest_child
FROM (
SELECT to_date(substring(c.relname FROM '_p(\d{8})$'), 'YYYYMMDD') AS d
FROM pg_inherits i
JOIN pg_class c ON c.oid = i.inhrelid
WHERE i.inhparent = r.pt::regclass
AND pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT'
SELECT to_date(substring(c.relname FROM '_p(\d{8})$'), 'YYYYMMDD') AS x
FROM pg_catalog.pg_inherits i
JOIN pg_catalog.pg_class c ON c.oid = i.inhrelid
WHERE i.inhparent = r.pt
AND pg_catalog.pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT'
) s;
days_ahead := newest_child - current_date;
v_default := to_regclass(format('%I.%I',
split_part(r.pt, '.', 1),
split_part(r.pt, '.', 2) || '_default'));
IF v_default IS NULL THEN
default_rows := NULL;
default_size := NULL;
ELSE
EXECUTE format('SELECT count(*) FROM %s', v_default::text) INTO default_rows;
default_size := pg_size_pretty(pg_total_relation_size(v_default));
END IF;
days_ahead := newest_child - current_date;
default_has_rows := r.has_rows;
default_size := pg_size_pretty(r.size);
RETURN NEXT;
END LOOP;
END;
$func$;
GRANT USAGE ON SCHEMA so_admin TO so_telegraf;
GRANT EXECUTE ON FUNCTION so_admin.telegraf_defaults() TO so_telegraf;
GRANT EXECUTE ON PROCEDURE so_admin.telegraf_maintenance() TO so_telegraf;
GRANT EXECUTE ON FUNCTION so_admin.telegraf_partition_status() TO so_telegraf;
-- Partitions created by a superuser session (an older repair) belong to
-- postgres; partman acting as so_telegraf could not drop them.
DO $$
DECLARE
r record;
BEGIN
FOR r IN
SELECT n.nspname, c.relname
FROM pg_catalog.pg_class c
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE c.relkind IN ('r', 'p')
AND (n.nspname = 'telegraf'
OR (n.nspname = 'partman' AND c.relname LIKE 'template\_telegraf\_%'))
AND c.relowner <> 'so_telegraf'::regrole
LOOP
EXECUTE format('ALTER TABLE %I.%I OWNER TO so_telegraf', r.nspname, r.relname);
END LOOP;
END
$$;
-- Drop the registration older releases left in so_telegraf.
SELECT CASE
WHEN current_setting('cron.database_name', true) IS DISTINCT FROM current_database()
@@ -175,20 +250,69 @@ DROP EXTENSION pg_cron CASCADE;
\endif
EOSQL
stalled=false
probe_failed=false
if out=$(docker exec -e PGOPTIONS="-c lock_timeout=$LOCK_TIMEOUT" so-postgres \
psql -U so_maint_telegraf -d so_telegraf -tAc \
"SELECT coalesce(bool_or(has_rows), false) FROM so_admin.telegraf_defaults()"); then
case "$out" in
t) stalled=true ;;
f) stalled=false ;;
*) probe_failed=true ;;
esac
else
probe_failed=true
fi
if $probe_failed; then
echo "ERROR: could not check Telegraf default partitions; the maintenance job was not changed." >&2
exit 1
fi
# Guarded on the live GUC so applying this before the postgresql.conf change
# has restarted the container skips rather than failing.
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d postgres <<'EOSQL'
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -v stalled="$stalled" \
-U postgres -d postgres <<'EOSQL'
SELECT CASE WHEN current_setting('cron.database_name', true) = current_database()
THEN 'true' ELSE 'false' END AS cron_here \gset
\if :cron_here
CREATE EXTENSION IF NOT EXISTS pg_cron;
-- cron.schedule_in_database is idempotent by jobname.
-- A highstate reconciles the job's definition but never its active flag; only so-telegraf-repair turns it on.
BEGIN;
SELECT coalesce(max(jobid)::text, '') AS jobid,
count(*) > 0 AS job_exists,
coalesce(bool_and(active), true) AS job_active
FROM cron.job
WHERE jobname = 'telegraf-partman-maintenance' AND username = 'so_maint_telegraf' \gset
SELECT cron.unschedule(jobid) FROM cron.job
WHERE jobname = 'telegraf-partman-maintenance' AND username <> 'so_maint_telegraf';
\if :job_exists
SELECT cron.alter_job(:jobid,
schedule := '17 * * * *',
command := 'CALL so_admin.telegraf_maintenance()',
database := 'so_telegraf');
\else
-- Registered inactive on a stalled grid so nothing drains the default
-- partitions before so-telegraf-repair truncates them.
SELECT cron.schedule_in_database(
'telegraf-partman-maintenance',
'17 * * * *',
'CALL so_admin.telegraf_maintenance()',
'so_telegraf'
'so_telegraf',
'so_maint_telegraf',
NOT :'stalled'::boolean
);
\endif
UPDATE cron.job SET nodename = current_setting('cron.host')
WHERE jobname = 'telegraf-partman-maintenance'
AND nodename IS DISTINCT FROM current_setting('cron.host');
COMMIT;
SELECT coalesce(bool_and(active), true) AS job_active
FROM cron.job
WHERE jobname = 'telegraf-partman-maintenance' AND username = 'so_maint_telegraf' \gset
\if :job_active
\else
\echo 'WARNING: the Telegraf maintenance job is inactive; run ''sudo so-telegraf-repair''.'
\endif
\else
\echo 'pg_cron metadata database is not `postgres` yet; skipping job registration.'
\endif
@@ -241,13 +365,15 @@ SET retention = :'retention_days' || ' days',
retention_keep_table = false,
premake = 7,
infinite_time_partitions = true
WHERE parent_table LIKE 'telegraf.%';
WHERE parent_table LIKE 'telegraf.%'
AND EXISTS (SELECT 1 FROM pg_catalog.pg_class c
WHERE c.oid = to_regclass(parent_table) AND c.relkind = 'p');
\endif
EOSQL
;;
maintenance)
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U so_maint_telegraf -d so_telegraf <<'EOSQL'
SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL
THEN 'true' ELSE 'false' END AS has_proc \gset
\if :has_proc
@@ -259,7 +385,7 @@ EOSQL
;;
check)
docker exec -i so-postgres psql -U postgres -d so_telegraf <<'EOSQL'
docker exec -i so-postgres psql -U so_maint_telegraf -d so_telegraf <<'EOSQL'
\pset border 2
SELECT * FROM so_admin.telegraf_partition_status();
EOSQL
@@ -278,9 +404,9 @@ LIMIT 5;
\echo 'pg_cron is not installed in this database.'
\endif
EOSQL
unhealthy=$(docker exec so-postgres psql -U postgres -d so_telegraf -tAc \
unhealthy=$(docker exec so-postgres psql -U so_maint_telegraf -d so_telegraf -tAc \
"SELECT count(*) FROM so_admin.telegraf_partition_status()
WHERE coalesce(default_rows, 0) > 0 OR coalesce(days_ahead, -1) < 1")
WHERE default_has_rows OR coalesce(days_ahead, -1) < 1")
if [ "${unhealthy:-1}" != "0" ]; then
echo "so-telegraf-postgres check: $unhealthy telegraf parent(s) unhealthy" >&2
exit 1
+225 -83
View File
@@ -11,26 +11,38 @@
# maintenance once. Healthy grids are reported and left alone.
#
# Usage: so-telegraf-repair [--check] [--yes] [--no-restart]
# --check Report health and change nothing.
# --check Report health and change nothing. Also reports a missing
# so_maint_telegraf role and a maintenance job that is missing
# or inactive.
# --yes Skip the confirmation prompt (for soup and other automation).
# --no-restart Never restart so-postgres, even if pg_cron's launcher is dead.
#
# Repair mode (without --check) registers the so_maint_telegraf role and the
# maintenance job if they are missing, pauses the job while it repairs, and
# always leaves it active afterwards. Data work runs as so_maint_telegraf.
#
# Exit status:
# 0 healthy, or repair completed
# 1 repair is needed (--check only)
# 2 cannot run here: so-postgres, so_telegraf or pg_partman is missing
# 1 repair is needed (--check), or the repair failed after it started
# 2 skipped: Telegraf output is not Postgres, or so-postgres, so_telegraf or pg_partman is missing
# 3 could not check, for example a connection error
set -e
set -eE
# Matches p_premake in telegraf.conf's create_parent template.
PREMAKE=7
JOB_NAME=telegraf-partman-maintenance
MAINT_ROLE=so_maint_telegraf
# Long enough for an hourly maintenance run to release its locks, short enough
# that soup and admins are not left hanging.
LOCK_TIMEOUT=60s
PG_OPTS="-c lock_timeout=$LOCK_TIMEOUT -c application_name=so-telegraf-repair"
CHECK_ONLY=false
ASSUME_YES=false
NO_RESTART=false
usage() { sed -n '/^# Usage:/,/^# 2 /p' "$0" | sed 's/^# \?//'; }
usage() { sed -n '/^# Usage:/,/^# 3 /p' "$0" | sed 's/^# \?//'; }
while [[ $# -gt 0 ]]; do
case "$1" in
@@ -38,75 +50,170 @@ while [[ $# -gt 0 ]]; do
--yes|-y) ASSUME_YES=true ;;
--no-restart) NO_RESTART=true ;;
-h|--help) usage; exit 0 ;;
*) echo "Unknown option: $1" >&2; usage >&2; exit 2 ;;
*) echo "Unknown option: $1" >&2; usage >&2; exit 3 ;;
esac
shift
done
skip() { echo "$*"; exit 2; }
# Strips C0 (except tab and newline), DEL and C1 characters, so names and
# messages written with Telegraf credentials can't drive the terminal.
sanitise() { LC_ALL=C tr -d '\000-\010\013-\037\177' | LC_ALL=C sed 's/\xc2[\x80-\x9f]//g'; }
psql_tg() { docker exec -i so-postgres psql -U postgres -d so_telegraf "$@"; }
psql_pg() { docker exec -i so-postgres psql -U postgres -d postgres "$@"; }
ERR_FILE=$(mktemp)
ERR_STATUS=3
ERR_PREFIX="Could not check Telegraf partitions"
job_id=
restore_active=
on_err() {
# Command substitutions inherit this trap; only the top level reports.
[[ $BASH_SUBSHELL -eq 0 ]] || exit "$ERR_STATUS"
local line
line=$(tail -n 1 "$ERR_FILE" | sanitise)
echo "$ERR_PREFIX: ${line:-command failed}" >&2
exit "$ERR_STATUS"
}
on_exit() {
if [[ -n "$restore_active" ]]; then
psql_cron -tAc "SELECT cron.alter_job($job_id, active := $restore_active)" >/dev/null || true
fi
rm -f "$ERR_FILE"
}
trap on_err ERR
trap on_exit EXIT
finish() { trap - ERR; exit "$1"; }
skip() { echo "$*"; finish 2; }
# Keeps the last stderr line for on_err while still showing all of it.
capture() {
local rc=0
: >"$ERR_FILE"
"$@" 2>"$ERR_FILE" || rc=$?
sanitise <"$ERR_FILE" >&2
return "$rc"
}
psql_tg() {
capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U "$MAINT_ROLE" -d so_telegraf "$@"
}
psql_pg() {
capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U postgres -d postgres "$@"
}
psql_cron() {
capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U postgres -d "$cron_db" "$@"
}
# A part_config row counts only while its parent is a partitioned table.
TELEGRAF_PARENTS="parent_table LIKE 'telegraf.%'
AND EXISTS (SELECT 1 FROM pg_class c
WHERE c.oid = to_regclass(parent_table) AND c.relkind = 'p')"
# query_to_xml so the per-table row counts need no helper function installed.
REPORT="
WITH parents AS (
SELECT pc.parent_table,
pc.premake,
split_part(pc.parent_table, '.', 1) AS sch,
split_part(pc.parent_table, '.', 2) AS tbl
FROM partman.part_config pc
WHERE pc.parent_table LIKE 'telegraf.%'
), children AS (
SELECT p.parent_table,
max(to_date(substring(c.relname FROM '_p(\d{8})\$'), 'YYYYMMDD')) AS newest_child
FROM parents p
JOIN pg_class pt ON pt.oid = p.parent_table::regclass
JOIN pg_inherits i ON i.inhparent = pt.oid
JOIN pg_class c ON c.oid = i.inhrelid
WHERE pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT'
GROUP BY p.parent_table
), defaults AS (
SELECT p.parent_table,
p.premake,
format('%I.%I', p.sch, p.tbl || '_default') AS default_table,
to_regclass(format('%I.%I', p.sch, p.tbl || '_default')) AS default_oid
FROM parents p
)
SELECT d.parent_table,
c.newest_child,
(c.newest_child - current_date) AS days_ahead,
d.premake,
CASE WHEN d.default_oid IS NULL THEN NULL ELSE
(xpath('/row/cnt/text()',
query_to_xml(format('SELECT count(*) AS cnt FROM %s', d.default_table),
false, true, '')))[1]::text::bigint
END AS default_rows,
CASE WHEN d.default_oid IS NULL THEN NULL
ELSE pg_size_pretty(pg_total_relation_size(d.default_oid)) END AS default_size
FROM defaults d
LEFT JOIN children c ON c.parent_table = d.parent_table
ORDER BY 1
pc.premake,
d.has_rows AS default_has_rows,
pg_size_pretty(d.default_size) AS default_size
FROM so_admin.telegraf_defaults() d
JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table
JOIN pg_class p ON p.oid = d.parent_table AND p.relkind = 'p'
LEFT JOIN LATERAL (
SELECT max(to_date(substring(ch.relname FROM '_p(\d{8})\$'), 'YYYYMMDD')) AS newest_child
FROM pg_inherits i
JOIN pg_class ch ON ch.oid = i.inhrelid
WHERE i.inhparent = d.parent_table
AND pg_get_expr(ch.relpartbound, ch.oid) <> 'DEFAULT'
) c ON true
ORDER BY d.parent_table::text
"
docker ps --format '{{.Names}}' | grep -qx so-postgres \
print_report() {
local report
report=$(psql_tg -c "$REPORT")
sanitise <<<"$report"
}
has_cron() {
[[ -n "$cron_db" ]] || return 0
psql_cron -tAc "SELECT 1 FROM pg_extension WHERE extname = 'pg_cron'"
}
lookup_job() {
local cron
cron=$(has_cron)
[[ -n "$cron" ]] || return 0
psql_cron -tA -F '|' -c \
"SELECT jobid, active FROM cron.job
WHERE jobname = '$JOB_NAME' AND username = '$MAINT_ROLE'
ORDER BY jobid LIMIT 1"
}
output=$(capture salt-call pillar.get telegraf:output --out=newline_values_only)
output=$(tr -d '[:space:]' <<<"$output" | tr '[:lower:]' '[:upper:]')
output=${output:-INFLUXDB}
case "$output" in
POSTGRES|BOTH) ;;
*) skip "Telegraf output is $(sanitise <<<"$output"); not checking Postgres partitions." ;;
esac
containers=$(capture docker ps --format '{{.Names}}')
grep -qx so-postgres <<<"$containers" \
|| skip "so-postgres is not running; nothing to repair."
docker exec so-postgres psql -U postgres -tAc \
"SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1 \
has_db=$(psql_pg -tAc "SELECT 1 FROM pg_database WHERE datname = 'so_telegraf'")
[[ -n "$has_db" ]] \
|| skip "The so_telegraf database does not exist; Telegraf is not writing to Postgres."
psql_tg -tAc "SELECT 1 FROM pg_extension WHERE extname='pg_partman'" | grep -q 1 \
# so_telegraf before the postgres state lands, postgres after.
cron_db=$(psql_pg -tAc "SELECT current_setting('cron.database_name', true)")
cron_db=$(tr -d '[:space:]' <<<"$cron_db")
role_ok=$(psql_pg -tAc \
"SELECT rolcanlogin AND NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls)
FROM pg_roles WHERE rolname = '$MAINT_ROLE'")
job=
[[ "$cron_db" == postgres ]] && job=$(lookup_job)
if [[ "$role_ok" != t ]] && $CHECK_ONLY; then
echo "Repair is needed:"
if [[ -z "$role_ok" ]]; then
echo " * maintenance role $MAINT_ROLE is missing"
else
echo " * maintenance role $MAINT_ROLE is misconfigured"
fi
echo
echo "Re-run without --check to repair."
finish 1
fi
if ! $CHECK_ONLY && [[ "$role_ok" != t || ( "$cron_db" == postgres && -z "$job" ) ]]; then
echo "Registering the maintenance role and job."
if ! /usr/sbin/so-telegraf-postgres group_role; then
echo "Could not register the maintenance job; run a highstate and try again." >&2
finish 3
fi
[[ "$cron_db" == postgres ]] && job=$(lookup_job)
fi
has_partman=$(psql_tg -tAc "SELECT 1 FROM pg_extension WHERE extname = 'pg_partman'")
[[ -n "$has_partman" ]] \
|| skip "pg_partman is not installed in so_telegraf; nothing to repair."
parents=$(psql_tg -tAc \
"SELECT count(*) FROM partman.part_config WHERE parent_table LIKE 'telegraf.%'")
stranded=$(psql_tg -tAc "SELECT coalesce(sum(default_rows), 0) FROM ( $REPORT ) t")
behind=$(psql_tg -tAc \
"SELECT count(*) FROM ( $REPORT ) t WHERE coalesce(days_ahead, -1) < 1")
"SELECT count(*) FROM partman.part_config WHERE $TELEGRAF_PARENTS")
counts=$(psql_tg -tAc \
"SELECT count(*) FILTER (WHERE default_has_rows) || '|' ||
count(*) FILTER (WHERE coalesce(days_ahead, -1) < 1)
FROM ( $REPORT ) t")
IFS='|' read -r stranded behind <<<"$counts"
# premake < 7, or infinite_time_partitions off: without the latter partman
# refuses to premake forward across the gap the stall left behind.
misconfigured=$(psql_tg -tAc \
"SELECT count(*) FROM partman.part_config
WHERE parent_table LIKE 'telegraf.%'
WHERE $TELEGRAF_PARENTS
AND (premake < $PREMAKE OR NOT infinite_time_partitions)")
# Both columns are matched because which one carries the launcher's name varies
@@ -115,16 +222,24 @@ launcher=$(psql_pg -tAc \
"SELECT count(*) FROM pg_stat_activity
WHERE backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%'")
# so_telegraf before the postgres state lands, postgres after.
cron_db=$(docker exec so-postgres psql -U postgres -tAc \
"SELECT current_setting('cron.database_name', true)" | tr -d '[:space:]')
last_run=never
if [[ -n "$cron_db" ]]; then
last_run=$(docker exec so-postgres psql -U postgres -d "$cron_db" -tAc \
"SELECT coalesce(max(d.start_time)::text, 'never')
cron_installed=$(has_cron)
if [[ -n "$cron_installed" ]]; then
run=$(psql_cron -tAc \
"SELECT d.status || ' at ' || d.start_time
FROM cron.job_run_details d JOIN cron.job j USING (jobid)
WHERE j.jobname = '$JOB_NAME'" 2>/dev/null | tr -d '[:space:]' || echo unknown)
[[ -n "$last_run" ]] || last_run=never
WHERE j.jobname = '$JOB_NAME'
ORDER BY d.start_time DESC LIMIT 1")
if [[ -n "$run" ]]; then
message=$(psql_cron -tAc \
"SELECT coalesce(d.return_message, '')
FROM cron.job_run_details d JOIN cron.job j USING (jobid)
WHERE j.jobname = '$JOB_NAME'
ORDER BY d.start_time DESC LIMIT 1")
message=$(sanitise <<<"$message")
message=${message//$'\n'/ }
last_run="$(sanitise <<<"$run"): ${message:0:200}"
fi
fi
# A grid that has never written a metric has nothing to recover, and an empty
@@ -132,43 +247,67 @@ fi
restart_needed=false
[[ "$launcher" -eq 0 && "$parents" -gt 0 && -n "$cron_db" ]] && restart_needed=true
# group_role registers the job only in the postgres database.
job_inactive=false
if [[ "$cron_db" == postgres ]]; then
[[ -z "$job" || "${job##*|}" == f ]] && job_inactive=true
fi
repair_needed=false
[[ "$stranded" -gt 0 ]] && repair_needed=true
[[ "$behind" -gt 0 ]] && repair_needed=true
[[ "$misconfigured" -gt 0 ]] && repair_needed=true
$restart_needed && repair_needed=true
$job_inactive && repair_needed=true
echo "Telegraf partition status:"
psql_tg -c "$REPORT"
echo "Rows stranded in default partitions: $stranded"
echo "pg_cron metadata database: ${cron_db:-unset}"
print_report
echo "Default partitions with rows: $stranded"
echo "pg_cron metadata database: $(sanitise <<<"${cron_db:-unset}")"
[[ "$cron_db" == postgres ]] \
|| echo "pg_cron metadata database is $(sanitise <<<"${cron_db:-unset}"); not checking the maintenance job."
echo "pg_cron launcher running: $([[ "$launcher" -gt 0 ]] && echo yes || echo no)"
echo "Last $JOB_NAME run: $last_run"
echo
if ! $repair_needed; then
echo "Telegraf partitions are healthy. Nothing to do."
exit 0
finish 0
fi
if $CHECK_ONLY; then
echo "Repair is needed:"
[[ "$stranded" -gt 0 ]] && echo " * $stranded row(s) stranded in default partitions"
[[ "$stranded" -gt 0 ]] && echo " * $stranded default partition(s) hold rows"
[[ "$behind" -gt 0 ]] && echo " * $behind parent(s) with no partition for the current window"
[[ "$misconfigured" -gt 0 ]] && echo " * $misconfigured parent(s) with stale partman settings"
$restart_needed && echo " * pg_cron's launcher is dead; maintenance is not running at all"
$job_inactive && echo " * the maintenance job is missing or inactive"
echo
echo "Re-run without --check to repair."
exit 1
finish 1
fi
if [[ "$stranded" -gt 0 ]] && ! $ASSUME_YES; then
echo "This will permanently discard the $stranded stranded row(s) above."
echo "This will permanently discard the rows in the $stranded default partition(s) above."
$restart_needed && ! $NO_RESTART && \
echo "so-postgres will also be restarted, which briefly interrupts SOC."
[[ -t 0 ]] || { echo "Not a terminal; re-run with --yes to confirm." >&2; exit 2; }
[[ -t 0 ]] || { echo "Not a terminal; re-run with --yes to confirm." >&2; finish 1; }
read -r -p "Continue? [y/N] " answer
[[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 0; }
[[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; finish 0; }
fi
ERR_STATUS=1
ERR_PREFIX="Telegraf partition repair failed"
if [[ "$cron_db" == postgres ]]; then
if [[ -z "$job" ]]; then
echo "Could not register the maintenance job; run a highstate and try again." >&2
finish 3
fi
job_id=${job%%|*}
[[ "${job##*|}" == t ]] && restore_active=true || restore_active=false
echo "Pausing the $JOB_NAME job during the repair."
psql_cron -tAc "SELECT cron.alter_job($job_id, active := false)" >/dev/null
fi
if [[ "$misconfigured" -gt 0 ]]; then
@@ -178,7 +317,7 @@ if [[ "$misconfigured" -gt 0 ]]; then
"UPDATE partman.part_config
SET premake = GREATEST(premake, $PREMAKE),
infinite_time_partitions = true
WHERE parent_table LIKE 'telegraf.%'"
WHERE $TELEGRAF_PARENTS"
fi
if [[ "$stranded" -gt 0 ]]; then
@@ -186,27 +325,24 @@ if [[ "$stranded" -gt 0 ]]; then
# One transaction: Telegraf is still writing, so a default emptied without
# its partition in place is refilled before maintenance can attach one.
psql_tg -v ON_ERROR_STOP=1 <<'EOSQL'
BEGIN;
DO $$
DECLARE
r record;
BEGIN
FOR r IN
SELECT pc.parent_table,
format('%I.%I', n.nspname, c.relname) AS default_table
FROM partman.part_config pc
JOIN pg_class p ON p.oid = pc.parent_table::regclass
JOIN pg_inherits i ON i.inhparent = p.oid
JOIN pg_class c ON c.oid = i.inhrelid
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE pc.parent_table LIKE 'telegraf.%'
AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT'
SELECT pc.parent_table, d.default_part
FROM so_admin.telegraf_defaults() d
JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table
WHERE d.default_part IS NOT NULL
LOOP
EXECUTE format('TRUNCATE TABLE %s', r.default_table);
EXECUTE format('TRUNCATE TABLE %s', r.default_part);
PERFORM partman.create_partition_time(
r.parent_table, ARRAY[date_trunc('day', now())]::timestamptz[]);
END LOOP;
END
$$;
COMMIT;
EOSQL
fi
@@ -222,7 +358,7 @@ if $restart_needed; then
sleep 2
done
docker exec so-postgres pg_isready -U postgres -q \
|| { echo "so-postgres did not come back; check 'docker logs so-postgres'." >&2; exit 1; }
|| { echo "so-postgres did not come back; check 'docker logs so-postgres'." >&2; finish 1; }
fi
fi
@@ -238,8 +374,14 @@ CALL partman.run_maintenance_proc();
\endif
EOSQL
if [[ -n "$job_id" ]]; then
psql_cron -tAc "SELECT cron.alter_job($job_id, active := true)" >/dev/null
restore_active=
echo "The $JOB_NAME job is active."
fi
echo
echo "Telegraf partition status after repair:"
psql_tg -c "$REPORT"
print_report
echo "The $JOB_NAME job runs hourly at :17. Confirm it fired with:"
echo " so-telegraf-repair --check"
+10
View File
@@ -251,6 +251,16 @@ socsensoronirepos:
- mode: 775
- makedirs: True
nsm_rules_dir:
file.directory:
- name: /nsm/rules
- user: socore
- group: socore
- recurse:
- user
- group
- show_changes: False
create_custom_local_yara_repo_template:
git.present:
- name: /nsm/rules/custom-local-repos/local-yara
+56 -7
View File
@@ -9,6 +9,7 @@
{%- set MDENGINE = GLOBALS.md_engine %}
{%- set LOGSTASH_ENABLED = LOGSTASH_MERGED.enabled %}
{%- set TG_OUT = TELEGRAFMERGED.output | upper %}
{%- set PG_ONLY = TG_OUT == 'POSTGRES' %}
{%- set PG_HOST = GLOBALS.manager_ip %}
{#- Per-minion telegraf creds live in the grid-wide telegraf/creds.sls pillar,
written by /usr/sbin/so-telegraf-cred on the manager. Each minion looks up
@@ -138,6 +139,10 @@
# AGGREGATOR PLUGINS #
###############################################################################
[[aggregators.merge]]
drop_original = true
namepass = ["system", "swap", "kafka_partition"]
###############################################################################
# INPUT PLUGINS #
###############################################################################
@@ -145,7 +150,12 @@
# Read metrics about cpu usage
[[inputs.cpu]]
## Whether to report per-cpu stats or not
{%- if PG_ONLY %}
{#- SOC and so-stats-show read only cpu-total #}
percpu = false
{%- else %}
percpu = true
{%- endif %}
## Whether to report total system cpu stats or not
totalcpu = true
## If true, collect raw CPU time metrics.
@@ -195,6 +205,11 @@
## The typical use case is for LVM volumes, to get the VG/LV name instead of
## the near-meaningless DM-0 name.
# name_templates = ["$ID_FS_LABEL","$DM_VG_NAME/$DM_LV_NAME"]
{%- if PG_ONLY %}
{#- Whole disks, md RAID included, already carry the I/O of their partitions and dm devices #}
[inputs.diskio.tagdrop]
name = ["sd*[0-9]", "vd*[0-9]", "xvd*[0-9]", "hd*[0-9]", "nvme*p[0-9]*", "mmcblk*p[0-9]*", "dm-*", "sr*", "loop*"]
{%- endif %}
# Get kernel statistics from /proc/stat
@@ -238,6 +253,12 @@
password = "{{ ES_PASS }}"
insecure_skip_verify = true
cluster_health = true
{%- if PG_ONLY %}
{#- SOC reads cluster status, indices and clusterstats_nodes ingest stats; alarms plan jvm and fs #}
cluster_health_level = "cluster"
node_stats = ["indices", "jvm", "fs"]
namedrop = ["elasticsearch_clusterstats_indices"]
{%- endif %}
{%- elif grains['role'] in ['so-searchnode'] %}
[[inputs.elasticsearch]]
servers = ["https://{{ NODEIP }}:9200"]
@@ -245,6 +266,10 @@
username = "{{ ES_USER }}"
password = "{{ ES_PASS }}"
insecure_skip_verify = true
{%- if PG_ONLY %}
{#- SOC reads indices; alarms plan jvm and fs #}
node_stats = ["indices", "jvm", "fs"]
{%- endif %}
{%- endif %}
# ## Timeout for HTTP requests to the elastic search server(s)
@@ -287,9 +312,13 @@
collect = ["pipelines"]
username = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user') }}"
password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}"
{%- if PG_ONLY %}
{#- SOC reads logstash_events only #}
namedrop = ["logstash_plugins"]
{%- endif %}
{%- endif %}
{% if grains.role in ['so-manager','so-managersearch','so-standalone','so-receiver'] and GLOBALS.pipeline == "KAFKA" -%}
{% if grains.role in ['so-manager','so-managerhype','so-managersearch','so-standalone','so-receiver'] and GLOBALS.pipeline == "KAFKA" -%}
[[inputs.jolokia2_agent]]
name_prefix= "kafka_"
urls = ["http://{{ NODEIP }}:8778/jolokia"]
@@ -346,6 +375,26 @@
interval = "60s"
{%- endif %}
{#- These report state that changes slowly; collecting them every interval only stores duplicate rows #}
{%- set SLOWSCRIPTS = [] %}
{%- for script in ['agentstatus.sh', 'os.sh', 'features.sh', 'raid.sh', 'zeekloss.sh'] %}
{%- if script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %}
{%- do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove(script) %}
{%- do SLOWSCRIPTS.append(script) %}
{%- endif %}
{%- endfor %}
{%- if SLOWSCRIPTS | length > 0 %}
[[inputs.exec]]
commands = [
{%- for script in SLOWSCRIPTS %}
["/scripts/{{script}}"]{% if not loop.last %},{% endif %}
{%- endfor %}
]
data_format = "influx"
timeout = "15s"
interval = "5m"
{%- endif %}
{%- if TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] | length > 0 %}
[[inputs.exec]]
commands = [
@@ -358,12 +407,6 @@
timeout = "15s"
{%- endif %}
{%- if salt['pillar.get']('healthcheck:enabled', False) %}
[[inputs.file]]
files = ["/host/nsm/zeek/logs/zeek_restart.log"]
data_format = "influx"
{%- endif %}
[[inputs.file]]
files = ["/etc/telegraf/node_config.json"]
name_override = "node_config"
@@ -374,6 +417,12 @@
# # Read metrics about network interface usage
[[inputs.net]]
{%- if PG_ONLY %}
{%- from 'docker/docker.map.jinja' import DOCKERMERGED %}
{#- Only virtual interfaces; every physical NIC, bond members included, is kept for SOC and alarms #}
[inputs.net.tagdrop]
interface = {{ (['veth*', 'docker*', 'lo'] + DOCKERMERGED.networks.keys() | list) | json }}
{%- endif %}
# Scripts run every 30s||TELEGRAFMERGED.config.interval - ES index script doesn't need to run as frequently
{%- if GLOBALS.is_manager or GLOBALS.role == 'so-heavynode' %}
+1 -1
View File
@@ -18,7 +18,7 @@
{% if GLOBALS.pipeline != 'REDIS' %}
{# When global pipeline is not REDIS remove redis.sh script. KAFKA metrics are collected via jolokia agent. Config in telegraf.conf #}
{% if GLOBALS.role in ['so-standalone', 'so-manager', 'so-managersearch', 'so-receiver', 'so-heavynode'] %}
{% if GLOBALS.role in ['so-standalone', 'so-manager', 'so-managerhype', 'so-managersearch', 'so-receiver', 'so-heavynode'] %}
{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('redis.sh') %}
{% endif %}
{% endif %}
+1 -1
View File
@@ -10,7 +10,7 @@
# if this script isn't already running
if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
FILES=$(ls -1x /host/nsm/strelka/unprocessed | wc -l)
FILES=$(find /host/nsm/strelka/unprocessed -maxdepth 1 -type f | wc -l)
echo "faffiles files=$FILES"
fi
+3 -2
View File
@@ -12,8 +12,9 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
CONTAINERSLOG=/var/log/somon/containers.log
# the collector rewrites this every minute; report nothing rather than repeating a stale
# file as if it were current, in case a run was skipped or the collector is wedged
MAXAGE=150
# file as if it were current, in case a run was skipped or the collector is wedged.
# Kept under two collector cycles so a 60s telegraf interval can't emit the same snapshot twice.
MAXAGE=90
if [ -r "$CONTAINERSLOG" ]; then
AGE=$(( $(date +%s) - $(stat -c %Y "$CONTAINERSLOG") ))
+31 -18
View File
@@ -11,27 +11,40 @@
if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
PREVCOUNTFILE='/tmp/eps.txt'
EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.events.in')"
STATS="$(curl -s localhost:9600/_node/stats)"
if [ ! -z "$EVENTCOUNTCURRENT" ]; then
if [ -f "$PREVCOUNTFILE" ]; then
EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE`
else
echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE
exit 0
fi
echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE
# the division by 30 is because the agent interval is 30 seconds
EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30))
if [ "$EVENTS" -lt 0 ]; then
EVENTS=0
fi
echo "consumptioneps eps=${EVENTS%%.*}"
if echo "$STATS" | jq -e '.pipelines.search' > /dev/null 2>&1; then
STAGE=search
COUNT="$(echo "$STATS" | jq '.pipelines.search.events.out')"
elif echo "$STATS" | jq -e '.pipelines.receiver' > /dev/null 2>&1; then
STAGE=receiver
COUNT="$(echo "$STATS" | jq '.pipelines.receiver.events.in')"
else
exit 0
fi
if ! [[ "$COUNT" =~ ^[0-9]+$ ]]; then
exit 0
fi
NOW=$(date +%s)
read -r PREVEPOCH PREVCOUNT 2>/dev/null < "$PREVCOUNTFILE"
echo "${NOW} ${COUNT}" > "$PREVCOUNTFILE"
if ! [[ "$PREVEPOCH" =~ ^[0-9]+$ && "$PREVCOUNT" =~ ^[0-9]+$ ]]; then
exit 0
fi
ELAPSED=$((NOW - PREVEPOCH))
DELTA=$((COUNT - PREVCOUNT))
if [ "$ELAPSED" -le 0 ] || [ "$DELTA" -lt 0 ]; then
EPS=0
else
EPS=$((DELTA / ELAPSED))
fi
echo "consumptioneps,stage=${STAGE} eps=${EPS}"
fi
exit 0
+1 -2
View File
@@ -10,8 +10,7 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
FPS_ENABLED=$(cat /var/log/sostatus/fps_enabled)
LKS_ENABLED=$(cat /var/log/sostatus/lks_enabled)
echo "features fps=$FPS_ENABLED"
echo "features lks=$LKS_ENABLED"
echo "features fps=$FPS_ENABLED,lks=$LKS_ENABLED"
fi
exit 0