Compare commits

..
1 Commits
20 changed files with 148 additions and 1498 deletions

No files matched your search

-2
View File
@@ -131,8 +131,6 @@ def beacon(config): # noqa: C901
'setting_id': setting_id,
'node_id': node_id,
})
log.info('postgres_pillar_beacon: audit_settings id=%d setting_id=%s node_id=%s',
row_id, setting_id, node_id)
if row_id > max_id:
max_id = row_id
-102
View File
@@ -656,108 +656,6 @@ set_version() {
fi
}
so_telegraf_output() {
local defaultsfile="${DEFAULT_SALT_DIR}/salt/telegraf/defaults.yaml"
local output
output=$(lookup_pillar output telegraf) || return 1
if [ -z "$output" ]; then
if [ ! -f "$defaultsfile" ]; then
echo "Could not find $defaultsfile" >&2
return 1
fi
output=$(grep -E "^ output: " "$defaultsfile" | awk -F: '{print $2}' | tr -d '[:space:]')
fi
echo "${output^^}"
}
so_telegraf_pg_check() {
local app="$1"
local databases role_ok
if ! docker_check_running postgres; then
echo "so-postgres is not running. Run $app on the manager." >&2
return 1
fi
if ! databases=$(docker exec so-postgres psql -U postgres -lqt 2>/dev/null); then
echo "Unable to list the so-postgres databases." >&2
return 1
fi
if ! echo "$databases" | cut -d\| -f1 | grep -qw so_telegraf; then
echo "Database so_telegraf not found. Is telegraf.output set to POSTGRES or BOTH?" >&2
return 1
fi
if ! role_ok=$(docker exec so-postgres psql -U postgres -tAc "SELECT rolcanlogin AND NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls) FROM pg_roles WHERE rolname = 'so_maint_telegraf'" 2>/dev/null); then
echo "Unable to query the so-postgres roles." >&2
return 1
fi
if [ -z "$role_ok" ]; then
echo "The so_maint_telegraf role is missing; run a highstate." >&2
return 1
elif [ "$role_ok" != t ]; then
echo "The so_maint_telegraf role is misconfigured; run a highstate." >&2
return 1
fi
}
so_telegraf_psql() {
local app="$1"
local sql="$2"
docker exec -e PGOPTIONS="-c application_name=${app} -c default_transaction_read_only=on -c client_min_messages=error" \
so-postgres psql -U so_maint_telegraf -d so_telegraf -v ON_ERROR_STOP=1 -At -F '|' -c "$sql"
}
# Prints one stream|current|then line per data stream, in bytes summed across hosts. current is
# each host's newest size in the past 4 hours; then is that host's newest size in the window
# [now - start, now - stop), or its current size when it has none there, so a host missing from the
# window adds no growth. Empty output means the window has no rows at all. Values over 15 digits
# (1 PB) are skipped so the sums here and in the callers stay far below the 64-bit limit; such a
# value can only come from a forged Telegraf writer.
so_es_index_size_growth() {
local app="$1"
local start="$2"
local stop="$3"
local relkind
relkind=$(so_telegraf_psql "$app" "SELECT c.relkind FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'telegraf' AND c.relname = 'elasticsearch_index_size'") || return 1
[[ "$relkind" == "r" || "$relkind" == "p" ]] || return 0
so_telegraf_psql "$app" "$(cat <<EOF
WITH sizes AS (
SELECT d.tag_id, d.time, f.key AS stream, f.value::bigint AS bytes
FROM telegraf.elasticsearch_index_size d
CROSS JOIN LATERAL pg_catalog.jsonb_each_text(d.fields) f
WHERE d.time >= now() - interval '${start}'
AND f.key ~ '^[^|"''[:space:][:cntrl:]]+$'
AND pg_catalog.strpos(f.key, pg_catalog.chr(92)) = 0
AND f.value ~ '^[0-9]{1,15}$'
),
cur AS (
SELECT DISTINCT ON (tag_id, stream) tag_id, stream, bytes
FROM sizes
WHERE time >= now() - interval '4 hours'
ORDER BY tag_id, stream, time DESC
),
past AS (
SELECT DISTINCT ON (tag_id, stream) tag_id, stream, bytes
FROM sizes
WHERE time < now() - interval '${stop}'
ORDER BY tag_id, stream, time DESC
)
SELECT c.stream, sum(c.bytes), sum(coalesce(p.bytes, c.bytes))
FROM cur c
LEFT JOIN past p USING (tag_id, stream)
WHERE EXISTS (
SELECT 1 FROM telegraf.elasticsearch_index_size
WHERE time >= now() - interval '${start}' AND time < now() - interval '${stop}'
)
GROUP BY c.stream
ORDER BY c.stream
EOF
)"
}
status () {
printf "\n=========================================================================\n$(date) | $1\n=========================================================================\n"
}
-1
View File
@@ -244,7 +244,6 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown" # so-dockerregistry logs a tag lookup miss during image copy; not tied to one docker version
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Could not index event to Elasticsearch.*\"version\" => \"9.0.8\"" # Expected during Elastic upgrade temporarily, as policies referencing older pipelines are updated
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|push result unknown.*AuthenticationError" # so-push-drainer WARNING: salt-master restarted while a push was queued; the queued state run may still complete
fi
RESULT=0
+1
View File
@@ -103,6 +103,7 @@ docker:
- 80:80
- 443:443
- 8443:8443
- 7788:7788
- 7789:7789
custom_bind_mounts: []
extra_hosts: []
@@ -5,10 +5,6 @@
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
# Reports per data stream Elasticsearch growth over the last 24 hours, 7 days and 30 days. The data
# source follows telegraf.output: Postgres in POSTGRES mode, where history is limited to
# postgres.telegraf.retention_days and the script must run on the manager; InfluxDB otherwise.
INFLUX_URL="https://localhost:8086/api/v2"
. /usr/sbin/so-common
@@ -22,6 +18,8 @@ lookup_org_id() {
echo "$response" | jq -r ".orgs[] | select(.name == \"Security Onion\").id"
}
ORG_ID=$(lookup_org_id)
run_flux_query() {
local query=$1
request "query?org=$ORG_ID" -H 'Accept:application/csv' -H 'Content-type:application/vnd.flux' -d "$query" -XPOST 2>/dev/null
@@ -41,137 +39,75 @@ bytes_to_gb() {
fi
}
influxdb_indices_growth() {
ORG_ID=$(lookup_org_id)
indexes_query='from(bucket: "telegraf/so_long_term")
|> range(start: -7d)
|> filter(fn: (r) => r._measurement == "elasticsearch_index_size")
|> distinct(column: "_field")
|> keep(columns: ["_field"])'
indexes_query='from(bucket: "telegraf/so_long_term")
|> range(start: -7d)
|> filter(fn: (r) => r._measurement == "elasticsearch_index_size")
|> distinct(column: "_field")
|> keep(columns: ["_field"])'
indexes_result=$(run_flux_query "$indexes_query")
indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | grep -v '^_field$' | sed 's/\r$//' | sort -u)
indexes_result=$(run_flux_query "$indexes_query")
indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | grep -v '^_field$' | sed 's/\r$//' | sort -u)
printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)"
printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})"
printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)"
printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})"
for index in $indexes; do
[[ -z "$index" ]] && continue
current_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -4h)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
current_result=$(run_flux_query "$current_query")
current_size=$(read_csv_result "$current_result")
current_size=${current_size:-0}
for index in $indexes; do
[[ -z "$index" ]] && continue
current_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -4h)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
current_result=$(run_flux_query "$current_query")
current_size=$(read_csv_result "$current_result")
current_size=${current_size:-0}
size_24h_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -25h, stop: -23h)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
size_24h_result=$(run_flux_query "$size_24h_query")
size_24h_ago=$(read_csv_result "$size_24h_result")
size_24h_ago=${size_24h_ago:-$current_size}
size_24h_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -25h, stop: -23h)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
size_24h_result=$(run_flux_query "$size_24h_query")
size_24h_ago=$(read_csv_result "$size_24h_result")
size_24h_ago=${size_24h_ago:-$current_size}
size_7d_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -7d8h, stop: -7d)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
size_7d_result=$(run_flux_query "$size_7d_query")
size_7d_ago=$(read_csv_result "$size_7d_result")
size_7d_ago=${size_7d_ago:-$current_size}
size_7d_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -7d8h, stop: -7d)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
size_7d_result=$(run_flux_query "$size_7d_query")
size_7d_ago=$(read_csv_result "$size_7d_result")
size_7d_ago=${size_7d_ago:-$current_size}
size_30d_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -30d8h, stop: -30d)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
size_30d_result=$(run_flux_query "$size_30d_query")
size_30d_ago=$(read_csv_result "$size_30d_result")
size_30d_ago=${size_30d_ago:-$current_size}
size_30d_query="from(bucket: \"telegraf/so_long_term\")
|> range(start: -30d8h, stop: -30d)
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|> last()
|> keep(columns: [\"_value\"])"
size_30d_result=$(run_flux_query "$size_30d_query")
size_30d_ago=$(read_csv_result "$size_30d_result")
size_30d_ago=${size_30d_ago:-$current_size}
# if an index was recently cleaned up by ilm it will result in a negative number for 'index growth'.
growth_24h=$(( current_size > size_24h_ago ? current_size - size_24h_ago : 0 ))
# if an index was recently cleaned up by ilm it will result in a negative number for 'index growth'.
growth_24h=$(( current_size > size_24h_ago ? current_size - size_24h_ago : 0 ))
growth_7d=$(( current_size > size_7d_ago ? current_size - size_7d_ago : 0 ))
growth_7d=$(( current_size > size_7d_ago ? current_size - size_7d_ago : 0 ))
growth_30d=$(( current_size > size_30d_ago ? current_size - size_30d_ago : 0 ))
growth_30d=$(( current_size > size_30d_ago ? current_size - size_30d_ago : 0 ))
growth_24h_gb=$(bytes_to_gb "$growth_24h")
growth_7d_gb=$(bytes_to_gb "$growth_7d")
growth_30d_gb=$(bytes_to_gb "$growth_30d")
growth_24h_gb=$(bytes_to_gb "$growth_24h")
growth_7d_gb=$(bytes_to_gb "$growth_7d")
growth_30d_gb=$(bytes_to_gb "$growth_30d")
# Only results for indices with atleast 1 metric above 0.00
if [[ "$growth_24h_gb" != "0.00" ]] || [[ "$growth_7d_gb" != "0.00" ]] || [[ "$growth_30d_gb" != "0.00" ]]; then
printf "%020.2f|%-50s %15s %15s %15s\n" \
"$growth_24h" \
"$index" \
"$growth_24h_gb" \
"$growth_7d_gb" \
"$growth_30d_gb"
fi
done | sort -t'|' -k1,1nr | cut -d'|' -f2-
# Only results for indices with atleast 1 metric above 0.00
if [[ "$growth_24h_gb" != "0.00" ]] || [[ "$growth_7d_gb" != "0.00" ]] || [[ "$growth_30d_gb" != "0.00" ]]; then
printf "%020.2f|%-50s %15s %15s %15s\n" \
"$growth_24h" \
"$index" \
"$growth_24h_gb" \
"$growth_7d_gb" \
"$growth_30d_gb"
fi
done | sort -t'|' -k1,1nr | cut -d'|' -f2-
}
postgres_indices_growth() {
local app="so-elasticsearch-indices-growth"
local -a starts=("25 hours" "7 days 8 hours" "30 days 8 hours")
local -a stops=("23 hours" "7 days" "30 days")
local -A current past window_has_data
local i rows stream current_size past_size growth growth_gb sort_key show
local -a columns
for i in 0 1 2; do
if ! rows=$(so_es_index_size_growth "$app" "${starts[i]}" "${stops[i]}"); then
echo "Failed to query telegraf.elasticsearch_index_size in so_telegraf." >&2
exit 1
fi
[[ -z "$rows" ]] && continue
window_has_data[$i]=1
while IFS='|' read -r stream current_size past_size; do
current[$stream]=$current_size
past[$i,$stream]=$past_size
done <<< "$rows"
done
printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)"
printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})"
for stream in "${!current[@]}"; do
columns=()
sort_key=0
show=false
for i in 0 1 2; do
if [[ -z "${window_has_data[$i]}" ]]; then
columns+=("n/a")
continue
fi
current_size=${current[$stream]}
past_size=${past[$i,$stream]-$current_size}
growth=$(( current_size > past_size ? current_size - past_size : 0 ))
growth_gb=$(bytes_to_gb "$growth")
[[ "$growth_gb" != "0.00" ]] && show=true
[[ $i -eq 0 ]] && sort_key=$growth
columns+=("$growth_gb")
done
if [[ "$show" == true ]]; then
printf "%020.2f|%-50s %15s %15s %15s\n" "$sort_key" "$stream" "${columns[@]}"
fi
done | sort -t'|' -k1,1nr | cut -d'|' -f2-
}
if ! TG_OUTPUT=$(so_telegraf_output); then
echo "Unable to read telegraf.output from pillar." >&2
exit 1
fi
if [[ "$TG_OUTPUT" == "POSTGRES" ]]; then
so_telegraf_pg_check so-elasticsearch-indices-growth || exit 1
postgres_indices_growth
else
influxdb_indices_growth
fi
@@ -58,9 +58,6 @@ Usage: $(basename "$0") [OPTIONS]
Estimate remaining days until Elasticsearch cluster reaches low watermark threshold.
Index growth history follows telegraf.output: in POSTGRES mode it is read from Postgres, limited to
postgres.telegraf.retention_days, and the script must run on the manager; otherwise from InfluxDB.
OPTIONS:
--json Output results in JSON format
-v, --verbose Show additional output
@@ -171,25 +168,11 @@ run_indices_growth() {
fi
}
fetch_postgres_total_bytes() {
local total
total=$(so_es_index_size_growth so-elasticsearch-retention-estimate "$1" "$2") || return 1
echo "$total" | awk -F'|' '{ sum += $3 } END { print sum + 0 }'
}
fetch_total_bytes() {
local start="$1"
local stop="$2"
local pg_start="$3"
local pg_stop="$4"
local range_line
if [ "$TG_OUTPUT" = "POSTGRES" ]; then
fetch_postgres_total_bytes "$pg_start" "$pg_stop"
return
fi
if [ -n "$stop" ]; then
range_line=" |> range(start: ${start}, stop: ${stop})"
else
@@ -225,13 +208,8 @@ fail() {
echo -e "\nDISCLAIMER: Script output is based on current data patterns, but are approximations solely intended to assist with getting a general ILM policy configured."
TG_OUTPUT=$(so_telegraf_output) || fail "Unable to read telegraf.output from pillar."
if [ "$TG_OUTPUT" = "POSTGRES" ]; then
pg_check_error=$(so_telegraf_pg_check so-elasticsearch-retention-estimate 2>&1) || fail "$pg_check_error"
else
ORG_ID=$(lookup_org_id)
[ -n "$ORG_ID" ] || fail "Unable to resolve InfluxDB org id"
fi
ORG_ID=$(lookup_org_id)
[ -n "$ORG_ID" ] || fail "Unable to resolve InfluxDB org id"
cluster_storage_size=0
indexed_storage_source="elasticsearch"
@@ -243,11 +221,10 @@ if [ -n "$cluster_storage_size_output" ]; then
fi
fi
# historical index sizes for growth calculation
query_error="Failed to query telegraf.elasticsearch_index_size in so_telegraf."
one_day_total=$(fetch_total_bytes "-25h" "-23h" "25 hours" "23 hours") || fail "$query_error"
seven_day_total=$(fetch_total_bytes "-7d8h" "-7d" "7 days 8 hours" "7 days") || fail "$query_error"
thirty_day_total=$(fetch_total_bytes "-30d8h" "-30d" "30 days 8 hours" "30 days") || fail "$query_error"
# historical data from influxdb for growth calculation
one_day_total=$(fetch_total_bytes "-25h" "-23h")
seven_day_total=$(fetch_total_bytes "-7d8h" "-7d")
thirty_day_total=$(fetch_total_bytes "-30d8h" "-30d")
# available historical windows (prefer 30d/7d when available, to avoid using recent 24h traffic spike as true daily ingest rate)
history_days=0
@@ -267,12 +244,7 @@ elif [ "$one_day_total" -gt 0 ]; then
historical_total=$one_day_total
fi
if [ "$TG_OUTPUT" = "POSTGRES" ]; then
history_unavailable="Historical Postgres data unavailable for growth calculation. Postgres keeps postgres.telegraf.retention_days days of history; if this is a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics."
else
history_unavailable="Historical InfluxDB data unavailable for growth calculation. If this a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics."
fi
[ "$history_days" -gt 0 ] || fail "$history_unavailable"
[ "$history_days" -gt 0 ] || fail "Historical InfluxDB data unavailable for growth calculation. If this a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics."
# Daily growth rate
growth_bytes=$(( cluster_storage_size - historical_total ))
@@ -757,7 +729,6 @@ if [ "$should_trigger_recommendations" = true ]; then
[ -n "$index" ] || continue
growth_24h_gb=$(echo "$line" | awk '{print $(NF-2)}')
[[ "$growth_24h_gb" =~ ^[0-9]+(\.[0-9]+)?$ ]] || growth_24h_gb=0
creation_date_display=""
retention_days=""
+10
View File
@@ -118,6 +118,10 @@ firewall:
tcp:
- 5601
udp: []
localrules:
tcp:
- 7788
udp: []
nginx:
tcp:
- 80
@@ -196,6 +200,7 @@ firewall:
- postgres
- elasticsearch_rest
- elasticsearch_node
- localrules
sensor:
portgroups:
- beats_5044
@@ -386,6 +391,7 @@ firewall:
- elastic_agent_control
- elastic_agent_data
- elastic_agent_update
- localrules
- sensoroni
fleet:
portgroups:
@@ -605,6 +611,7 @@ firewall:
- elastic_agent_control
- elastic_agent_data
- elastic_agent_update
- localrules
- sensoroni
fleet:
portgroups:
@@ -822,6 +829,7 @@ firewall:
- elastic_agent_control
- elastic_agent_data
- elastic_agent_update
- localrules
- sensoroni
fleet:
portgroups:
@@ -1050,6 +1058,7 @@ firewall:
- elastic_agent_update
- endgame
- strelka_frontend
- localrules
fleet:
portgroups:
- elasticsearch_rest
@@ -1509,6 +1518,7 @@ firewall:
- elasticsearch_rest
- elasticsearch_node
- elastic_agent_control
- localrules
sensor:
portgroups:
- beats_5044
+3
View File
@@ -138,6 +138,9 @@ firewall:
kibana:
tcp: *tcpsettings
udp: *udpsettings
localrules:
tcp: *tcpsettings
udp: *udpsettings
nginx:
tcp: *tcpsettings
udp: *udpsettings
+6 -245
View File
@@ -16,12 +16,9 @@ For each intent file under /opt/so/state/push_pending/*.json whose last_touch
is older than debounce_seconds, this script:
* concatenates the actions lists from every ready intent
* dedupes by (state or __highstate__, tgt, tgt_type)
* drops actions whose target matches no accepted minion (e.g. a role with no nodes yet)
* dispatches a single `salt-run state.orchestrate orch.push_batch --async`
with the deduped actions list passed as pillar kwargs
* deletes the contributed intent files on successful dispatch
* records the orchestration jid under /opt/so/state/push_dispatched and, on
later passes, looks up its result and logs success or per-minion failures
Reactor sls files (push_files, push_pillar) write intents
but never dispatch directly
@@ -33,36 +30,18 @@ import json
import logging
import logging.handlers
import os
import re
import subprocess
import sys
import time
import salt.client
import salt.config
import salt.utils.minions
PENDING_DIR = '/opt/so/state/push_pending'
LOCK_FILE = os.path.join(PENDING_DIR, '.lock')
LOG_FILE = '/opt/so/log/salt/so-push-drainer.log'
MASTER_CONFIG = '/etc/salt/master'
DISPATCHED_DIR = '/opt/so/state/push_dispatched'
HIGHSTATE_SENTINEL = '__highstate__'
RESULT_CHECK_DELAY = 30
RESULT_RECHECK_MAX = 300
RESULT_MAX_AGE = 7200
RESULT_CHECKS_PER_PASS = 5
TEXT_LIMIT = 500
# Lead-in salt puts on the comment of an orchestration step that raised.
STEP_RAISED = 'An exception occurred in this state:'
# salt-run --async reports the jid only in a log line (stderr by default).
JID_RE = re.compile(r'salt/run/(\d{20})')
def _make_logger():
logger = logging.getLogger('so-push-drainer')
@@ -117,29 +96,6 @@ def _dedupe_actions(actions):
return deduped
def _targeted(actions, log):
"""Drops actions whose target matches no accepted minion (e.g. a role with no nodes yet), which
the orchestration would otherwise report as failed with 'No minions responded'."""
try:
ckminions = salt.utils.minions.CkMinions(salt.config.master_config(MASTER_CONFIG))
except Exception:
log.exception('cannot load master config; dispatching without checking targets')
return actions
targeted = []
for action in actions:
try:
matched = ckminions.check_minions(action['tgt'], action.get('tgt_type', 'compound')).get('minions')
except Exception:
log.exception('cannot resolve tgt=%s; dispatching it anyway', action.get('tgt'))
matched = True
if matched:
targeted.append(action)
else:
log.info('skipping %s: no minions match tgt=%s',
'highstate' if action.get('highstate') else action.get('state'), action.get('tgt'))
return targeted
def _dispatch(actions, log):
pillar_arg = json.dumps({'actions': actions})
cmd = [
@@ -157,196 +113,14 @@ def _dispatch(actions, log):
except subprocess.CalledProcessError as exc:
log.error('dispatch failed (rc=%s): stdout=%s stderr=%s',
exc.returncode, exc.stdout, exc.stderr)
return None
return False
except subprocess.TimeoutExpired:
log.error('dispatch timed out after 60s')
return None
return False
except Exception:
log.exception('dispatch raised')
return None
output = '{}\n{}'.format(result.stderr or '', result.stdout or '')
match = JID_RE.search(output)
if not match:
log.warning('dispatch accepted but no jid found, result will not be tracked: output=%s',
_trim(output))
return ''
log.info('dispatch accepted: jid=%s', match.group(1))
return match.group(1)
def _trim(value):
text = value if isinstance(value, str) else json.dumps(value, default=str)
lines = [line.strip() for line in text.splitlines() if line.strip()]
if 'Traceback (most recent call last):' in text:
# Keep the lead-in and the raised exception; the frames are noise in a log line.
lines = [text.split('Traceback (most recent call last):', 1)[0].strip(), lines[-1]]
text = ' '.join(line for line in lines if line)
return text if len(text) <= TEXT_LIMIT else text[:TEXT_LIMIT] + '...'
def _unlink(path, log):
try:
os.unlink(path)
except FileNotFoundError:
pass
except OSError:
log.exception('failed to remove %s', path)
def _write_record(path, record, log):
try:
os.makedirs(DISPATCHED_DIR, exist_ok=True)
tmp_path = path + '.tmp'
with open(tmp_path, 'w') as f:
json.dump(record, f)
os.rename(tmp_path, path)
except Exception:
log.exception('failed to record dispatch %s', record.get('jid'))
def _record_dispatch(jid, actions, paths, log):
record = {'jid': jid, 'dispatched_at': time.time(), 'actions': actions, 'paths': paths}
_write_record(os.path.join(DISPATCHED_DIR, '{}.json'.format(jid)), record, log)
def _lookup_jid(jid, log):
"""Returns the job cache entry for jid, {} while it is still running, or None on error."""
cmd = ['salt-run', 'jobs.lookup_jid', jid, '--out=json']
try:
result = subprocess.run(cmd, check=True, capture_output=True, text=True, timeout=60)
return json.loads(result.stdout or '{}')
except (subprocess.CalledProcessError, subprocess.TimeoutExpired, ValueError) as exc:
log.warning('lookup of jid %s failed: %s', jid, exc)
return None
def _minion_failure(minion_ret):
if isinstance(minion_ret, dict):
return '; '.join(
'{}: {}'.format(state.get('__id__', state_key), _trim(state.get('comment', '')))
for state_key, state in minion_ret.items()
if isinstance(state, dict) and state.get('result') is False
)
# A state run rejected before it starts (e.g. another state run is in
# progress) returns a list of error strings instead of state results.
if isinstance(minion_ret, (list, str)):
return _trim(minion_ret)
return ''
def _step_failures(step):
if not isinstance(step, dict) or step.get('result') is not False:
return []
failures = ['{}: {}'.format(step.get('__id__', step.get('name')), _trim(step.get('comment', '')))]
changes = step.get('changes')
minion_rets = changes.get('ret') if isinstance(changes, dict) else None
if isinstance(minion_rets, dict):
for minion, minion_ret in minion_rets.items():
text = _minion_failure(minion_ret)
if text:
failures.append('{}: {}'.format(minion, text))
return failures
def _orch_failures(ret):
if not isinstance(ret, dict):
return [_trim(ret)]
failures = []
for job in ret.values():
if not isinstance(job, dict):
continue
job_ret = job.get('return')
data = job_ret.get('data') if isinstance(job_ret, dict) else {}
if not isinstance(data, dict):
if data:
failures.append(_trim(data))
data = {}
for steps in data.values():
if not isinstance(steps, dict):
failures.append(_trim(steps))
continue
for step in steps.values():
failures.extend(_step_failures(step))
if job.get('success') is False and not failures:
failures.append('orchestration reported failure: {}'.format(_trim(job.get('return'))))
return failures
def _failed_steps(ret):
steps = []
for job in ret.values() if isinstance(ret, dict) else []:
job_ret = job.get('return') if isinstance(job, dict) else None
data = job_ret.get('data') if isinstance(job_ret, dict) else None
for group in data.values() if isinstance(data, dict) else []:
if isinstance(group, dict):
steps.extend(step for step in group.values() if isinstance(step, dict) and step.get('result') is False)
return steps
def _result_unknown(ret):
# A step that raised (e.g. salt-master restarted while it waited on a queued state run)
# never collected the minion's return, so the state run may still have completed.
steps = _failed_steps(ret)
return bool(steps) and all(str(step.get('comment', '')).startswith(STEP_RAISED) for step in steps)
def _recheck_delay(age):
return min(RESULT_RECHECK_MAX, max(RESULT_CHECK_DELAY, age / 4))
def _check_dispatched(log, now):
due = []
for path in glob.glob(os.path.join(DISPATCHED_DIR, '*.json')):
record = _read_intent(path, log)
if not isinstance(record, dict) or not record.get('jid'):
_unlink(path, log)
continue
try:
age = now - record.get('dispatched_at', 0)
last_check = record.get('checked_at', record.get('dispatched_at', 0))
is_due = now - last_check >= _recheck_delay(age)
except TypeError:
log.warning('malformed record for jid=%s; no longer tracking', record.get('jid'))
_unlink(path, log)
continue
if is_due:
due.append((last_check, path, record, age))
# Least recently checked first, so pushes that are still running can't starve finished ones.
for _, path, record, age in sorted(due, key=lambda item: item[:2])[:RESULT_CHECKS_PER_PASS]:
jid = record['jid']
try:
if _report_result(record, age, log):
_unlink(path, log)
else:
record['checked_at'] = now
_write_record(path, record, log)
except Exception:
# Drop the record so one unreadable result can't fail every pass ahead of the drain.
log.exception('cannot evaluate result for jid=%s; no longer tracking', jid)
_unlink(path, log)
def _report_result(record, age, log):
"""Logs the outcome of a dispatched push. Returns True once the record is finished with."""
jid = record['jid']
paths = record.get('paths', [])
ret = _lookup_jid(jid, log)
if not ret:
if age > RESULT_MAX_AGE:
log.warning('no result for jid=%s after %ds, no longer tracking; paths=%s', jid, age, paths)
return True
return False
failures = _orch_failures(ret)
if failures and _result_unknown(ret):
# so-log-check excludes this message by its text.
log.warning('push result unknown jid=%s paths=%s; the orchestration lost track of the state run, which may '
'still have completed (if not, the change will be applied at the next scheduled highstate): %s',
jid, paths, ' | '.join(failures))
elif failures:
log.error('push failed jid=%s paths=%s; change will be applied at the next scheduled highstate: %s',
jid, paths, ' | '.join(failures))
else:
log.info('push succeeded jid=%s paths=%s', jid, paths)
log.info('dispatch accepted: %s', (result.stdout or '').strip())
return True
@@ -369,9 +143,6 @@ def main():
debounce_seconds = int(push.get('debounce_seconds', 30))
# Outside the lock: lookups are slow and the reactors take the same lock.
_check_dispatched(log, time.time())
os.makedirs(PENDING_DIR, exist_ok=True)
lock_fd = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR, 0o644)
try:
@@ -437,20 +208,10 @@ def main():
len(ready), len(deduped), len(combined_actions),
debounce_duration, all_paths[:20],
)
for action in deduped:
log.info('action: %s tgt=%s', 'highstate' if action.get('highstate') else action.get('state'),
action.get('tgt'))
targeted = _targeted(deduped, log)
if targeted:
jid = _dispatch(targeted, log)
if jid is None:
log.warning('dispatch failed; leaving intent files in place for retry')
return 1
if jid:
_record_dispatch(jid, targeted, all_paths[:20], log)
else:
log.info('no action matches a minion; nothing to dispatch')
if not _dispatch(deduped, log):
log.warning('dispatch failed; leaving intent files in place for retry')
return 1
for path, _ in ready:
try:
@@ -1,585 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
import importlib.util
import json
import logging
import os
import shutil
import subprocess
import sys
import tempfile
import time
import unittest
from importlib.machinery import SourceFileLoader
from unittest.mock import MagicMock, patch
HERE = os.path.dirname(os.path.abspath(__file__))
SCRIPT = os.path.join(HERE, 'so-push-drainer')
_loader = SourceFileLoader('so_push_drainer', SCRIPT)
_spec = importlib.util.spec_from_loader('so_push_drainer', _loader)
drainer = importlib.util.module_from_spec(_spec)
# salt is not installed where these tests run; the drainer only needs salt.client.Caller and CkMinions.
# Mocked only while the drainer loads: run from the repo root, 'salt' is this repo's salt/ directory.
_salt = MagicMock()
with patch.dict(sys.modules, {'salt': _salt, 'salt.client': _salt.client, 'salt.config': _salt.config,
'salt.utils': _salt.utils, 'salt.utils.minions': _salt.utils.minions}):
_loader.exec_module(drainer)
MASTER = 'manager.localdomain_master'
JID = '20260930171554259426'
ASYNC_STDERR = ('[WARNING ] Running in asynchronous mode. Results of this execution may be collected '
'by attaching to the master event bus or by examining the master job cache, if '
'configured. This execution is running under tag salt/run/{}\n'.format(JID))
CONFLICT = ('The function "state.sls" is running as PID 372218 and was started at '
'2026, Sep 30 17:15:40.466233 with jid 20260930171540466233')
def _orch_ret(steps, success=True):
return {MASTER: {
'fun': 'runner.state.orchestrate',
'jid': JID,
'return': {'data': {MASTER: steps}, 'outputter': 'highstate', 'retcode': 0 if success else 1},
'success': success,
}}
REFRESH_STEP = {
'salt_|-refresh_pillar_1_|-saltutil.refresh_pillar_|-function': {
'__id__': 'refresh_pillar_1', 'result': True,
'changes': {'ret': {'manager_standalone': True}},
'comment': 'Function ran successfully.',
},
}
CONFLICT_RET = _orch_ret(dict(REFRESH_STEP, **{
'salt_|-apply_soc_1_|-apply_soc_1_|-state': {
'__id__': 'apply_soc_1', 'result': False,
'changes': {'out': 'highstate', 'ret': {'manager_standalone': [CONFLICT]}},
'comment': 'Run failed on minions: manager_standalone',
},
}), success=False)
STATE_FAIL_RET = _orch_ret({
'salt_|-apply_hydra_1_|-apply_hydra_1_|-state': {
'__id__': 'apply_hydra_1', 'result': False,
'changes': {'out': 'highstate', 'ret': {'manager_standalone': {
'test_|-no_license_|-no_license_|-fail_without_changes': {
'__id__': 'hydra.enabled_no_license_detected', 'result': False,
'comment': 'This is a feature supported only for customers with a valid license.',
},
'file_|-hydra_conf_|-/opt/so/conf/hydra_|-managed': {'result': True, 'comment': 'ok'},
}}},
'comment': 'Run failed on minions: manager_standalone',
},
}, success=False)
RAISED = ('An exception occurred in this state: Traceback (most recent call last):\n'
' File "salt/client/__init__.py", line 1934, in pub\n'
' raise AuthenticationError(err_msg)\n'
'salt.exceptions.AuthenticationError: Authentication error occurred.\n')
RAISED_RET = _orch_ret(dict(REFRESH_STEP, **{
'salt_|-apply_hydra_1_|-apply_hydra_1_|-state': {
'__id__': 'apply_hydra_1', 'result': False, 'changes': {}, 'comment': RAISED,
},
}), success=False)
SUCCESS_RET = _orch_ret(dict(REFRESH_STEP, **{
'salt_|-apply_telegraf_1_|-apply_telegraf_1_|-state': {
'__id__': 'apply_telegraf_1', 'result': True,
'changes': {'out': 'highstate', 'ret': {'manager_standalone': {
'file_|-tgrafconf_|-/opt/so/conf/telegraf/etc/telegraf.conf_|-managed': {'result': True},
}}},
'comment': 'States ran successfully.',
},
}))
class DrainerTestCase(unittest.TestCase):
def setUp(self):
self.tmpdir = tempfile.mkdtemp()
self.pending = os.path.join(self.tmpdir, 'push_pending')
self.dispatched = os.path.join(self.tmpdir, 'push_dispatched')
os.makedirs(self.pending)
for name, value in (
('PENDING_DIR', self.pending),
('LOCK_FILE', os.path.join(self.pending, '.lock')),
('DISPATCHED_DIR', self.dispatched),
('LOG_FILE', os.path.join(self.tmpdir, 'log', 'so-push-drainer.log')),
):
patcher = patch.object(drainer, name, value)
patcher.start()
self.addCleanup(patcher.stop)
self.log = MagicMock()
def tearDown(self):
shutil.rmtree(self.tmpdir, ignore_errors=True)
def write_json(self, directory, name, data):
os.makedirs(directory, exist_ok=True)
path = os.path.join(directory, name)
with open(path, 'w') as f:
if isinstance(data, str):
f.write(data)
else:
json.dump(data, f)
return path
def logged(self, level):
return ' '.join(c.args[0] % c.args[1:] for c in getattr(self.log, level).call_args_list)
class TestHelpers(DrainerTestCase):
def test_make_logger_adds_handler_once(self):
logger = logging.getLogger('so-push-drainer')
def close_handlers():
for handler in logger.handlers:
handler.close()
logger.handlers.clear()
self.addCleanup(close_handlers)
logger.handlers.clear()
self.assertIs(drainer._make_logger(), logger)
drainer._make_logger()
self.assertEqual(len(logger.handlers), 1)
self.assertTrue(os.path.isdir(os.path.dirname(drainer.LOG_FILE)))
def test_load_push_cfg(self):
with patch.object(drainer.salt.client, 'Caller') as caller:
caller.return_value.cmd.return_value = {'enabled': False}
self.assertEqual(drainer._load_push_cfg(), {'enabled': False})
caller.return_value.cmd.return_value = 'garbage'
self.assertEqual(drainer._load_push_cfg(), {})
def test_read_intent(self):
good = self.write_json(self.pending, 'good.json', {'a': 1})
bad = self.write_json(self.pending, 'bad.json', '{nope')
self.assertEqual(drainer._read_intent(good, self.log), {'a': 1})
self.assertIsNone(drainer._read_intent(bad, self.log))
with patch('builtins.open', side_effect=RuntimeError('boom')):
self.assertIsNone(drainer._read_intent(good, self.log))
self.log.exception.assert_called_once()
def test_dedupe_actions(self):
actions = [
'not a dict',
{'state': 'soc'},
{'state': 'soc', 'tgt': '*'},
{'state': 'soc', 'tgt': '*', 'tgt_type': 'compound'},
{'highstate': True, 'tgt': '*'},
{'state': 'soc', 'tgt': 'node1', 'tgt_type': 'glob'},
]
self.assertEqual(drainer._dedupe_actions(actions), [actions[2], actions[4], actions[5]])
def test_trim(self):
self.assertEqual(drainer._trim(' text \n'), 'text')
self.assertEqual(drainer._trim(['a']), '["a"]')
self.assertEqual(drainer._trim(None), 'null')
self.assertEqual(drainer._trim('x' * 600), 'x' * drainer.TEXT_LIMIT + '...')
def test_trim_traceback(self):
comment = ('An exception occurred in this state: Traceback (most recent call last):\n'
' File "salt/client/__init__.py", line 1934, in pub\n'
' raise AuthenticationError(err_msg)\n'
'salt.exceptions.AuthenticationError: Authentication error occurred.\n')
self.assertEqual(drainer._trim(comment), 'An exception occurred in this state: '
'salt.exceptions.AuthenticationError: Authentication error occurred.')
self.assertEqual(drainer._trim('line one\n line two\n'), 'line one line two')
def test_unlink(self):
drainer._unlink(os.path.join(self.tmpdir, 'missing'), self.log)
self.log.exception.assert_not_called()
drainer._unlink(self.tmpdir, self.log)
self.log.exception.assert_called_once()
class TestDispatch(DrainerTestCase):
def run_dispatch(self, **kwargs):
with patch.object(drainer.subprocess, 'run', **kwargs) as run:
jid = drainer._dispatch([{'state': 'soc', 'tgt': '*'}], self.log)
return jid, run
def test_jid_parsed_from_stderr(self):
jid, run = self.run_dispatch(return_value=MagicMock(stdout='', stderr=ASYNC_STDERR))
self.assertEqual(jid, JID)
cmd = run.call_args[0][0]
self.assertEqual(cmd[:3], ['salt-run', 'state.orchestrate', 'orch.push_batch'])
self.assertIn('--async', cmd)
def test_jid_parsed_from_stdout(self):
jid, _ = self.run_dispatch(return_value=MagicMock(stdout=ASYNC_STDERR, stderr=None))
self.assertEqual(jid, JID)
def test_no_jid(self):
jid, _ = self.run_dispatch(return_value=MagicMock(stdout='unexpected output', stderr=None))
self.assertEqual(jid, '')
self.assertIn('output=unexpected output', self.logged('warning'))
def test_failures_return_none(self):
for exc in (subprocess.CalledProcessError(1, 'salt-run', 'out', 'err'),
subprocess.TimeoutExpired('salt-run', 60),
RuntimeError('boom')):
jid, _ = self.run_dispatch(side_effect=exc)
self.assertIsNone(jid)
def test_record_dispatch(self):
drainer._record_dispatch(JID, [{'state': 'soc'}], ['audit:soc.config.licenseKey'], self.log)
with open(os.path.join(self.dispatched, JID + '.json')) as f:
record = json.load(f)
self.assertEqual(record['jid'], JID)
self.assertEqual(record['paths'], ['audit:soc.config.licenseKey'])
self.assertIn('dispatched_at', record)
def test_record_dispatch_errors(self):
with patch.object(drainer.os, 'makedirs', side_effect=OSError('ro')):
drainer._record_dispatch(JID, [], [], self.log)
drainer._record_dispatch(JID, [object()], [], self.log)
self.assertEqual(self.log.exception.call_count, 2)
self.assertFalse(os.path.exists(os.path.join(self.dispatched, JID + '.json')))
class TestResults(DrainerTestCase):
def test_lookup_jid(self):
with patch.object(drainer.subprocess, 'run') as run:
run.return_value = MagicMock(stdout=json.dumps(SUCCESS_RET))
self.assertEqual(drainer._lookup_jid(JID, self.log), SUCCESS_RET)
self.assertEqual(run.call_args[0][0], ['salt-run', 'jobs.lookup_jid', JID, '--out=json'])
run.return_value = MagicMock(stdout='')
self.assertEqual(drainer._lookup_jid(JID, self.log), {})
run.return_value = MagicMock(stdout='not json')
self.assertIsNone(drainer._lookup_jid(JID, self.log))
run.side_effect = subprocess.TimeoutExpired('salt-run', 60)
self.assertIsNone(drainer._lookup_jid(JID, self.log))
def test_minion_failure_shapes(self):
self.assertEqual(drainer._minion_failure([CONFLICT]), json.dumps([CONFLICT]))
self.assertEqual(drainer._minion_failure('Rendering SLS failed'), 'Rendering SLS failed')
self.assertEqual(drainer._minion_failure(True), '')
self.assertEqual(drainer._minion_failure({'a': {'result': True}}), '')
def test_orch_failures_conflict(self):
failures = drainer._orch_failures(CONFLICT_RET)
self.assertEqual(failures[0], 'apply_soc_1: Run failed on minions: manager_standalone')
self.assertIn('manager_standalone', failures[1])
self.assertIn('is running as PID 372218', failures[1])
self.assertEqual(len(failures), 2)
def test_orch_failures_failed_state(self):
failures = drainer._orch_failures(STATE_FAIL_RET)
self.assertEqual(len(failures), 2)
self.assertIn('hydra.enabled_no_license_detected: This is a feature', failures[1])
self.assertNotIn('hydra_conf', failures[1])
def test_orch_failures_success(self):
self.assertEqual(drainer._orch_failures(SUCCESS_RET), [])
def test_orch_failures_render_error(self):
ret = {MASTER: {'return': {'data': {MASTER: ['Rendering SLS failed']}}, 'success': False}}
self.assertEqual(drainer._orch_failures(ret), ['["Rendering SLS failed"]'])
def test_orch_failures_not_a_dict(self):
self.assertEqual(drainer._orch_failures(['No minions matched']), ['["No minions matched"]'])
self.assertEqual(drainer._orch_failures('Runner error'), ['Runner error'])
def test_orch_failures_data_not_a_dict(self):
ret = {MASTER: {'return': {'data': ["Rendering SLS 'orch.push_batch' failed"]}, 'success': False}}
self.assertEqual(drainer._orch_failures(ret), ['["Rendering SLS \'orch.push_batch\' failed"]'])
def test_orch_failures_odd_changes(self):
for changes in ('Run failed', {'ret': ['manager_standalone']}):
ret = _orch_ret({'salt_|-apply_soc_1_|-apply_soc_1_|-state': {
'__id__': 'apply_soc_1', 'result': False, 'changes': changes, 'comment': 'Run failed on minions',
}}, success=False)
self.assertEqual(drainer._orch_failures(ret), ['apply_soc_1: Run failed on minions'])
def test_orch_failures_unparsed(self):
self.assertEqual(drainer._orch_failures({MASTER: 'odd'}), [])
ret = {MASTER: {'return': 'Exception occurred', 'success': False}}
self.assertEqual(drainer._orch_failures(ret), ['orchestration reported failure: Exception occurred'])
def test_result_unknown(self):
self.assertTrue(drainer._result_unknown(RAISED_RET))
for ret in (CONFLICT_RET, STATE_FAIL_RET, SUCCESS_RET, ['No minions matched'], {MASTER: 'odd'},
{MASTER: {'return': {'data': {MASTER: ['Rendering SLS failed']}}, 'success': False}}):
self.assertFalse(drainer._result_unknown(ret), ret)
mixed = _orch_ret(dict(RAISED_RET[MASTER]['return']['data'][MASTER],
**CONFLICT_RET[MASTER]['return']['data'][MASTER]), success=False)
self.assertFalse(drainer._result_unknown(mixed))
def record(self, jid, age, now):
return self.write_json(self.dispatched, jid + '.json', {
'jid': jid, 'dispatched_at': now - age, 'actions': [], 'paths': ['audit:' + jid],
})
def test_check_dispatched(self):
now = time.time()
results = {
'1_failed': CONFLICT_RET,
'2_ok': SUCCESS_RET,
'3_pending': {},
'4_expired': None,
'6_unknown': RAISED_RET,
}
young = self.record('0_young', 5, now)
paths = {jid: self.record(jid, 60, now) for jid in results}
paths['4_expired'] = self.record('4_expired', drainer.RESULT_MAX_AGE + 1, now)
bad = self.write_json(self.dispatched, '5_bad.json', '{nope')
with patch.object(drainer, '_lookup_jid', side_effect=lambda jid, log: results[jid]):
drainer._check_dispatched(self.log, now)
self.assertTrue(os.path.exists(young))
with open(paths['3_pending']) as f:
self.assertEqual(json.load(f)['checked_at'], now)
for jid in ('1_failed', '2_ok', '4_expired', '6_unknown'):
self.assertFalse(os.path.exists(paths[jid]), jid)
self.assertFalse(os.path.exists(bad))
self.assertIn('push failed jid=1_failed', self.logged('error'))
self.assertIn('is running as PID 372218', self.logged('error'))
self.assertIn('push succeeded jid=2_ok', self.logged('info'))
self.assertIn('no result for jid=4_expired', self.logged('warning'))
self.assertIn('push result unknown jid=6_unknown', self.logged('warning'))
self.assertIn('AuthenticationError: Authentication error occurred.', self.logged('warning'))
self.assertNotIn('6_unknown', self.logged('error'))
def test_check_dispatched_survives_bad_result(self):
now = time.time()
bad = self.record('1_bad', 60, now)
good = self.record('2_ok', 60, now)
def orch_failures(ret):
if ret == 'boom':
raise ValueError('unexpected shape')
return []
with patch.object(drainer, '_lookup_jid', side_effect=lambda jid, log: 'boom' if jid == '1_bad' else SUCCESS_RET), \
patch.object(drainer, '_orch_failures', side_effect=orch_failures):
drainer._check_dispatched(self.log, now)
self.assertFalse(os.path.exists(bad))
self.assertFalse(os.path.exists(good))
self.log.exception.assert_called_once()
self.assertIn('jid=1_bad', self.log.exception.call_args[0][0] % self.log.exception.call_args[0][1:])
self.assertIn('push succeeded jid=2_ok', self.logged('info'))
def test_check_dispatched_drops_malformed_record(self):
now = time.time()
bad_dispatched = self.write_json(self.dispatched, '1_bad.json', {'jid': '1_bad', 'dispatched_at': 'soon'})
bad_checked = self.write_json(self.dispatched, '2_bad.json', {
'jid': '2_bad', 'dispatched_at': now - 60, 'checked_at': None,
})
good = self.record('3_ok', 60, now)
with patch.object(drainer, '_lookup_jid', return_value=SUCCESS_RET) as lookup:
drainer._check_dispatched(self.log, now)
for path in (bad_dispatched, bad_checked, good):
self.assertFalse(os.path.exists(path), path)
self.assertEqual([c.args[0] for c in lookup.call_args_list], ['3_ok'])
self.assertIn('malformed record for jid=1_bad', self.logged('warning'))
self.assertIn('malformed record for jid=2_bad', self.logged('warning'))
self.assertIn('push succeeded jid=3_ok', self.logged('info'))
def test_recheck_delay(self):
self.assertEqual(drainer._recheck_delay(10), drainer.RESULT_CHECK_DELAY)
self.assertEqual(drainer._recheck_delay(400), 100)
self.assertEqual(drainer._recheck_delay(drainer.RESULT_MAX_AGE), drainer.RESULT_RECHECK_MAX)
def test_check_dispatched_limit_rotates(self):
now = time.time()
limit = drainer.RESULT_CHECKS_PER_PASS
jids = ['{:02d}'.format(i) for i in range(limit + 2)]
for jid in jids:
self.record(jid, 60, now)
with patch.object(drainer, '_lookup_jid', return_value={}) as lookup:
drainer._check_dispatched(self.log, now)
self.assertEqual([c.args[0] for c in lookup.call_args_list], jids[:limit])
lookup.reset_mock()
drainer._check_dispatched(self.log, now + 40)
self.assertEqual([c.args[0] for c in lookup.call_args_list], jids[limit:] + jids[:limit - 2])
def test_check_dispatched_not_blocked_by_running(self):
now = time.time()
for i in range(drainer.RESULT_CHECKS_PER_PASS):
self.record('1_running{}'.format(i), 600, now)
done = self.record('2_done', 60, now)
def lookup(jid, log):
return SUCCESS_RET if jid == '2_done' else {}
with patch.object(drainer, '_lookup_jid', side_effect=lookup) as lookup_jid:
drainer._check_dispatched(self.log, now)
self.assertTrue(os.path.exists(done))
lookup_jid.reset_mock()
drainer._check_dispatched(self.log, now + 15)
self.assertEqual([c.args[0] for c in lookup_jid.call_args_list], ['2_done'])
self.assertFalse(os.path.exists(done))
self.assertIn('push succeeded jid=2_done', self.logged('info'))
class TestTargeted(DrainerTestCase):
def setUp(self):
super().setUp()
self.ckminions = MagicMock()
patcher = patch.object(drainer.salt.utils.minions, 'CkMinions', return_value=self.ckminions)
self.CkMinions = patcher.start()
self.addCleanup(patcher.stop)
def test_drops_unmatched(self):
matches = {'G@role:so-sensor': ['sensor_sensor'], 'manager_manager': ['manager_manager']}
self.ckminions.check_minions.side_effect = lambda tgt, tgt_type: {'minions': matches.get(tgt, [])}
actions = [
{'state': 'strelka', 'tgt': 'G@role:so-sensor'},
{'state': 'suricata', 'tgt': 'G@role:so-heavynode'},
{'highstate': True, 'tgt': 'G@role:so-idh'},
{'state': 'soc', 'tgt': 'manager_manager', 'tgt_type': 'list'},
]
self.assertEqual(drainer._targeted(actions, self.log), [actions[0], actions[3]])
self.CkMinions.assert_called_once()
self.assertEqual([c.args for c in self.ckminions.check_minions.call_args_list], [
('G@role:so-sensor', 'compound'), ('G@role:so-heavynode', 'compound'),
('G@role:so-idh', 'compound'), ('manager_manager', 'list'),
])
info = self.logged('info')
self.assertIn('skipping suricata: no minions match tgt=G@role:so-heavynode', info)
self.assertIn('skipping highstate: no minions match tgt=G@role:so-idh', info)
self.assertNotIn('strelka', info)
def test_master_config_error_keeps_actions(self):
actions = [{'state': 'soc', 'tgt': '*'}]
with patch.object(drainer.salt.config, 'master_config', side_effect=OSError('no config')):
self.assertEqual(drainer._targeted(actions, self.log), actions)
self.log.exception.assert_called_once()
self.CkMinions.assert_not_called()
def test_check_error_keeps_action(self):
actions = [{'state': 'soc', 'tgt': 'G@role:'}]
self.ckminions.check_minions.side_effect = RuntimeError('bad target')
self.assertEqual(drainer._targeted(actions, self.log), actions)
self.log.exception.assert_called_once()
class TestMain(DrainerTestCase):
def setUp(self):
super().setUp()
self.cfg = {'enabled': True, 'debounce_seconds': 30}
for name, kwargs in (
('_make_logger', {'return_value': self.log}),
('_load_push_cfg', {'side_effect': lambda: self.cfg}),
('_check_dispatched', {}),
('_targeted', {'side_effect': lambda actions, log: actions}),
):
patcher = patch.object(drainer, name, **kwargs)
setattr(self, name, patcher.start())
self.addCleanup(patcher.stop)
def intent(self, name, age=60, actions=None, paths=None):
now = time.time()
return self.write_json(self.pending, name, {
'first_touch': now - age - 5, 'last_touch': now - age,
'actions': [{'state': 'soc', 'tgt': '*'}] if actions is None else actions,
'paths': paths or ['audit:soc.config.licenseKey'],
})
def test_no_pending_dir(self):
shutil.rmtree(self.pending)
self.assertEqual(drainer.main(), 0)
self._load_push_cfg.assert_not_called()
def test_cfg_error(self):
self._load_push_cfg.side_effect = RuntimeError('no salt')
self.assertEqual(drainer.main(), 1)
def test_disabled(self):
self.cfg['enabled'] = False
self.assertEqual(drainer.main(), 0)
self._check_dispatched.assert_not_called()
def test_no_intents_still_checks_results(self):
self.assertEqual(drainer.main(), 0)
self._check_dispatched.assert_called_once()
def test_debounce_and_broken(self):
young = self.intent('young.json', age=1)
broken = self.write_json(self.pending, 'broken.json', '{nope')
with patch.object(drainer, '_dispatch') as dispatch:
self.assertEqual(drainer.main(), 0)
dispatch.assert_not_called()
self.assertTrue(os.path.exists(young))
self.assertFalse(os.path.exists(broken))
def test_broken_unlink_error_ignored(self):
self.write_json(self.pending, 'broken.json', '{nope')
with patch.object(drainer.os, 'unlink', side_effect=OSError('busy')):
self.assertEqual(drainer.main(), 0)
def test_no_usable_actions(self):
path = self.intent('empty.json', actions=[{'state': 'soc'}])
self.assertEqual(drainer.main(), 0)
self.assertFalse(os.path.exists(path))
self.intent('empty.json', actions=[{'state': 'soc'}])
with patch.object(drainer.os, 'unlink', side_effect=OSError('busy')):
self.assertEqual(drainer.main(), 0)
def test_dispatch_failure_keeps_intents(self):
path = self.intent('pillar_soc.json')
with patch.object(drainer, '_dispatch', return_value=None):
self.assertEqual(drainer.main(), 1)
self.assertTrue(os.path.exists(path))
def test_dispatch_records_jid(self):
soc = self.intent('pillar_soc.json')
hs = self.intent('pillar_global.json', actions=[{'highstate': True, 'tgt': '*'}], paths=['audit:global.x'])
with patch.object(drainer, '_dispatch', return_value=JID) as dispatch, \
patch.object(drainer, '_record_dispatch') as record:
self.assertEqual(drainer.main(), 0)
self.assertEqual(len(dispatch.call_args[0][0]), 2)
record.assert_called_once()
self.assertEqual(record.call_args[0][0], JID)
self.assertEqual(sorted(record.call_args[0][2]), ['audit:global.x', 'audit:soc.config.licenseKey'])
self.assertFalse(os.path.exists(soc))
self.assertFalse(os.path.exists(hs))
self.assertIn('action: highstate tgt=*', self.logged('info'))
def test_dispatch_without_jid_not_recorded(self):
self.intent('pillar_soc.json')
with patch.object(drainer, '_dispatch', return_value=''), \
patch.object(drainer, '_record_dispatch') as record, \
patch.object(drainer.os, 'unlink', side_effect=OSError('busy')):
self.assertEqual(drainer.main(), 0)
record.assert_not_called()
self.log.exception.assert_called_once()
def test_untargeted_not_dispatched(self):
path = self.intent('strelka.json', actions=[{'state': 'strelka', 'tgt': 'G@role:so-sensor'}])
self._targeted.side_effect = None
self._targeted.return_value = []
with patch.object(drainer, '_dispatch') as dispatch, \
patch.object(drainer, '_record_dispatch') as record:
self.assertEqual(drainer.main(), 0)
dispatch.assert_not_called()
record.assert_not_called()
self.assertFalse(os.path.exists(path))
self.assertIn('no action matches a minion; nothing to dispatch', self.logged('info'))
def test_dispatches_only_targeted(self):
kept = {'state': 'soc', 'tgt': '*'}
self.intent('mixed.json', actions=[kept, {'state': 'strelka', 'tgt': 'G@role:so-sensor'}])
self._targeted.side_effect = None
self._targeted.return_value = [kept]
with patch.object(drainer, '_dispatch', return_value=JID) as dispatch, \
patch.object(drainer, '_record_dispatch') as record:
self.assertEqual(drainer.main(), 0)
self.assertEqual(dispatch.call_args[0][0], [kept])
self.assertEqual(record.call_args[0][1], [kept])
if __name__ == '__main__':
unittest.main()
-33
View File
@@ -20,7 +20,6 @@ def showUsage(args):
print(' append - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
print(' appendlistobject - Append an object to a yaml list key. Requires KEY and JSON_OBJECT args.', file=sys.stderr)
print(' removelistitem - Remove a list item from a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
print(' purgelistitem - Remove every occurrence of a list item from all lists nested under a yaml key, if it exists. Requires KEY and LISTITEM args.', file=sys.stderr)
print(' replacelistobject - Replace a list object based on a condition. Requires KEY, CONDITION_FIELD, CONDITION_VALUE, and JSON_OBJECT args.', file=sys.stderr)
print(' add - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr)
print(' get [-r] - Displays (to stdout) the value stored in the given key. Requires KEY arg. Use -r for raw output without YAML formatting.', file=sys.stderr)
@@ -200,33 +199,6 @@ def removelistitem(args):
return 0
def purgeListItem(content, listItem):
if isinstance(content, dict):
for value in content.values():
purgeListItem(value, listItem)
elif isinstance(content, list):
content[:] = [item for item in content if item != listItem]
for item in content:
purgeListItem(item, listItem)
def purgelistitem(args):
if len(args) != 3:
print('Missing filename, key arg, or list item to purge', file=sys.stderr)
showUsage(None)
return 1
filename = args[0]
key = args[1]
listItem = args[2]
content = loadYaml(filename)
purgeListItem(getKeyValue(content, key), convertType(listItem))
writeYaml(filename, content)
return 0
def replaceListObjectByCondition(content, key, conditionField, conditionValue, newObject):
pieces = key.split(".", 1)
if len(pieces) > 1:
@@ -319,8 +291,6 @@ def add(args):
def removeKey(content, key):
if not isinstance(content, dict):
return
pieces = key.split(".", 1)
if len(pieces) > 1:
if pieces[0] in content:
@@ -364,8 +334,6 @@ def replace(args):
def getKeyValue(content, key):
if not isinstance(content, dict):
return None
pieces = key.split(".", 1)
if len(pieces) > 1 and pieces[0] in content:
return getKeyValue(content[pieces[0]], pieces[1])
@@ -417,7 +385,6 @@ def main():
"append": append,
"appendlistobject": appendlistobject,
"removelistitem": removelistitem,
"purgelistitem": purgelistitem,
"replacelistobject": replacelistobject,
"get": get,
"remove": remove,
-278
View File
@@ -128,22 +128,6 @@ class TestRemove(unittest.TestCase):
sysmock.assert_called_once_with(1)
self.assertIn("Missing filename or key arg\n", mock_stderr.getvalue())
def test_remove_null_parent(self):
filename = "/tmp/so-yaml_test-remove.yaml"
file = open(filename, "w")
file.write("key1: null\n")
file.close()
result = soyaml.remove([filename, "key1.child1.deep1"])
self.assertEqual(result, 0)
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1: null\n"
self.assertEqual(actual, expected)
def test_append_missing_arg(self):
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -581,19 +565,6 @@ class TestRemove(unittest.TestCase):
self.assertEqual(result, 2)
self.assertEqual("", mock_stdout.getvalue())
def test_get_null_parent(self):
with patch('sys.stdout', new=StringIO()) as mock_stdout:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
filename = "/tmp/so-yaml_test-get.yaml"
file = open(filename, "w")
file.write("key1: null\n")
file.close()
result = soyaml.get([filename, "key1.child1"])
self.assertEqual(result, 2)
self.assertEqual("", mock_stdout.getvalue())
self.assertIn("Key 'key1.child1' not found by so-yaml.py", mock_stderr.getvalue())
def test_get_empty_file(self):
with patch('sys.stdout', new=StringIO()) as mock_stdout:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -738,255 +709,6 @@ class TestRemoveListItem(unittest.TestCase):
self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())
class TestPurgeListItem(unittest.TestCase):
def test_purgelistitem_missing_arg(self):
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
sys.argv = ["cmd", "help"]
soyaml.purgelistitem(["file", "key"])
sysmock.assert_called()
self.assertIn("Missing filename, key arg, or list item to purge", mock_stderr.getvalue())
def test_purgelistitem(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: [a,b,c], key2: false}")
file.close()
soyaml.purgelistitem([filename, "key1", "b"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- a\n- c\nkey2: false\n"
self.assertEqual(actual, expected)
def test_purgelistitem_nested_lists(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: { child1: [a,b,c], child2: { deep1: [b,d], deep2: 45 } }, key2: [b,e]}")
file.close()
soyaml.purgelistitem([filename, "key1", "b"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n child1:\n - a\n - c\n child2:\n deep1:\n - d\n deep2: 45\nkey2:\n- b\n- e\n"
self.assertEqual(actual, expected)
def test_purgelistitem_duplicates(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: [b,a,b,c,b]}")
file.close()
soyaml.purgelistitem([filename, "key1", "b"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- a\n- c\n"
self.assertEqual(actual, expected)
def test_purgelistitem_list_of_objects(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: [{name: item1, values: [a,b]}, b, {name: item2, values: [b,c]}]}")
file.close()
soyaml.purgelistitem([filename, "key1", "b"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- name: item1\n values:\n - a\n- name: item2\n values:\n - c\n"
self.assertEqual(actual, expected)
def test_purgelistitem_typed(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: [1, '1', 2]}")
file.close()
soyaml.purgelistitem([filename, "key1", "1"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- '1'\n- 2\n"
self.assertEqual(actual, expected)
def test_purgelistitem_item_not_in_list(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: { child1: [a,b,c] }}")
file.close()
soyaml.purgelistitem([filename, "key1", "d"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n child1:\n - a\n - b\n - c\n"
self.assertEqual(actual, expected)
def test_purgelistitem_key_noexist(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: [a,b,c]}")
file.close()
result = soyaml.purgelistitem([filename, "key2.child1", "b"])
self.assertEqual(result, 0)
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- a\n- b\n- c\n"
self.assertEqual(actual, expected)
def test_purgelistitem_key_noexist_deep(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: { child1: [a,b,c] }}")
file.close()
result = soyaml.purgelistitem([filename, "key1.child2.deep1", "b"])
self.assertEqual(result, 0)
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n child1:\n - a\n - b\n - c\n"
self.assertEqual(actual, expected)
def test_purgelistitem_scalar_and_null_values(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: { child1: b, child2: null, child3: [b] }}")
file.close()
soyaml.purgelistitem([filename, "key1", "b"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n child1: b\n child2: null\n child3: []\n"
self.assertEqual(actual, expected)
def test_purgelistitem_lists_of_lists(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: [[a,b], [b,[b,c]], b]}")
file.close()
soyaml.purgelistitem([filename, "key1", "b"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- - a\n- - - c\n"
self.assertEqual(actual, expected)
def test_purgelistitem_file_object(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: [{name: x}, a, {name: y}, {name: x}]}")
file.close()
itemfile = "/tmp/so-yaml_test-purgelistitem-item.yaml"
file = open(itemfile, "w")
file.write("name: x\n")
file.close()
soyaml.purgelistitem([filename, "key1", "file:" + itemfile])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- a\n- name: y\n"
self.assertEqual(actual, expected)
def test_purgelistitem_main(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: { child1: [a,b,c] }, key2: [b]}")
file.close()
with patch('sys.exit', new=MagicMock()) as sysmock:
sys.argv = ["cmd", "purgelistitem", filename, "key1", "b"]
soyaml.main()
sysmock.assert_called_once_with(0)
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n child1:\n - a\n - c\nkey2:\n- b\n"
self.assertEqual(actual, expected)
def test_purgelistitem_with_remove(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: { child1: { deep1: [a] }, child2: [b] }, key2: { group1: { items: [x,child1,y] }, group2: { items: [child1] } }}")
file.close()
soyaml.remove([filename, "key1.child1"])
soyaml.purgelistitem([filename, "key2", "child1"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n child2:\n - b\nkey2:\n group1:\n items:\n - x\n - y\n group2:\n items: []\n"
self.assertEqual(actual, expected)
def test_purgelistitem_null_parent(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("key1: null\n")
file.close()
result = soyaml.purgelistitem([filename, "key1.child1", "b"])
self.assertEqual(result, 0)
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1: null\n"
self.assertEqual(actual, expected)
def test_purgelistitem_non_dict_parent(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("key1:\n- a\n- b\n")
file.close()
result = soyaml.purgelistitem([filename, "key1.child1", "b"])
self.assertEqual(result, 0)
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- a\n- b\n"
self.assertEqual(actual, expected)
class TestAppendListObject(unittest.TestCase):
def test_appendlistobject_missing_arg(self):
+13 -19
View File
@@ -5,6 +5,7 @@
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
. /usr/sbin/so-common
. /usr/sbin/so-image-common
@@ -16,7 +17,6 @@ INSTALLEDVERSION=$(cat /etc/soversion)
# not finish its post-upgrade steps; its contents are the resume point. It is read
# here before preupgrade_changes mutates INSTALLEDVERSION and before any highstate
# stamps /etc/soversion from the pillar.
POSTVERSION_FILE=/etc/sopostversion
if [ -f "$POSTVERSION_FILE" ]; then
@@ -55,6 +55,7 @@ fi
# used to display messages to the user at the end of soup
declare -a FINAL_MESSAGE_QUEUE=()
SOUP_ERR_CONTEXT=
TELEGRAF_REPAIR_RAN=
check_err() {
@@ -528,7 +529,6 @@ preupgrade_changes() {
[[ "$INSTALLEDVERSION" == "3.1.0" ]] && up_to_3.2.0
[[ "$INSTALLEDVERSION" == "3.2.0" ]] && up_to_3.3.0
[[ "$INSTALLEDVERSION" == "3.3.0" ]] && up_to_3.4.0
[[ "$INSTALLEDVERSION" == "3.4.0" ]] && up_to_3.5.0
true
}
@@ -1064,6 +1064,17 @@ up_to_3.3.0() {
}
telegraf_repair() {
# post_to_3.3.0 and post_to_3.5.0 both call this; run it once per soup.
[[ -n "$TELEGRAF_REPAIR_RAN" ]] && return 0
TELEGRAF_REPAIR_RAN=true
local output
output=$(salt-call pillar.get telegraf:output --out=newline_values_only)
output=$(tr -d '[:space:]' <<<"$output" | tr '[:lower:]' '[:upper:]')
output=${output:-INFLUXDB}
case "$output" in
POSTGRES|BOTH) ;;
*) return 0 ;;
esac
# Only grids whose Telegraf partitions stalled need this; --check exits 1
# when there is something to repair, so everyone else is left alone.
local repair=/usr/sbin/so-telegraf-repair
@@ -1220,23 +1231,6 @@ add_detection_runtime_fields() {
### 3.4.0 End ###
### 3.5.0 Scripts ###
up_to_3.5.0() {
remove_localrules_portgroup
INSTALLEDVERSION=3.5.0
}
remove_localrules_portgroup() {
local pillar_file
for pillar_file in /opt/so/saltstack/local/pillar/firewall/{soc,adv}_firewall.sls; do
grep -qs localrules "$pillar_file" || continue
so-yaml.py get "$pillar_file" firewall 2>/dev/null | grep -q '\<localrules\>' || continue
echo "Removing the localrules portgroup from $pillar_file."
so-yaml.py remove "$pillar_file" firewall.portgroups.localrules
so-yaml.py purgelistitem "$pillar_file" firewall.role localrules
done
}
post_to_3.5.0() {
telegraf_repair
+16 -12
View File
@@ -10,25 +10,29 @@
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'nginx/map.jinja' import NGINXMERGED %}
{% if GLOBALS.role != 'so-fleet' %}
{% set container_config = 'so-nginx' %}
{% else %}
{# if this is an so-fleet node then we want to use the port bindings, custom bind mounts defined for fleet #}
{% set container_config = 'so-nginx-fleet-node' %}
{% endif %}
include:
- ca.trust
- nginx.ssl
- nginx.config
- nginx.sostatus
fleet_beats_dir_nginx:
{% if GLOBALS.role != 'so-fleet' %}
{% set container_config = 'so-nginx' %}
make-rule-dir-nginx:
file.directory:
- name: /nsm/elastic-fleet/artifacts/beats
- makedirs: True
- name: /nsm/rules
- user: socore
- group: socore
- recurse:
- user
- group
- show_changes: False
{% else %}
{# if this is an so-fleet node then we want to use the port bindings, custom bind mounts defined for fleet #}
{% set container_config = 'so-nginx-fleet-node' %}
{% endif %}
so-nginx:
docker_container.running:
- image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-nginx:{{ GLOBALS.so_version }}
@@ -50,7 +54,7 @@ so-nginx:
- /opt/so/log/nginx/:/var/log/nginx:rw
- /opt/so/tmp/nginx/:/var/lib/nginx:rw
- /opt/so/tmp/nginx/:/run:rw
- /nsm/elastic-fleet/artifacts/beats/:/opt/socore/elastic-artifacts/artifacts/beats:ro
- /nsm/elastic-fleet/artifacts/:/opt/socore/elastic-artifacts/artifacts:ro
{% if GLOBALS.is_manager %}
- /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages
- /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro
@@ -60,6 +64,7 @@ so-nginx:
- /opt/so/conf/navigator/config.json:/opt/socore/html/navigator/assets/config.json:ro
- /nsm/repo:/opt/socore/html/repo:ro
- /nsm/kernelrepo:/opt/socore/html/kernelrepo:ro
- /nsm/rules:/nsm/rules:ro
{% if NGINXMERGED.external_suricata %}
- /opt/so/rules/nids/suri:/surirules:ro
{% endif %}
@@ -101,7 +106,6 @@ so-nginx:
{% endif %}
- require:
- file: nginxconf
- file: fleet_beats_dir_nginx
{% if GLOBALS.is_manager %}
{% if NGINXMERGED.ssl.replace_cert %}
- file: managerssl_key
+16 -1
View File
@@ -57,7 +57,7 @@ http {
server_name {{ GLOBALS.hostname }};
{%- endif %}
root /opt/socore/elastic-artifacts;
location /artifacts/beats/ {
location /artifacts/ {
try_files $uri =404;
}
location / {
@@ -92,6 +92,21 @@ http {
{%- endif %}
{%- if GLOBALS.is_manager %}
server {
listen 7788;
server_name {{ full_server_name }};
root /nsm/rules;
location / {
allow all;
sendfile on;
sendfile_max_chunk 1m;
autoindex on;
autoindex_exact_size off;
autoindex_format html;
autoindex_localtime on;
}
}
{%- if NGINXMERGED.external_suricata %}
server {
listen 7789 ssl;
+4 -4
View File
@@ -3,8 +3,6 @@
{% set BATCH = AUTOAPPLY.batch %}
{% set BATCH_WAIT = AUTOAPPLY.batch_wait %}
{# queue must be a top-level salt.state arg (kwarg is ignored); an int is max_queue and still fails on conflict #}
{% for action in actions %}
{% if action.get('highstate') %}
apply_highstate_{{ loop.index }}:
@@ -14,7 +12,8 @@ apply_highstate_{{ loop.index }}:
- highstate: True
- batch: {{ action.get('batch', BATCH) }}
- batch_wait: {{ action.get('batch_wait', BATCH_WAIT) }}
- queue: True
- kwarg:
queue: 2
{% else %}
refresh_pillar_{{ loop.index }}:
salt.function:
@@ -30,7 +29,8 @@ apply_{{ action.state | replace('.', '_') }}_{{ loop.index }}:
- {{ action.state }}
- batch: {{ action.get('batch', BATCH) }}
- batch_wait: {{ action.get('batch_wait', BATCH_WAIT) }}
- queue: True
- kwarg:
queue: 2
- require:
- salt: refresh_pillar_{{ loop.index }}
{% endif %}
+1 -1
View File
@@ -3,7 +3,7 @@
#
# One pillar directory can map to multiple (state, tgt) actions.
# tgt is a raw salt compound expression. tgt_type is always "compound".
# Per-action `batch` / `batch_wait` override the orch defaults (10% / 15s).
# Per-action `batch` / `batch_wait` override the orch defaults (25% / 15s).
# An action with `highstate: True` triggers state.highstate instead of
# state.apply -- see salt/orch/push_batch.sls.
#
+9 -12
View File
@@ -7,7 +7,7 @@
# Two branches, keyed on node_id:
# A) node_id populated -> the change is scoped to that one minion. Look up the
# app in pillar_push_map.yaml and write an intent that runs the app's mapped
# state(s) on just that node, and only if the node is in the app's target.
# state(s) targeted to just that node.
# B) node_id empty -> grid-wide app change. Look up the app in
# pillar_push_map.yaml and write an intent with the entry's actions as-is.
#
@@ -116,17 +116,15 @@ def _app_from_setting(setting_id):
def _node_actions(entry, node_id):
# Copy the app's mapped actions but narrow each one to the single node.
# Copy the app's mapped actions but retarget each one to the single node.
# Preserves the state/highstate selection and any batch/batch_wait overrides.
# The app's own target is kept so the state is not applied to a node that does not
# run the app (e.g. an elasticsearch setting saved for a sensor fails to render there).
actions = []
for action in entry:
if not isinstance(action, dict):
continue
node_action = dict(action)
node_action['tgt'] = 'L@{} and ( {} )'.format(node_id, action.get('tgt', '*'))
node_action['tgt_type'] = 'compound'
node_action['tgt'] = node_id
node_action['tgt_type'] = 'glob'
actions.append(node_action)
return actions
@@ -140,7 +138,6 @@ def run():
# top level so the reactor is robust to either shape.
event = data.get('data', data) # noqa: F821 -- data provided by reactor
setting_id = event.get('setting_id', '')
audit_id = event.get('id')
node_id = (event.get('node_id') or '').strip()
app = _app_from_setting(setting_id)
@@ -153,8 +150,8 @@ def run():
if not entry:
LOG.warning(
'push_pillar: app "%s" is not in pillar_push_map.yaml; change will be '
'picked up at the next scheduled highstate (setting_id=%s audit_id=%s)',
app, setting_id, audit_id,
'picked up at the next scheduled highstate (setting_id=%s)',
app, setting_id,
)
return {}
@@ -168,12 +165,12 @@ def run():
'node_{}_{}'.format(node_id, app), actions,
'audit:{}@{}'.format(setting_id, node_id),
)
LOG.info('push_pillar: per-node intent updated for %s on %s (setting_id=%s audit_id=%s)',
app, node_id, setting_id, audit_id)
LOG.info('push_pillar: per-node intent updated for %s on %s (setting_id=%s)',
app, node_id, setting_id)
return {}
# Branch B: grid-wide app change -> use the map entry's actions as-is.
actions = list(entry) # copy to avoid mutating the cache
_write_intent('pillar_{}'.format(app), actions, 'audit:{}'.format(setting_id))
LOG.info('push_pillar: app intent updated for %s (setting_id=%s audit_id=%s)', app, setting_id, audit_id)
LOG.info('push_pillar: app intent updated for %s (setting_id=%s)', app, setting_id)
return {}
-10
View File
@@ -251,16 +251,6 @@ socsensoronirepos:
- mode: 775
- makedirs: True
nsm_rules_dir:
file.directory:
- name: /nsm/rules
- user: socore
- group: socore
- recurse:
- user
- group
- show_changes: False
create_custom_local_yara_repo_template:
git.present:
- name: /nsm/rules/custom-local-repos/local-yara
-31
View File
@@ -9,7 +9,6 @@
{%- set MDENGINE = GLOBALS.md_engine %}
{%- set LOGSTASH_ENABLED = LOGSTASH_MERGED.enabled %}
{%- set TG_OUT = TELEGRAFMERGED.output | upper %}
{%- set PG_ONLY = TG_OUT == 'POSTGRES' %}
{%- set PG_HOST = GLOBALS.manager_ip %}
{#- Per-minion telegraf creds live in the grid-wide telegraf/creds.sls pillar,
written by /usr/sbin/so-telegraf-cred on the manager. Each minion looks up
@@ -150,12 +149,7 @@
# Read metrics about cpu usage
[[inputs.cpu]]
## Whether to report per-cpu stats or not
{%- if PG_ONLY %}
{#- SOC and so-stats-show read only cpu-total #}
percpu = false
{%- else %}
percpu = true
{%- endif %}
## Whether to report total system cpu stats or not
totalcpu = true
## If true, collect raw CPU time metrics.
@@ -205,11 +199,6 @@
## The typical use case is for LVM volumes, to get the VG/LV name instead of
## the near-meaningless DM-0 name.
# name_templates = ["$ID_FS_LABEL","$DM_VG_NAME/$DM_LV_NAME"]
{%- if PG_ONLY %}
{#- Whole disks, md RAID included, already carry the I/O of their partitions and dm devices #}
[inputs.diskio.tagdrop]
name = ["sd*[0-9]", "vd*[0-9]", "xvd*[0-9]", "hd*[0-9]", "nvme*p[0-9]*", "mmcblk*p[0-9]*", "dm-*", "sr*", "loop*"]
{%- endif %}
# Get kernel statistics from /proc/stat
@@ -253,12 +242,6 @@
password = "{{ ES_PASS }}"
insecure_skip_verify = true
cluster_health = true
{%- if PG_ONLY %}
{#- SOC reads cluster status, indices and clusterstats_nodes ingest stats; alarms plan jvm and fs #}
cluster_health_level = "cluster"
node_stats = ["indices", "jvm", "fs"]
namedrop = ["elasticsearch_clusterstats_indices"]
{%- endif %}
{%- elif grains['role'] in ['so-searchnode'] %}
[[inputs.elasticsearch]]
servers = ["https://{{ NODEIP }}:9200"]
@@ -266,10 +249,6 @@
username = "{{ ES_USER }}"
password = "{{ ES_PASS }}"
insecure_skip_verify = true
{%- if PG_ONLY %}
{#- SOC reads indices; alarms plan jvm and fs #}
node_stats = ["indices", "jvm", "fs"]
{%- endif %}
{%- endif %}
# ## Timeout for HTTP requests to the elastic search server(s)
@@ -312,10 +291,6 @@
collect = ["pipelines"]
username = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user') }}"
password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}"
{%- if PG_ONLY %}
{#- SOC reads logstash_events only #}
namedrop = ["logstash_plugins"]
{%- endif %}
{%- endif %}
{% if grains.role in ['so-manager','so-managerhype','so-managersearch','so-standalone','so-receiver'] and GLOBALS.pipeline == "KAFKA" -%}
@@ -417,12 +392,6 @@
# # Read metrics about network interface usage
[[inputs.net]]
{%- if PG_ONLY %}
{%- from 'docker/docker.map.jinja' import DOCKERMERGED %}
{#- Only virtual interfaces; every physical NIC, bond members included, is kept for SOC and alarms #}
[inputs.net.tagdrop]
interface = {{ (['veth*', 'docker*', 'lo'] + DOCKERMERGED.networks.keys() | list) | json }}
{%- endif %}
# Scripts run every 30s||TELEGRAFMERGED.config.interval - ES index script doesn't need to run as frequently
{%- if GLOBALS.is_manager or GLOBALS.role == 'so-heavynode' %}