diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index c8ebae084..929103de5 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -656,6 +656,108 @@ set_version() { fi } +so_telegraf_output() { + local defaultsfile="${DEFAULT_SALT_DIR}/salt/telegraf/defaults.yaml" + local output + + output=$(lookup_pillar output telegraf) || return 1 + if [ -z "$output" ]; then + if [ ! -f "$defaultsfile" ]; then + echo "Could not find $defaultsfile" >&2 + return 1 + fi + output=$(grep -E "^ output: " "$defaultsfile" | awk -F: '{print $2}' | tr -d '[:space:]') + fi + echo "${output^^}" +} + +so_telegraf_pg_check() { + local app="$1" + local databases role_ok + + if ! docker_check_running postgres; then + echo "so-postgres is not running. Run $app on the manager." >&2 + return 1 + fi + if ! databases=$(docker exec so-postgres psql -U postgres -lqt 2>/dev/null); then + echo "Unable to list the so-postgres databases." >&2 + return 1 + fi + if ! echo "$databases" | cut -d\| -f1 | grep -qw so_telegraf; then + echo "Database so_telegraf not found. Is telegraf.output set to POSTGRES or BOTH?" >&2 + return 1 + fi + if ! role_ok=$(docker exec so-postgres psql -U postgres -tAc "SELECT rolcanlogin AND NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls) FROM pg_roles WHERE rolname = 'so_maint_telegraf'" 2>/dev/null); then + echo "Unable to query the so-postgres roles." >&2 + return 1 + fi + if [ -z "$role_ok" ]; then + echo "The so_maint_telegraf role is missing; run a highstate." >&2 + return 1 + elif [ "$role_ok" != t ]; then + echo "The so_maint_telegraf role is misconfigured; run a highstate." >&2 + return 1 + fi +} + +so_telegraf_psql() { + local app="$1" + local sql="$2" + + docker exec -e PGOPTIONS="-c application_name=${app} -c default_transaction_read_only=on -c client_min_messages=error" \ + so-postgres psql -U so_maint_telegraf -d so_telegraf -v ON_ERROR_STOP=1 -At -F '|' -c "$sql" +} + +# Prints one stream|current|then line per data stream, in bytes summed across hosts. current is +# each host's newest size in the past 4 hours; then is that host's newest size in the window +# [now - start, now - stop), or its current size when it has none there, so a host missing from the +# window adds no growth. Empty output means the window has no rows at all. Values over 15 digits +# (1 PB) are skipped so the sums here and in the callers stay far below the 64-bit limit; such a +# value can only come from a forged Telegraf writer. +so_es_index_size_growth() { + local app="$1" + local start="$2" + local stop="$3" + local relkind + + relkind=$(so_telegraf_psql "$app" "SELECT c.relkind FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'telegraf' AND c.relname = 'elasticsearch_index_size'") || return 1 + [[ "$relkind" == "r" || "$relkind" == "p" ]] || return 0 + + so_telegraf_psql "$app" "$(cat <= now() - interval '${start}' + AND f.key ~ '^[^|"''[:space:][:cntrl:]]+$' + AND pg_catalog.strpos(f.key, pg_catalog.chr(92)) = 0 + AND f.value ~ '^[0-9]{1,15}$' +), +cur AS ( + SELECT DISTINCT ON (tag_id, stream) tag_id, stream, bytes + FROM sizes + WHERE time >= now() - interval '4 hours' + ORDER BY tag_id, stream, time DESC +), +past AS ( + SELECT DISTINCT ON (tag_id, stream) tag_id, stream, bytes + FROM sizes + WHERE time < now() - interval '${stop}' + ORDER BY tag_id, stream, time DESC +) +SELECT c.stream, sum(c.bytes), sum(coalesce(p.bytes, c.bytes)) +FROM cur c +LEFT JOIN past p USING (tag_id, stream) +WHERE EXISTS ( + SELECT 1 FROM telegraf.elasticsearch_index_size + WHERE time >= now() - interval '${start}' AND time < now() - interval '${stop}' +) +GROUP BY c.stream +ORDER BY c.stream +EOF +)" +} + status () { printf "\n=========================================================================\n$(date) | $1\n=========================================================================\n" } diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index b1e0d9162..2b3cd7605 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -103,7 +103,6 @@ docker: - 80:80 - 443:443 - 8443:8443 - - 7788:7788 - 7789:7789 custom_bind_mounts: [] extra_hosts: [] diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth index 3381947eb..9acdaafcd 100644 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-growth @@ -5,6 +5,10 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +# Reports per data stream Elasticsearch growth over the last 24 hours, 7 days and 30 days. The data +# source follows telegraf.output: Postgres in POSTGRES mode, where history is limited to +# postgres.telegraf.retention_days and the script must run on the manager; InfluxDB otherwise. + INFLUX_URL="https://localhost:8086/api/v2" . /usr/sbin/so-common @@ -18,8 +22,6 @@ lookup_org_id() { echo "$response" | jq -r ".orgs[] | select(.name == \"Security Onion\").id" } -ORG_ID=$(lookup_org_id) - run_flux_query() { local query=$1 request "query?org=$ORG_ID" -H 'Accept:application/csv' -H 'Content-type:application/vnd.flux' -d "$query" -XPOST 2>/dev/null @@ -39,75 +41,137 @@ bytes_to_gb() { fi } -indexes_query='from(bucket: "telegraf/so_long_term") -|> range(start: -7d) -|> filter(fn: (r) => r._measurement == "elasticsearch_index_size") -|> distinct(column: "_field") -|> keep(columns: ["_field"])' +influxdb_indices_growth() { + ORG_ID=$(lookup_org_id) -indexes_result=$(run_flux_query "$indexes_query") -indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | grep -v '^_field$' | sed 's/\r$//' | sort -u) + indexes_query='from(bucket: "telegraf/so_long_term") + |> range(start: -7d) + |> filter(fn: (r) => r._measurement == "elasticsearch_index_size") + |> distinct(column: "_field") + |> keep(columns: ["_field"])' -printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)" -printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" + indexes_result=$(run_flux_query "$indexes_query") + indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | grep -v '^_field$' | sed 's/\r$//' | sort -u) -for index in $indexes; do - [[ -z "$index" ]] && continue - current_query="from(bucket: \"telegraf/so_long_term\") - |> range(start: -4h) - |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") - |> last() - |> keep(columns: [\"_value\"])" - current_result=$(run_flux_query "$current_query") - current_size=$(read_csv_result "$current_result") - current_size=${current_size:-0} + printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)" + printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" - size_24h_query="from(bucket: \"telegraf/so_long_term\") - |> range(start: -25h, stop: -23h) - |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") - |> last() - |> keep(columns: [\"_value\"])" - size_24h_result=$(run_flux_query "$size_24h_query") - size_24h_ago=$(read_csv_result "$size_24h_result") - size_24h_ago=${size_24h_ago:-$current_size} + for index in $indexes; do + [[ -z "$index" ]] && continue + current_query="from(bucket: \"telegraf/so_long_term\") + |> range(start: -4h) + |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") + |> last() + |> keep(columns: [\"_value\"])" + current_result=$(run_flux_query "$current_query") + current_size=$(read_csv_result "$current_result") + current_size=${current_size:-0} - size_7d_query="from(bucket: \"telegraf/so_long_term\") - |> range(start: -7d8h, stop: -7d) - |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") - |> last() - |> keep(columns: [\"_value\"])" - size_7d_result=$(run_flux_query "$size_7d_query") - size_7d_ago=$(read_csv_result "$size_7d_result") - size_7d_ago=${size_7d_ago:-$current_size} + size_24h_query="from(bucket: \"telegraf/so_long_term\") + |> range(start: -25h, stop: -23h) + |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") + |> last() + |> keep(columns: [\"_value\"])" + size_24h_result=$(run_flux_query "$size_24h_query") + size_24h_ago=$(read_csv_result "$size_24h_result") + size_24h_ago=${size_24h_ago:-$current_size} - size_30d_query="from(bucket: \"telegraf/so_long_term\") - |> range(start: -30d8h, stop: -30d) - |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") - |> last() - |> keep(columns: [\"_value\"])" - size_30d_result=$(run_flux_query "$size_30d_query") - size_30d_ago=$(read_csv_result "$size_30d_result") - size_30d_ago=${size_30d_ago:-$current_size} + size_7d_query="from(bucket: \"telegraf/so_long_term\") + |> range(start: -7d8h, stop: -7d) + |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") + |> last() + |> keep(columns: [\"_value\"])" + size_7d_result=$(run_flux_query "$size_7d_query") + size_7d_ago=$(read_csv_result "$size_7d_result") + size_7d_ago=${size_7d_ago:-$current_size} - # if an index was recently cleaned up by ilm it will result in a negative number for 'index growth'. - growth_24h=$(( current_size > size_24h_ago ? current_size - size_24h_ago : 0 )) + size_30d_query="from(bucket: \"telegraf/so_long_term\") + |> range(start: -30d8h, stop: -30d) + |> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\") + |> last() + |> keep(columns: [\"_value\"])" + size_30d_result=$(run_flux_query "$size_30d_query") + size_30d_ago=$(read_csv_result "$size_30d_result") + size_30d_ago=${size_30d_ago:-$current_size} - growth_7d=$(( current_size > size_7d_ago ? current_size - size_7d_ago : 0 )) + # if an index was recently cleaned up by ilm it will result in a negative number for 'index growth'. + growth_24h=$(( current_size > size_24h_ago ? current_size - size_24h_ago : 0 )) - growth_30d=$(( current_size > size_30d_ago ? current_size - size_30d_ago : 0 )) + growth_7d=$(( current_size > size_7d_ago ? current_size - size_7d_ago : 0 )) - growth_24h_gb=$(bytes_to_gb "$growth_24h") - growth_7d_gb=$(bytes_to_gb "$growth_7d") - growth_30d_gb=$(bytes_to_gb "$growth_30d") + growth_30d=$(( current_size > size_30d_ago ? current_size - size_30d_ago : 0 )) - # Only results for indices with atleast 1 metric above 0.00 - if [[ "$growth_24h_gb" != "0.00" ]] || [[ "$growth_7d_gb" != "0.00" ]] || [[ "$growth_30d_gb" != "0.00" ]]; then - printf "%020.2f|%-50s %15s %15s %15s\n" \ - "$growth_24h" \ - "$index" \ - "$growth_24h_gb" \ - "$growth_7d_gb" \ - "$growth_30d_gb" - fi -done | sort -t'|' -k1,1nr | cut -d'|' -f2- + growth_24h_gb=$(bytes_to_gb "$growth_24h") + growth_7d_gb=$(bytes_to_gb "$growth_7d") + growth_30d_gb=$(bytes_to_gb "$growth_30d") + # Only results for indices with atleast 1 metric above 0.00 + if [[ "$growth_24h_gb" != "0.00" ]] || [[ "$growth_7d_gb" != "0.00" ]] || [[ "$growth_30d_gb" != "0.00" ]]; then + printf "%020.2f|%-50s %15s %15s %15s\n" \ + "$growth_24h" \ + "$index" \ + "$growth_24h_gb" \ + "$growth_7d_gb" \ + "$growth_30d_gb" + fi + done | sort -t'|' -k1,1nr | cut -d'|' -f2- +} + +postgres_indices_growth() { + local app="so-elasticsearch-indices-growth" + local -a starts=("25 hours" "7 days 8 hours" "30 days 8 hours") + local -a stops=("23 hours" "7 days" "30 days") + local -A current past window_has_data + local i rows stream current_size past_size growth growth_gb sort_key show + local -a columns + + for i in 0 1 2; do + if ! rows=$(so_es_index_size_growth "$app" "${starts[i]}" "${stops[i]}"); then + echo "Failed to query telegraf.elasticsearch_index_size in so_telegraf." >&2 + exit 1 + fi + [[ -z "$rows" ]] && continue + window_has_data[$i]=1 + while IFS='|' read -r stream current_size past_size; do + current[$stream]=$current_size + past[$i,$stream]=$past_size + done <<< "$rows" + done + + printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)" + printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" + + for stream in "${!current[@]}"; do + columns=() + sort_key=0 + show=false + for i in 0 1 2; do + if [[ -z "${window_has_data[$i]}" ]]; then + columns+=("n/a") + continue + fi + current_size=${current[$stream]} + past_size=${past[$i,$stream]-$current_size} + growth=$(( current_size > past_size ? current_size - past_size : 0 )) + growth_gb=$(bytes_to_gb "$growth") + [[ "$growth_gb" != "0.00" ]] && show=true + [[ $i -eq 0 ]] && sort_key=$growth + columns+=("$growth_gb") + done + if [[ "$show" == true ]]; then + printf "%020.2f|%-50s %15s %15s %15s\n" "$sort_key" "$stream" "${columns[@]}" + fi + done | sort -t'|' -k1,1nr | cut -d'|' -f2- +} + +if ! TG_OUTPUT=$(so_telegraf_output); then + echo "Unable to read telegraf.output from pillar." >&2 + exit 1 +fi + +if [[ "$TG_OUTPUT" == "POSTGRES" ]]; then + so_telegraf_pg_check so-elasticsearch-indices-growth || exit 1 + postgres_indices_growth +else + influxdb_indices_growth +fi diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-retention-estimate b/salt/elasticsearch/tools/sbin/so-elasticsearch-retention-estimate index 1eb1e33ca..3c501434e 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-retention-estimate +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-retention-estimate @@ -58,6 +58,9 @@ Usage: $(basename "$0") [OPTIONS] Estimate remaining days until Elasticsearch cluster reaches low watermark threshold. +Index growth history follows telegraf.output: in POSTGRES mode it is read from Postgres, limited to +postgres.telegraf.retention_days, and the script must run on the manager; otherwise from InfluxDB. + OPTIONS: --json Output results in JSON format -v, --verbose Show additional output @@ -168,11 +171,25 @@ run_indices_growth() { fi } +fetch_postgres_total_bytes() { + local total + + total=$(so_es_index_size_growth so-elasticsearch-retention-estimate "$1" "$2") || return 1 + echo "$total" | awk -F'|' '{ sum += $3 } END { print sum + 0 }' +} + fetch_total_bytes() { local start="$1" local stop="$2" + local pg_start="$3" + local pg_stop="$4" local range_line + if [ "$TG_OUTPUT" = "POSTGRES" ]; then + fetch_postgres_total_bytes "$pg_start" "$pg_stop" + return + fi + if [ -n "$stop" ]; then range_line=" |> range(start: ${start}, stop: ${stop})" else @@ -208,8 +225,13 @@ fail() { echo -e "\nDISCLAIMER: Script output is based on current data patterns, but are approximations solely intended to assist with getting a general ILM policy configured." -ORG_ID=$(lookup_org_id) -[ -n "$ORG_ID" ] || fail "Unable to resolve InfluxDB org id" +TG_OUTPUT=$(so_telegraf_output) || fail "Unable to read telegraf.output from pillar." +if [ "$TG_OUTPUT" = "POSTGRES" ]; then + pg_check_error=$(so_telegraf_pg_check so-elasticsearch-retention-estimate 2>&1) || fail "$pg_check_error" +else + ORG_ID=$(lookup_org_id) + [ -n "$ORG_ID" ] || fail "Unable to resolve InfluxDB org id" +fi cluster_storage_size=0 indexed_storage_source="elasticsearch" @@ -221,10 +243,11 @@ if [ -n "$cluster_storage_size_output" ]; then fi fi -# historical data from influxdb for growth calculation -one_day_total=$(fetch_total_bytes "-25h" "-23h") -seven_day_total=$(fetch_total_bytes "-7d8h" "-7d") -thirty_day_total=$(fetch_total_bytes "-30d8h" "-30d") +# historical index sizes for growth calculation +query_error="Failed to query telegraf.elasticsearch_index_size in so_telegraf." +one_day_total=$(fetch_total_bytes "-25h" "-23h" "25 hours" "23 hours") || fail "$query_error" +seven_day_total=$(fetch_total_bytes "-7d8h" "-7d" "7 days 8 hours" "7 days") || fail "$query_error" +thirty_day_total=$(fetch_total_bytes "-30d8h" "-30d" "30 days 8 hours" "30 days") || fail "$query_error" # available historical windows (prefer 30d/7d when available, to avoid using recent 24h traffic spike as true daily ingest rate) history_days=0 @@ -244,7 +267,12 @@ elif [ "$one_day_total" -gt 0 ]; then historical_total=$one_day_total fi -[ "$history_days" -gt 0 ] || fail "Historical InfluxDB data unavailable for growth calculation. If this a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics." +if [ "$TG_OUTPUT" = "POSTGRES" ]; then + history_unavailable="Historical Postgres data unavailable for growth calculation. Postgres keeps postgres.telegraf.retention_days days of history; if this is a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics." +else + history_unavailable="Historical InfluxDB data unavailable for growth calculation. If this a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics." +fi +[ "$history_days" -gt 0 ] || fail "$history_unavailable" # Daily growth rate growth_bytes=$(( cluster_storage_size - historical_total )) @@ -729,6 +757,7 @@ if [ "$should_trigger_recommendations" = true ]; then [ -n "$index" ] || continue growth_24h_gb=$(echo "$line" | awk '{print $(NF-2)}') + [[ "$growth_24h_gb" =~ ^[0-9]+(\.[0-9]+)?$ ]] || growth_24h_gb=0 creation_date_display="" retention_days="" diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 5c1229787..74edd8dba 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -118,10 +118,6 @@ firewall: tcp: - 5601 udp: [] - localrules: - tcp: - - 7788 - udp: [] nginx: tcp: - 80 @@ -200,7 +196,6 @@ firewall: - postgres - elasticsearch_rest - elasticsearch_node - - localrules sensor: portgroups: - beats_5044 @@ -391,7 +386,6 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update - - localrules - sensoroni fleet: portgroups: @@ -611,7 +605,6 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update - - localrules - sensoroni fleet: portgroups: @@ -829,7 +822,6 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update - - localrules - sensoroni fleet: portgroups: @@ -1058,7 +1050,6 @@ firewall: - elastic_agent_update - endgame - strelka_frontend - - localrules fleet: portgroups: - elasticsearch_rest @@ -1518,7 +1509,6 @@ firewall: - elasticsearch_rest - elasticsearch_node - elastic_agent_control - - localrules sensor: portgroups: - beats_5044 diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index a5181e50f..1e855c8af 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -138,9 +138,6 @@ firewall: kibana: tcp: *tcpsettings udp: *udpsettings - localrules: - tcp: *tcpsettings - udp: *udpsettings nginx: tcp: *tcpsettings udp: *udpsettings diff --git a/salt/manager/tools/sbin/so-push-drainer b/salt/manager/tools/sbin/so-push-drainer index 569f47ef2..aad62b5c7 100644 --- a/salt/manager/tools/sbin/so-push-drainer +++ b/salt/manager/tools/sbin/so-push-drainer @@ -17,6 +17,7 @@ is older than debounce_seconds, this script: * concatenates the actions lists from every ready intent * dedupes by (state or __highstate__, tgt, tgt_type) * drops actions whose target matches no accepted minion (e.g. a role with no nodes yet) + * holds everything for a later pass while salt-master has only just started * dispatches a single `salt-run state.orchestrate orch.push_batch --async` with the deduped actions list passed as pillar kwargs * deletes the contributed intent files on successful dispatch @@ -46,6 +47,12 @@ PENDING_DIR = '/opt/so/state/push_pending' LOCK_FILE = os.path.join(PENDING_DIR, '.lock') LOG_FILE = '/opt/so/log/salt/so-push-drainer.log' MASTER_CONFIG = '/etc/salt/master' +MASTER_PIDFILE = '/var/run/salt-master.pid' + +# Minions rejoin a restarted master's publisher up to ~22s after it returns (zmq reconnect interval +# plus jitter, each up to recon_default + recon_max) and a job published before then reaches nobody. +# Doubled to allow for a slow master start. +MASTER_SETTLE_SECONDS = 45 DISPATCHED_DIR = '/opt/so/state/push_dispatched' @@ -140,6 +147,14 @@ def _targeted(actions, log): return targeted +def _master_settling(now): + try: + age = now - os.path.getmtime(MASTER_PIDFILE) + except OSError: + return False + return 0 <= age < MASTER_SETTLE_SECONDS + + def _dispatch(actions, log): pillar_arg = json.dumps({'actions': actions}) cmd = [ @@ -410,6 +425,10 @@ def main(): log.debug('no ready intents (%d still in debounce window)', skipped) return 0 + if _master_settling(now): + log.info('salt-master just started; holding %d intent(s) until minions reconnect', len(ready)) + return 0 + combined_actions = [] oldest_first_touch = now all_paths = [] diff --git a/salt/manager/tools/sbin/so-push-drainer_test.py b/salt/manager/tools/sbin/so-push-drainer_test.py index a6deab5f2..3f26d1822 100644 --- a/salt/manager/tools/sbin/so-push-drainer_test.py +++ b/salt/manager/tools/sbin/so-push-drainer_test.py @@ -199,6 +199,17 @@ class TestHelpers(DrainerTestCase): drainer._unlink(self.tmpdir, self.log) self.log.exception.assert_called_once() + def test_master_settling(self): + pidfile = os.path.join(self.tmpdir, 'salt-master.pid') + now = time.time() + with patch.object(drainer, 'MASTER_PIDFILE', pidfile): + self.assertFalse(drainer._master_settling(now)) + self.write_json(self.tmpdir, 'salt-master.pid', '1100') + for age, settling in ((0, True), (drainer.MASTER_SETTLE_SECONDS - 1, True), + (drainer.MASTER_SETTLE_SECONDS, False), (-60, False)): + os.utime(pidfile, (now - age, now - age)) + self.assertIs(drainer._master_settling(now), settling, age) + class TestDispatch(DrainerTestCase): @@ -475,6 +486,7 @@ class TestMain(DrainerTestCase): ('_load_push_cfg', {'side_effect': lambda: self.cfg}), ('_check_dispatched', {}), ('_targeted', {'side_effect': lambda actions, log: actions}), + ('_master_settling', {'return_value': False}), ): patcher = patch.object(drainer, name, **kwargs) setattr(self, name, patcher.start()) @@ -528,6 +540,16 @@ class TestMain(DrainerTestCase): with patch.object(drainer.os, 'unlink', side_effect=OSError('busy')): self.assertEqual(drainer.main(), 0) + def test_holds_while_master_settling(self): + path = self.intent('pillar_soc.json') + self._master_settling.return_value = True + with patch.object(drainer, '_dispatch') as dispatch: + self.assertEqual(drainer.main(), 0) + dispatch.assert_not_called() + self._check_dispatched.assert_called_once() + self.assertTrue(os.path.exists(path)) + self.assertIn('holding 1 intent(s)', self.logged('info')) + def test_dispatch_failure_keeps_intents(self): path = self.intent('pillar_soc.json') with patch.object(drainer, '_dispatch', return_value=None): diff --git a/salt/manager/tools/sbin/so-yaml.py b/salt/manager/tools/sbin/so-yaml.py index d6465e97b..c38d523db 100755 --- a/salt/manager/tools/sbin/so-yaml.py +++ b/salt/manager/tools/sbin/so-yaml.py @@ -20,6 +20,7 @@ def showUsage(args): print(' append - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr) print(' appendlistobject - Append an object to a yaml list key. Requires KEY and JSON_OBJECT args.', file=sys.stderr) print(' removelistitem - Remove a list item from a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr) + print(' purgelistitem - Remove every occurrence of a list item from all lists nested under a yaml key, if it exists. Requires KEY and LISTITEM args.', file=sys.stderr) print(' replacelistobject - Replace a list object based on a condition. Requires KEY, CONDITION_FIELD, CONDITION_VALUE, and JSON_OBJECT args.', file=sys.stderr) print(' add - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr) print(' get [-r] - Displays (to stdout) the value stored in the given key. Requires KEY arg. Use -r for raw output without YAML formatting.', file=sys.stderr) @@ -199,6 +200,33 @@ def removelistitem(args): return 0 +def purgeListItem(content, listItem): + if isinstance(content, dict): + for value in content.values(): + purgeListItem(value, listItem) + elif isinstance(content, list): + content[:] = [item for item in content if item != listItem] + for item in content: + purgeListItem(item, listItem) + + +def purgelistitem(args): + if len(args) != 3: + print('Missing filename, key arg, or list item to purge', file=sys.stderr) + showUsage(None) + return 1 + + filename = args[0] + key = args[1] + listItem = args[2] + + content = loadYaml(filename) + purgeListItem(getKeyValue(content, key), convertType(listItem)) + writeYaml(filename, content) + + return 0 + + def replaceListObjectByCondition(content, key, conditionField, conditionValue, newObject): pieces = key.split(".", 1) if len(pieces) > 1: @@ -291,6 +319,8 @@ def add(args): def removeKey(content, key): + if not isinstance(content, dict): + return pieces = key.split(".", 1) if len(pieces) > 1: if pieces[0] in content: @@ -334,6 +364,8 @@ def replace(args): def getKeyValue(content, key): + if not isinstance(content, dict): + return None pieces = key.split(".", 1) if len(pieces) > 1 and pieces[0] in content: return getKeyValue(content[pieces[0]], pieces[1]) @@ -385,6 +417,7 @@ def main(): "append": append, "appendlistobject": appendlistobject, "removelistitem": removelistitem, + "purgelistitem": purgelistitem, "replacelistobject": replacelistobject, "get": get, "remove": remove, diff --git a/salt/manager/tools/sbin/so-yaml_test.py b/salt/manager/tools/sbin/so-yaml_test.py index 7fa4cf8a7..71f748703 100644 --- a/salt/manager/tools/sbin/so-yaml_test.py +++ b/salt/manager/tools/sbin/so-yaml_test.py @@ -128,6 +128,22 @@ class TestRemove(unittest.TestCase): sysmock.assert_called_once_with(1) self.assertIn("Missing filename or key arg\n", mock_stderr.getvalue()) + def test_remove_null_parent(self): + filename = "/tmp/so-yaml_test-remove.yaml" + file = open(filename, "w") + file.write("key1: null\n") + file.close() + + result = soyaml.remove([filename, "key1.child1.deep1"]) + self.assertEqual(result, 0) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1: null\n" + self.assertEqual(actual, expected) + def test_append_missing_arg(self): with patch('sys.exit', new=MagicMock()) as sysmock: with patch('sys.stderr', new=StringIO()) as mock_stderr: @@ -565,6 +581,19 @@ class TestRemove(unittest.TestCase): self.assertEqual(result, 2) self.assertEqual("", mock_stdout.getvalue()) + def test_get_null_parent(self): + with patch('sys.stdout', new=StringIO()) as mock_stdout: + with patch('sys.stderr', new=StringIO()) as mock_stderr: + filename = "/tmp/so-yaml_test-get.yaml" + file = open(filename, "w") + file.write("key1: null\n") + file.close() + + result = soyaml.get([filename, "key1.child1"]) + self.assertEqual(result, 2) + self.assertEqual("", mock_stdout.getvalue()) + self.assertIn("Key 'key1.child1' not found by so-yaml.py", mock_stderr.getvalue()) + def test_get_empty_file(self): with patch('sys.stdout', new=StringIO()) as mock_stdout: with patch('sys.stderr', new=StringIO()) as mock_stderr: @@ -709,6 +738,255 @@ class TestRemoveListItem(unittest.TestCase): self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue()) +class TestPurgeListItem(unittest.TestCase): + + def test_purgelistitem_missing_arg(self): + with patch('sys.exit', new=MagicMock()) as sysmock: + with patch('sys.stderr', new=StringIO()) as mock_stderr: + sys.argv = ["cmd", "help"] + soyaml.purgelistitem(["file", "key"]) + sysmock.assert_called() + self.assertIn("Missing filename, key arg, or list item to purge", mock_stderr.getvalue()) + + def test_purgelistitem(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: [a,b,c], key2: false}") + file.close() + + soyaml.purgelistitem([filename, "key1", "b"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n- a\n- c\nkey2: false\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_nested_lists(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: { child1: [a,b,c], child2: { deep1: [b,d], deep2: 45 } }, key2: [b,e]}") + file.close() + + soyaml.purgelistitem([filename, "key1", "b"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1:\n - a\n - c\n child2:\n deep1:\n - d\n deep2: 45\nkey2:\n- b\n- e\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_duplicates(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: [b,a,b,c,b]}") + file.close() + + soyaml.purgelistitem([filename, "key1", "b"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n- a\n- c\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_list_of_objects(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: [{name: item1, values: [a,b]}, b, {name: item2, values: [b,c]}]}") + file.close() + + soyaml.purgelistitem([filename, "key1", "b"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n- name: item1\n values:\n - a\n- name: item2\n values:\n - c\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_typed(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: [1, '1', 2]}") + file.close() + + soyaml.purgelistitem([filename, "key1", "1"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n- '1'\n- 2\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_item_not_in_list(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: { child1: [a,b,c] }}") + file.close() + + soyaml.purgelistitem([filename, "key1", "d"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1:\n - a\n - b\n - c\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_key_noexist(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: [a,b,c]}") + file.close() + + result = soyaml.purgelistitem([filename, "key2.child1", "b"]) + self.assertEqual(result, 0) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n- a\n- b\n- c\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_key_noexist_deep(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: { child1: [a,b,c] }}") + file.close() + + result = soyaml.purgelistitem([filename, "key1.child2.deep1", "b"]) + self.assertEqual(result, 0) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1:\n - a\n - b\n - c\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_scalar_and_null_values(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: { child1: b, child2: null, child3: [b] }}") + file.close() + + soyaml.purgelistitem([filename, "key1", "b"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1: b\n child2: null\n child3: []\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_lists_of_lists(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: [[a,b], [b,[b,c]], b]}") + file.close() + + soyaml.purgelistitem([filename, "key1", "b"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n- - a\n- - - c\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_file_object(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: [{name: x}, a, {name: y}, {name: x}]}") + file.close() + + itemfile = "/tmp/so-yaml_test-purgelistitem-item.yaml" + file = open(itemfile, "w") + file.write("name: x\n") + file.close() + + soyaml.purgelistitem([filename, "key1", "file:" + itemfile]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n- a\n- name: y\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_main(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: { child1: [a,b,c] }, key2: [b]}") + file.close() + + with patch('sys.exit', new=MagicMock()) as sysmock: + sys.argv = ["cmd", "purgelistitem", filename, "key1", "b"] + soyaml.main() + sysmock.assert_called_once_with(0) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child1:\n - a\n - c\nkey2:\n- b\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_with_remove(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("{key1: { child1: { deep1: [a] }, child2: [b] }, key2: { group1: { items: [x,child1,y] }, group2: { items: [child1] } }}") + file.close() + + soyaml.remove([filename, "key1.child1"]) + soyaml.purgelistitem([filename, "key2", "child1"]) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n child2:\n - b\nkey2:\n group1:\n items:\n - x\n - y\n group2:\n items: []\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_null_parent(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("key1: null\n") + file.close() + + result = soyaml.purgelistitem([filename, "key1.child1", "b"]) + self.assertEqual(result, 0) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1: null\n" + self.assertEqual(actual, expected) + + def test_purgelistitem_non_dict_parent(self): + filename = "/tmp/so-yaml_test-purgelistitem.yaml" + file = open(filename, "w") + file.write("key1:\n- a\n- b\n") + file.close() + + result = soyaml.purgelistitem([filename, "key1.child1", "b"]) + self.assertEqual(result, 0) + + file = open(filename, "r") + actual = file.read() + file.close() + + expected = "key1:\n- a\n- b\n" + self.assertEqual(actual, expected) + + class TestAppendListObject(unittest.TestCase): def test_appendlistobject_missing_arg(self): diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index ba27ca004..10f37e537 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -5,7 +5,6 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - . /usr/sbin/so-common . /usr/sbin/so-image-common @@ -17,7 +16,9 @@ INSTALLEDVERSION=$(cat /etc/soversion) # not finish its post-upgrade steps; its contents are the resume point. It is read # here before preupgrade_changes mutates INSTALLEDVERSION and before any highstate # stamps /etc/soversion from the pillar. + POSTVERSION_FILE=/etc/sopostversion + if [ -f "$POSTVERSION_FILE" ]; then POSTVERSION=$(cat "$POSTVERSION_FILE") else @@ -527,6 +528,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == "3.1.0" ]] && up_to_3.2.0 [[ "$INSTALLEDVERSION" == "3.2.0" ]] && up_to_3.3.0 [[ "$INSTALLEDVERSION" == "3.3.0" ]] && up_to_3.4.0 + [[ "$INSTALLEDVERSION" == "3.4.0" ]] && up_to_3.5.0 true } @@ -546,6 +548,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0 [[ "$POSTVERSION" == "3.2.0" ]] && post_to_3.3.0 [[ "$POSTVERSION" == "3.3.0" ]] && post_to_3.4.0 + [[ "$POSTVERSION" == "3.4.0" ]] && post_to_3.5.0 # All applicable post-upgrade steps completed; clear the resume marker. rm -f "$POSTVERSION_FILE" true @@ -1065,17 +1068,31 @@ telegraf_repair() { # when there is something to repair, so everyone else is left alone. local repair=/usr/sbin/so-telegraf-repair [[ -x "$repair" ]] || return 0 - docker ps --format '{{.Names}}' | grep -qx so-postgres || return 0 + local state + state=$(docker inspect -f '{{.State.Running}} {{.State.Restarting}}' so-postgres 2>/dev/null) || return 0 + if [[ "$state" != "true false" ]]; then + echo " so-postgres is not running; skipping the Telegraf partition check." + FINAL_MESSAGE_QUEUE+=("WARNING: so-postgres is not running, so Telegraf partitions were not checked. If /nsm is full, follow \"Telegraf Partition Repair\" at:") + FINAL_MESSAGE_QUEUE+=(" - https://securityonion.net/docs/postgresql") + return 0 + fi echo "Checking Telegraf metric partitions." - local status=0 - "$repair" --check >> "$SOUP_LOG" 2>&1 || status=$? + local out status=0 + out=$("$repair" --check 2>&1) || status=$? + echo "$out" >> "$SOUP_LOG" case "$status" in 0) echo " Telegraf partitions are healthy; nothing to repair." ;; 1) echo " Repairing stalled Telegraf partitions." - "$repair" --yes \ - || echo " warning: so-telegraf-repair failed; run it manually" >&2 ;; - *) echo " Skipping; Telegraf is not storing metrics in Postgres on this host." ;; + if ! "$repair" --yes; then + echo " warning: so-telegraf-repair failed; run it manually" >&2 + FINAL_MESSAGE_QUEUE+=("WARNING: Telegraf partition repair failed; Postgres metrics will keep filling /nsm until it is repaired:") + FINAL_MESSAGE_QUEUE+=(" - sudo so-telegraf-repair") + fi ;; + 2) echo " Skipping: $(tail -n 1 <<<"$out")" ;; + *) echo " warning: could not check Telegraf partitions" >&2 + FINAL_MESSAGE_QUEUE+=("WARNING: Telegraf partitions could not be checked during the upgrade; run:") + FINAL_MESSAGE_QUEUE+=(" - sudo so-telegraf-repair --check") ;; esac } @@ -1202,6 +1219,32 @@ add_detection_runtime_fields() { } ### 3.4.0 End ### +### 3.5.0 Scripts ### +up_to_3.5.0() { + remove_localrules_portgroup + + INSTALLEDVERSION=3.5.0 +} + +remove_localrules_portgroup() { + local pillar_file + + for pillar_file in /opt/so/saltstack/local/pillar/firewall/{soc,adv}_firewall.sls; do + grep -qs localrules "$pillar_file" || continue + so-yaml.py get "$pillar_file" firewall 2>/dev/null | grep -q '\' || continue + echo "Removing the localrules portgroup from $pillar_file." + so-yaml.py remove "$pillar_file" firewall.portgroups.localrules + so-yaml.py purgelistitem "$pillar_file" firewall.role localrules + done +} + +post_to_3.5.0() { + telegraf_repair + + set_postversion 3.5.0 +} +### 3.5.0 End ### + repo_sync() { echo "Sync the local repo." diff --git a/salt/nginx/enabled.sls b/salt/nginx/enabled.sls index cf3cb1e81..f508d6c0a 100644 --- a/salt/nginx/enabled.sls +++ b/salt/nginx/enabled.sls @@ -10,29 +10,25 @@ {% from 'docker/docker.map.jinja' import DOCKERMERGED %} {% from 'nginx/map.jinja' import NGINXMERGED %} +{% if GLOBALS.role != 'so-fleet' %} +{% set container_config = 'so-nginx' %} +{% else %} +{# if this is an so-fleet node then we want to use the port bindings, custom bind mounts defined for fleet #} +{% set container_config = 'so-nginx-fleet-node' %} +{% endif %} + include: - ca.trust - nginx.ssl - nginx.config - nginx.sostatus -{% if GLOBALS.role != 'so-fleet' %} -{% set container_config = 'so-nginx' %} -make-rule-dir-nginx: +fleet_beats_dir_nginx: file.directory: - - name: /nsm/rules - - user: socore - - group: socore - - recurse: - - user - - group + - name: /nsm/elastic-fleet/artifacts/beats + - makedirs: True - show_changes: False -{% else %} -{# if this is an so-fleet node then we want to use the port bindings, custom bind mounts defined for fleet #} -{% set container_config = 'so-nginx-fleet-node' %} -{% endif %} - so-nginx: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-nginx:{{ GLOBALS.so_version }} @@ -54,9 +50,9 @@ so-nginx: - /opt/so/log/nginx/:/var/log/nginx:rw - /opt/so/tmp/nginx/:/var/lib/nginx:rw - /opt/so/tmp/nginx/:/run:rw - - /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages - - /nsm/elastic-fleet/artifacts/:/opt/socore/html/artifacts + - /nsm/elastic-fleet/artifacts/beats/:/opt/socore/elastic-artifacts/artifacts/beats:ro {% if GLOBALS.is_manager %} + - /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages - /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro - /etc/pki/managerssl.key:/etc/pki/nginx/server.key:ro # ATT&CK Navigator binds @@ -64,7 +60,6 @@ so-nginx: - /opt/so/conf/navigator/config.json:/opt/socore/html/navigator/assets/config.json:ro - /nsm/repo:/opt/socore/html/repo:ro - /nsm/kernelrepo:/opt/socore/html/kernelrepo:ro - - /nsm/rules:/nsm/rules:ro {% if NGINXMERGED.external_suricata %} - /opt/so/rules/nids/suri:/surirules:ro {% endif %} @@ -106,6 +101,7 @@ so-nginx: {% endif %} - require: - file: nginxconf + - file: fleet_beats_dir_nginx {% if GLOBALS.is_manager %} {% if NGINXMERGED.ssl.replace_cert %} - file: managerssl_key diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index 357fc7e6d..0da187ff7 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -4,6 +4,8 @@ {%- set role = grains.id.split('_') | last %} {%- set influxpass = salt['pillar.get']('secrets:influx_pass') %} {%- set influxauth = ('so:' + influxpass) | base64_encode %} +{%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %} +{%- set full_server_name = all_names | unique | join(' ') %} worker_processes auto; error_log /var/log/nginx/error.log; @@ -39,52 +41,31 @@ http { include /etc/nginx/conf.d/*.conf; - {%- if role in ['fleet'] %} - - server { - listen 8443; - server_name {{ GLOBALS.hostname }}; - root /opt/socore/html; - location /artifacts/ { - try_files $uri =206; - proxy_read_timeout 90; - proxy_connect_timeout 90; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Proxy ""; - proxy_set_header X-Forwarded-Proto $scheme; - } - } - - {%- endif %} - {%- if GLOBALS.is_manager %} - {%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %} - {%- set full_server_name = all_names | unique | join(' ') %} - server { listen 80 default_server; server_name _; return 307 https://{{ GLOBALS.url_base }}$request_uri; } - + {%- endif %} + {%- if GLOBALS.is_manager or role == 'fleet' %} server { listen 8443; + {%- if GLOBALS.is_manager %} server_name {{ full_server_name }}; - root /opt/socore/html; - location /artifacts/ { - try_files $uri =206; - proxy_read_timeout 90; - proxy_connect_timeout 90; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Proxy ""; - proxy_set_header X-Forwarded-Proto $scheme; + {%- else %} + server_name {{ GLOBALS.hostname }}; + {%- endif %} + root /opt/socore/elastic-artifacts; + location /artifacts/beats/ { + try_files $uri =404; + } + location / { + return 404; } } - + {%- endif %} + {%- if GLOBALS.is_manager %} server { listen 443 ssl default_server; http2 on; @@ -111,21 +92,6 @@ http { {%- endif %} {%- if GLOBALS.is_manager %} - - server { - listen 7788; - server_name {{ full_server_name }}; - root /nsm/rules; - location / { - allow all; - sendfile on; - sendfile_max_chunk 1m; - autoindex on; - autoindex_exact_size off; - autoindex_format html; - autoindex_localtime on; - } - } {%- if NGINXMERGED.external_suricata %} server { listen 7789 ssl; diff --git a/salt/orch/push_batch.sls b/salt/orch/push_batch.sls index 2fff2f646..97ffba048 100644 --- a/salt/orch/push_batch.sls +++ b/salt/orch/push_batch.sls @@ -4,8 +4,19 @@ {% set BATCH_WAIT = AUTOAPPLY.batch_wait %} {# queue must be a top-level salt.state arg (kwarg is ignored); an int is max_queue and still fails on conflict #} +{# refresh_pillar doubles as the reachability check: minions take up to ~22s to rejoin a restarted master's + publisher, and a job published before then reaches nobody. Each failed attempt takes ~15s to time out. #} {% for action in actions %} +refresh_pillar_{{ loop.index }}: + salt.function: + - name: saltutil.refresh_pillar + - tgt: '{{ action.tgt }}' + - tgt_type: {{ action.get('tgt_type', 'compound') }} + - retry: + attempts: 3 + interval: 10 + {% if action.get('highstate') %} apply_highstate_{{ loop.index }}: salt.state: @@ -15,13 +26,9 @@ apply_highstate_{{ loop.index }}: - batch: {{ action.get('batch', BATCH) }} - batch_wait: {{ action.get('batch_wait', BATCH_WAIT) }} - queue: True + - require: + - salt: refresh_pillar_{{ loop.index }} {% else %} -refresh_pillar_{{ loop.index }}: - salt.function: - - name: saltutil.refresh_pillar - - tgt: '{{ action.tgt }}' - - tgt_type: {{ action.get('tgt_type', 'compound') }} - apply_{{ action.state | replace('.', '_') }}_{{ loop.index }}: salt.state: - tgt: '{{ action.tgt }}' diff --git a/salt/postgres/defaults.yaml b/salt/postgres/defaults.yaml index 754a0d44c..12925b318 100644 --- a/salt/postgres/defaults.yaml +++ b/salt/postgres/defaults.yaml @@ -17,3 +17,4 @@ postgres: log_min_messages: 'warning' shared_preload_libraries: pg_cron cron.database_name: postgres + cron.host: /run/postgresql diff --git a/salt/postgres/soc_postgres.yaml b/salt/postgres/soc_postgres.yaml index 4ab1e4dac..0c3af92f1 100644 --- a/salt/postgres/soc_postgres.yaml +++ b/salt/postgres/soc_postgres.yaml @@ -87,3 +87,9 @@ postgres: global: True advanced: True helpLink: postgres + cron.host: + description: Where pg_cron connects to run scheduled jobs such as Telegraf partition maintenance. Must be the container's PostgreSQL socket directory, /run/postgresql, because pg_cron connects without a password and only local socket connections are trusted. Any other value makes every scheduled job fail. Changing it restarts PostgreSQL. + readonly: True + global: True + advanced: True + helpLink: postgres diff --git a/salt/postgres/tools/sbin/so-stats-show b/salt/postgres/tools/sbin/so-stats-show index 3cf7a05d8..8fb180370 100644 --- a/salt/postgres/tools/sbin/so-stats-show +++ b/salt/postgres/tools/sbin/so-stats-show @@ -52,7 +52,7 @@ if [ -n "$FILTER_HOST" ] && ! [[ "$FILTER_HOST" =~ $HOST_RE ]]; then fi so_psql() { - docker exec so-postgres psql -U postgres -d so_telegraf -At -F $'\t' "$@" + docker exec -e PGOPTIONS='-c application_name=so-stats-show' so-postgres psql -U so_maint_telegraf -d so_telegraf -At -F $'\t' "$@" } if ! docker exec so-postgres psql -U postgres -lqt 2>/dev/null | cut -d\| -f1 | grep -qw so_telegraf; then @@ -60,6 +60,15 @@ if ! docker exec so-postgres psql -U postgres -lqt 2>/dev/null | cut -d\| -f1 | exit 2 fi +role_ok=$(docker exec so-postgres psql -U postgres -tAc "SELECT rolcanlogin AND NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls) FROM pg_roles WHERE rolname = 'so_maint_telegraf'" 2>/dev/null) +if [ -z "$role_ok" ]; then + echo "The so_maint_telegraf role is missing; run a highstate." + exit 1 +elif [ "$role_ok" != t ]; then + echo "The so_maint_telegraf role is misconfigured; run a highstate." + exit 1 +fi + table_exists() { local table="$1" [ -n "$(so_psql -c "SELECT 1 FROM information_schema.tables WHERE table_schema='${SCHEMA}' AND table_name='${table}' LIMIT 1;")" ] diff --git a/salt/postgres/tools/sbin/so-telegraf-postgres b/salt/postgres/tools/sbin/so-telegraf-postgres index df7d16e65..7e907cc00 100644 --- a/salt/postgres/tools/sbin/so-telegraf-postgres +++ b/salt/postgres/tools/sbin/so-telegraf-postgres @@ -4,15 +4,18 @@ set -e # Provision Telegraf state inside the so-postgres container. # Usage: so-telegraf-postgres # create_db Ensure the so_telegraf database exists. -# group_role Provision the so_telegraf group role, telegraf/partman schemas, -# pg_partman, the so_admin maintenance routines, and the hourly -# pg_cron maintenance job. +# group_role Provision the so_telegraf group role, the so_maint_telegraf +# maintenance login role, telegraf/partman schemas, pg_partman, +# the so_admin maintenance routines, and the hourly pg_cron +# maintenance job. # user Create or update a per-minion login role granted to so_telegraf. # Env: ROLE_USER, ROLE_PASS. # retention Reconcile partman retention and premake on telegraf parents. # Env: RETENTION_DAYS. -# maintenance Drain default partitions and run partman maintenance. -# check Report partition health. Non-zero if any parent is unhealthy. +# maintenance Drain default partitions and run partman maintenance, as +# so_maint_telegraf. +# check Report partition health, as so_maint_telegraf. Non-zero if any +# parent is unhealthy. # # A default partition holding rows for a day blocks creating that day's child, # so maintenance drains defaults before calling partman. Use so-telegraf-repair @@ -20,6 +23,10 @@ set -e cmd="${1:?subcommand required}" +# Long enough for an hourly maintenance run to release its locks, short enough +# that soup and admins are not left hanging. +LOCK_TIMEOUT=60s + case "$cmd" in create_db) if ! docker exec so-postgres psql -U postgres -tAc \ @@ -30,7 +37,9 @@ case "$cmd" in ;; group_role) - docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL' + docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -v lock_timeout="$LOCK_TIMEOUT" \ + -U postgres -d so_telegraf <<'EOSQL' +SET lock_timeout = :'lock_timeout'; DO $$ BEGIN IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'so_telegraf') THEN @@ -38,6 +47,17 @@ BEGIN END IF; END $$; +DO $$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'so_maint_telegraf') THEN + CREATE ROLE so_maint_telegraf; + END IF; +END +$$; +ALTER ROLE so_maint_telegraf LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD NULL; +GRANT so_telegraf TO so_maint_telegraf; +ALTER ROLE so_maint_telegraf SET role = so_telegraf; +ALTER ROLE so_maint_telegraf SET search_path = pg_catalog; GRANT CONNECT ON DATABASE so_telegraf TO so_telegraf; CREATE SCHEMA IF NOT EXISTS telegraf AUTHORIZATION so_telegraf; GRANT USAGE, CREATE ON SCHEMA telegraf TO so_telegraf; @@ -58,52 +78,90 @@ ALTER DEFAULT PRIVILEGES IN SCHEMA partman ALTER DEFAULT PRIVILEGES IN SCHEMA partman GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO so_telegraf; --- pg_cron runs these as postgres, so they must not sit in a schema any --- Telegraf role can create objects in. +-- Owned by postgres so no Telegraf role can create objects here; the routines +-- run with the caller's privileges. CREATE SCHEMA IF NOT EXISTS so_admin AUTHORIZATION postgres; REVOKE ALL ON SCHEMA so_admin FROM PUBLIC; +-- Resolved through pg_inherits, never by name, so a view or a table that only +-- carries the _default name is never read or written. +CREATE OR REPLACE FUNCTION so_admin.telegraf_defaults() +RETURNS TABLE ( + parent_table regclass, + default_part regclass, + has_rows boolean, + default_size bigint +) +LANGUAGE plpgsql +STABLE +SECURITY INVOKER +AS $func$ +DECLARE + r record; +BEGIN + FOR r IN + SELECT c.oid::regclass AS pt + FROM partman.part_config pc + JOIN pg_catalog.pg_class c ON c.oid = to_regclass(pc.parent_table) + WHERE pc.parent_table LIKE 'telegraf.%' + AND c.relkind = 'p' + ORDER BY pc.parent_table + LOOP + parent_table := r.pt; + + SELECT c.oid::regclass INTO default_part + FROM pg_catalog.pg_inherits i + JOIN pg_catalog.pg_class c ON c.oid = i.inhrelid + WHERE i.inhparent = r.pt + AND c.relkind = 'r' AND c.relispartition + AND pg_catalog.pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT'; + + IF default_part IS NULL THEN + has_rows := false; + default_size := NULL; + ELSE + EXECUTE format('SELECT EXISTS (SELECT 1 FROM %s LIMIT 1)', default_part) + INTO has_rows; + default_size := pg_catalog.pg_total_relation_size(default_part); + END IF; + + RETURN NEXT; + END LOOP; +END; +$func$; + CREATE OR REPLACE PROCEDURE so_admin.telegraf_maintenance() LANGUAGE plpgsql +SECURITY INVOKER +-- No SET clause: it would forbid the COMMITs below, so so_maint_telegraf pins search_path instead. AS $proc$ DECLARE - r record; - v_default text; - v_rows bigint; + r record; BEGIN -- No per-parent EXCEPTION handler: partition_data_proc commits internally, -- and COMMIT is illegal while a subtransaction is active. A failing parent -- aborts the run and the next pass retries. FOR r IN - SELECT parent_table, retention - FROM partman.part_config - WHERE parent_table LIKE 'telegraf.%' - ORDER BY parent_table + SELECT pc.parent_table, pc.retention, d.default_part, d.default_size + FROM so_admin.telegraf_defaults() d + JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table + WHERE d.default_part IS NOT NULL AND d.has_rows + ORDER BY pc.parent_table LOOP - v_default := format('%I.%I', - split_part(r.parent_table, '.', 1), - split_part(r.parent_table, '.', 2) || '_default'); - - CONTINUE WHEN to_regclass(v_default) IS NULL; - - EXECUTE format('SELECT count(*) FROM %s', v_default) INTO v_rows; - CONTINUE WHEN v_rows = 0; - - RAISE WARNING 'so_admin.telegraf_maintenance: % rows stranded in %, draining', - v_rows, v_default; + RAISE WARNING 'so_admin.telegraf_maintenance: % holds rows (%), draining', + r.default_part, pg_size_pretty(r.default_size); -- Cheaper to delete expired rows than to repartition and then drop them. IF r.retention IS NOT NULL THEN EXECUTE format('DELETE FROM %s WHERE "time" < now() - %L::interval', - v_default, r.retention); + r.default_part, r.retention); COMMIT; END IF; -- Bounded so a large backlog drains across several runs. CALL partman.partition_data_proc( p_parent_table := r.parent_table, - p_loop_count := 200, - p_source_table := v_default + p_loop_count := 200 ); COMMIT; END LOOP; @@ -112,58 +170,75 @@ BEGIN END; $proc$; -CREATE OR REPLACE FUNCTION so_admin.telegraf_partition_status() +DROP FUNCTION IF EXISTS so_admin.telegraf_partition_status(); +CREATE FUNCTION so_admin.telegraf_partition_status() RETURNS TABLE ( - parent_table text, - oldest_child date, - newest_child date, - days_ahead int, - retention text, - default_rows bigint, - default_size text + parent_table text, + oldest_child date, + newest_child date, + days_ahead int, + retention text, + default_has_rows boolean, + default_size text ) LANGUAGE plpgsql +STABLE AS $func$ DECLARE - r record; - v_default regclass; + r record; BEGIN FOR r IN - SELECT pc.parent_table AS pt, pc.retention AS ret - FROM partman.part_config pc - WHERE pc.parent_table LIKE 'telegraf.%' - ORDER BY pc.parent_table + SELECT d.parent_table AS pt, d.has_rows, d.default_size AS size, pc.retention AS ret + FROM so_admin.telegraf_defaults() d + JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table + ORDER BY d.parent_table::text LOOP - parent_table := r.pt; + parent_table := r.pt::text; retention := r.ret; - SELECT min(d), max(d) INTO oldest_child, newest_child + SELECT min(x), max(x) INTO oldest_child, newest_child FROM ( - SELECT to_date(substring(c.relname FROM '_p(\d{8})$'), 'YYYYMMDD') AS d - FROM pg_inherits i - JOIN pg_class c ON c.oid = i.inhrelid - WHERE i.inhparent = r.pt::regclass - AND pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT' + SELECT to_date(substring(c.relname FROM '_p(\d{8})$'), 'YYYYMMDD') AS x + FROM pg_catalog.pg_inherits i + JOIN pg_catalog.pg_class c ON c.oid = i.inhrelid + WHERE i.inhparent = r.pt + AND pg_catalog.pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT' ) s; - days_ahead := newest_child - current_date; - - v_default := to_regclass(format('%I.%I', - split_part(r.pt, '.', 1), - split_part(r.pt, '.', 2) || '_default')); - IF v_default IS NULL THEN - default_rows := NULL; - default_size := NULL; - ELSE - EXECUTE format('SELECT count(*) FROM %s', v_default::text) INTO default_rows; - default_size := pg_size_pretty(pg_total_relation_size(v_default)); - END IF; + days_ahead := newest_child - current_date; + default_has_rows := r.has_rows; + default_size := pg_size_pretty(r.size); RETURN NEXT; END LOOP; END; $func$; +GRANT USAGE ON SCHEMA so_admin TO so_telegraf; +GRANT EXECUTE ON FUNCTION so_admin.telegraf_defaults() TO so_telegraf; +GRANT EXECUTE ON PROCEDURE so_admin.telegraf_maintenance() TO so_telegraf; +GRANT EXECUTE ON FUNCTION so_admin.telegraf_partition_status() TO so_telegraf; + +-- Partitions created by a superuser session (an older repair) belong to +-- postgres; partman acting as so_telegraf could not drop them. +DO $$ +DECLARE + r record; +BEGIN + FOR r IN + SELECT n.nspname, c.relname + FROM pg_catalog.pg_class c + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r', 'p') + AND (n.nspname = 'telegraf' + OR (n.nspname = 'partman' AND c.relname LIKE 'template\_telegraf\_%')) + AND c.relowner <> 'so_telegraf'::regrole + LOOP + EXECUTE format('ALTER TABLE %I.%I OWNER TO so_telegraf', r.nspname, r.relname); + END LOOP; +END +$$; + -- Drop the registration older releases left in so_telegraf. SELECT CASE WHEN current_setting('cron.database_name', true) IS DISTINCT FROM current_database() @@ -175,20 +250,69 @@ DROP EXTENSION pg_cron CASCADE; \endif EOSQL + stalled=false + probe_failed=false + if out=$(docker exec -e PGOPTIONS="-c lock_timeout=$LOCK_TIMEOUT" so-postgres \ + psql -U so_maint_telegraf -d so_telegraf -tAc \ + "SELECT coalesce(bool_or(has_rows), false) FROM so_admin.telegraf_defaults()"); then + case "$out" in + t) stalled=true ;; + f) stalled=false ;; + *) probe_failed=true ;; + esac + else + probe_failed=true + fi + if $probe_failed; then + echo "ERROR: could not check Telegraf default partitions; the maintenance job was not changed." >&2 + exit 1 + fi + # Guarded on the live GUC so applying this before the postgresql.conf change # has restarted the container skips rather than failing. - docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d postgres <<'EOSQL' + docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -v stalled="$stalled" \ + -U postgres -d postgres <<'EOSQL' SELECT CASE WHEN current_setting('cron.database_name', true) = current_database() THEN 'true' ELSE 'false' END AS cron_here \gset \if :cron_here CREATE EXTENSION IF NOT EXISTS pg_cron; --- cron.schedule_in_database is idempotent by jobname. +-- A highstate reconciles the job's definition but never its active flag; only so-telegraf-repair turns it on. +BEGIN; +SELECT coalesce(max(jobid)::text, '') AS jobid, + count(*) > 0 AS job_exists, + coalesce(bool_and(active), true) AS job_active +FROM cron.job +WHERE jobname = 'telegraf-partman-maintenance' AND username = 'so_maint_telegraf' \gset +SELECT cron.unschedule(jobid) FROM cron.job +WHERE jobname = 'telegraf-partman-maintenance' AND username <> 'so_maint_telegraf'; +\if :job_exists +SELECT cron.alter_job(:jobid, + schedule := '17 * * * *', + command := 'CALL so_admin.telegraf_maintenance()', + database := 'so_telegraf'); +\else +-- Registered inactive on a stalled grid so nothing drains the default +-- partitions before so-telegraf-repair truncates them. SELECT cron.schedule_in_database( 'telegraf-partman-maintenance', '17 * * * *', 'CALL so_admin.telegraf_maintenance()', - 'so_telegraf' + 'so_telegraf', + 'so_maint_telegraf', + NOT :'stalled'::boolean ); +\endif +UPDATE cron.job SET nodename = current_setting('cron.host') +WHERE jobname = 'telegraf-partman-maintenance' + AND nodename IS DISTINCT FROM current_setting('cron.host'); +COMMIT; +SELECT coalesce(bool_and(active), true) AS job_active +FROM cron.job +WHERE jobname = 'telegraf-partman-maintenance' AND username = 'so_maint_telegraf' \gset +\if :job_active +\else +\echo 'WARNING: the Telegraf maintenance job is inactive; run ''sudo so-telegraf-repair''.' +\endif \else \echo 'pg_cron metadata database is not `postgres` yet; skipping job registration.' \endif @@ -241,13 +365,15 @@ SET retention = :'retention_days' || ' days', retention_keep_table = false, premake = 7, infinite_time_partitions = true -WHERE parent_table LIKE 'telegraf.%'; +WHERE parent_table LIKE 'telegraf.%' + AND EXISTS (SELECT 1 FROM pg_catalog.pg_class c + WHERE c.oid = to_regclass(parent_table) AND c.relkind = 'p'); \endif EOSQL ;; maintenance) - docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL' + docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U so_maint_telegraf -d so_telegraf <<'EOSQL' SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL THEN 'true' ELSE 'false' END AS has_proc \gset \if :has_proc @@ -259,7 +385,7 @@ EOSQL ;; check) - docker exec -i so-postgres psql -U postgres -d so_telegraf <<'EOSQL' + docker exec -i so-postgres psql -U so_maint_telegraf -d so_telegraf <<'EOSQL' \pset border 2 SELECT * FROM so_admin.telegraf_partition_status(); EOSQL @@ -278,9 +404,9 @@ LIMIT 5; \echo 'pg_cron is not installed in this database.' \endif EOSQL - unhealthy=$(docker exec so-postgres psql -U postgres -d so_telegraf -tAc \ + unhealthy=$(docker exec so-postgres psql -U so_maint_telegraf -d so_telegraf -tAc \ "SELECT count(*) FROM so_admin.telegraf_partition_status() - WHERE coalesce(default_rows, 0) > 0 OR coalesce(days_ahead, -1) < 1") + WHERE default_has_rows OR coalesce(days_ahead, -1) < 1") if [ "${unhealthy:-1}" != "0" ]; then echo "so-telegraf-postgres check: $unhealthy telegraf parent(s) unhealthy" >&2 exit 1 diff --git a/salt/postgres/tools/sbin/so-telegraf-repair b/salt/postgres/tools/sbin/so-telegraf-repair index 75004402e..f6d9cf9f9 100644 --- a/salt/postgres/tools/sbin/so-telegraf-repair +++ b/salt/postgres/tools/sbin/so-telegraf-repair @@ -11,26 +11,38 @@ # maintenance once. Healthy grids are reported and left alone. # # Usage: so-telegraf-repair [--check] [--yes] [--no-restart] -# --check Report health and change nothing. +# --check Report health and change nothing. Also reports a missing +# so_maint_telegraf role and a maintenance job that is missing +# or inactive. # --yes Skip the confirmation prompt (for soup and other automation). # --no-restart Never restart so-postgres, even if pg_cron's launcher is dead. # +# Repair mode (without --check) registers the so_maint_telegraf role and the +# maintenance job if they are missing, pauses the job while it repairs, and +# always leaves it active afterwards. Data work runs as so_maint_telegraf. +# # Exit status: # 0 healthy, or repair completed -# 1 repair is needed (--check only) -# 2 cannot run here: so-postgres, so_telegraf or pg_partman is missing +# 1 repair is needed (--check), or the repair failed after it started +# 2 skipped: Telegraf output is not Postgres, or so-postgres, so_telegraf or pg_partman is missing +# 3 could not check, for example a connection error -set -e +set -eE # Matches p_premake in telegraf.conf's create_parent template. PREMAKE=7 JOB_NAME=telegraf-partman-maintenance +MAINT_ROLE=so_maint_telegraf +# Long enough for an hourly maintenance run to release its locks, short enough +# that soup and admins are not left hanging. +LOCK_TIMEOUT=60s +PG_OPTS="-c lock_timeout=$LOCK_TIMEOUT -c application_name=so-telegraf-repair" CHECK_ONLY=false ASSUME_YES=false NO_RESTART=false -usage() { sed -n '/^# Usage:/,/^# 2 /p' "$0" | sed 's/^# \?//'; } +usage() { sed -n '/^# Usage:/,/^# 3 /p' "$0" | sed 's/^# \?//'; } while [[ $# -gt 0 ]]; do case "$1" in @@ -38,75 +50,170 @@ while [[ $# -gt 0 ]]; do --yes|-y) ASSUME_YES=true ;; --no-restart) NO_RESTART=true ;; -h|--help) usage; exit 0 ;; - *) echo "Unknown option: $1" >&2; usage >&2; exit 2 ;; + *) echo "Unknown option: $1" >&2; usage >&2; exit 3 ;; esac shift done -skip() { echo "$*"; exit 2; } +# Strips C0 (except tab and newline), DEL and C1 characters, so names and +# messages written with Telegraf credentials can't drive the terminal. +sanitise() { LC_ALL=C tr -d '\000-\010\013-\037\177' | LC_ALL=C sed 's/\xc2[\x80-\x9f]//g'; } -psql_tg() { docker exec -i so-postgres psql -U postgres -d so_telegraf "$@"; } -psql_pg() { docker exec -i so-postgres psql -U postgres -d postgres "$@"; } +ERR_FILE=$(mktemp) +ERR_STATUS=3 +ERR_PREFIX="Could not check Telegraf partitions" +job_id= +restore_active= + +on_err() { + # Command substitutions inherit this trap; only the top level reports. + [[ $BASH_SUBSHELL -eq 0 ]] || exit "$ERR_STATUS" + local line + line=$(tail -n 1 "$ERR_FILE" | sanitise) + echo "$ERR_PREFIX: ${line:-command failed}" >&2 + exit "$ERR_STATUS" +} + +on_exit() { + if [[ -n "$restore_active" ]]; then + psql_cron -tAc "SELECT cron.alter_job($job_id, active := $restore_active)" >/dev/null || true + fi + rm -f "$ERR_FILE" +} + +trap on_err ERR +trap on_exit EXIT + +finish() { trap - ERR; exit "$1"; } +skip() { echo "$*"; finish 2; } + +# Keeps the last stderr line for on_err while still showing all of it. +capture() { + local rc=0 + : >"$ERR_FILE" + "$@" 2>"$ERR_FILE" || rc=$? + sanitise <"$ERR_FILE" >&2 + return "$rc" +} + +psql_tg() { + capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U "$MAINT_ROLE" -d so_telegraf "$@" +} +psql_pg() { + capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U postgres -d postgres "$@" +} +psql_cron() { + capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U postgres -d "$cron_db" "$@" +} + +# A part_config row counts only while its parent is a partitioned table. +TELEGRAF_PARENTS="parent_table LIKE 'telegraf.%' + AND EXISTS (SELECT 1 FROM pg_class c + WHERE c.oid = to_regclass(parent_table) AND c.relkind = 'p')" -# query_to_xml so the per-table row counts need no helper function installed. REPORT=" -WITH parents AS ( - SELECT pc.parent_table, - pc.premake, - split_part(pc.parent_table, '.', 1) AS sch, - split_part(pc.parent_table, '.', 2) AS tbl - FROM partman.part_config pc - WHERE pc.parent_table LIKE 'telegraf.%' -), children AS ( - SELECT p.parent_table, - max(to_date(substring(c.relname FROM '_p(\d{8})\$'), 'YYYYMMDD')) AS newest_child - FROM parents p - JOIN pg_class pt ON pt.oid = p.parent_table::regclass - JOIN pg_inherits i ON i.inhparent = pt.oid - JOIN pg_class c ON c.oid = i.inhrelid - WHERE pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT' - GROUP BY p.parent_table -), defaults AS ( - SELECT p.parent_table, - p.premake, - format('%I.%I', p.sch, p.tbl || '_default') AS default_table, - to_regclass(format('%I.%I', p.sch, p.tbl || '_default')) AS default_oid - FROM parents p -) SELECT d.parent_table, c.newest_child, (c.newest_child - current_date) AS days_ahead, - d.premake, - CASE WHEN d.default_oid IS NULL THEN NULL ELSE - (xpath('/row/cnt/text()', - query_to_xml(format('SELECT count(*) AS cnt FROM %s', d.default_table), - false, true, '')))[1]::text::bigint - END AS default_rows, - CASE WHEN d.default_oid IS NULL THEN NULL - ELSE pg_size_pretty(pg_total_relation_size(d.default_oid)) END AS default_size -FROM defaults d -LEFT JOIN children c ON c.parent_table = d.parent_table -ORDER BY 1 + pc.premake, + d.has_rows AS default_has_rows, + pg_size_pretty(d.default_size) AS default_size +FROM so_admin.telegraf_defaults() d +JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table +JOIN pg_class p ON p.oid = d.parent_table AND p.relkind = 'p' +LEFT JOIN LATERAL ( + SELECT max(to_date(substring(ch.relname FROM '_p(\d{8})\$'), 'YYYYMMDD')) AS newest_child + FROM pg_inherits i + JOIN pg_class ch ON ch.oid = i.inhrelid + WHERE i.inhparent = d.parent_table + AND pg_get_expr(ch.relpartbound, ch.oid) <> 'DEFAULT' +) c ON true +ORDER BY d.parent_table::text " -docker ps --format '{{.Names}}' | grep -qx so-postgres \ +print_report() { + local report + report=$(psql_tg -c "$REPORT") + sanitise <<<"$report" +} + +has_cron() { + [[ -n "$cron_db" ]] || return 0 + psql_cron -tAc "SELECT 1 FROM pg_extension WHERE extname = 'pg_cron'" +} + +lookup_job() { + local cron + cron=$(has_cron) + [[ -n "$cron" ]] || return 0 + psql_cron -tA -F '|' -c \ + "SELECT jobid, active FROM cron.job + WHERE jobname = '$JOB_NAME' AND username = '$MAINT_ROLE' + ORDER BY jobid LIMIT 1" +} + +output=$(capture salt-call pillar.get telegraf:output --out=newline_values_only) +output=$(tr -d '[:space:]' <<<"$output" | tr '[:lower:]' '[:upper:]') +output=${output:-INFLUXDB} +case "$output" in + POSTGRES|BOTH) ;; + *) skip "Telegraf output is $(sanitise <<<"$output"); not checking Postgres partitions." ;; +esac + +containers=$(capture docker ps --format '{{.Names}}') +grep -qx so-postgres <<<"$containers" \ || skip "so-postgres is not running; nothing to repair." -docker exec so-postgres psql -U postgres -tAc \ - "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1 \ +has_db=$(psql_pg -tAc "SELECT 1 FROM pg_database WHERE datname = 'so_telegraf'") +[[ -n "$has_db" ]] \ || skip "The so_telegraf database does not exist; Telegraf is not writing to Postgres." -psql_tg -tAc "SELECT 1 FROM pg_extension WHERE extname='pg_partman'" | grep -q 1 \ + +# so_telegraf before the postgres state lands, postgres after. +cron_db=$(psql_pg -tAc "SELECT current_setting('cron.database_name', true)") +cron_db=$(tr -d '[:space:]' <<<"$cron_db") + +role_ok=$(psql_pg -tAc \ + "SELECT rolcanlogin AND NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls) + FROM pg_roles WHERE rolname = '$MAINT_ROLE'") +job= +[[ "$cron_db" == postgres ]] && job=$(lookup_job) + +if [[ "$role_ok" != t ]] && $CHECK_ONLY; then + echo "Repair is needed:" + if [[ -z "$role_ok" ]]; then + echo " * maintenance role $MAINT_ROLE is missing" + else + echo " * maintenance role $MAINT_ROLE is misconfigured" + fi + echo + echo "Re-run without --check to repair." + finish 1 +fi + +if ! $CHECK_ONLY && [[ "$role_ok" != t || ( "$cron_db" == postgres && -z "$job" ) ]]; then + echo "Registering the maintenance role and job." + if ! /usr/sbin/so-telegraf-postgres group_role; then + echo "Could not register the maintenance job; run a highstate and try again." >&2 + finish 3 + fi + [[ "$cron_db" == postgres ]] && job=$(lookup_job) +fi + +has_partman=$(psql_tg -tAc "SELECT 1 FROM pg_extension WHERE extname = 'pg_partman'") +[[ -n "$has_partman" ]] \ || skip "pg_partman is not installed in so_telegraf; nothing to repair." parents=$(psql_tg -tAc \ - "SELECT count(*) FROM partman.part_config WHERE parent_table LIKE 'telegraf.%'") -stranded=$(psql_tg -tAc "SELECT coalesce(sum(default_rows), 0) FROM ( $REPORT ) t") -behind=$(psql_tg -tAc \ - "SELECT count(*) FROM ( $REPORT ) t WHERE coalesce(days_ahead, -1) < 1") + "SELECT count(*) FROM partman.part_config WHERE $TELEGRAF_PARENTS") +counts=$(psql_tg -tAc \ + "SELECT count(*) FILTER (WHERE default_has_rows) || '|' || + count(*) FILTER (WHERE coalesce(days_ahead, -1) < 1) + FROM ( $REPORT ) t") +IFS='|' read -r stranded behind <<<"$counts" # premake < 7, or infinite_time_partitions off: without the latter partman # refuses to premake forward across the gap the stall left behind. misconfigured=$(psql_tg -tAc \ "SELECT count(*) FROM partman.part_config - WHERE parent_table LIKE 'telegraf.%' + WHERE $TELEGRAF_PARENTS AND (premake < $PREMAKE OR NOT infinite_time_partitions)") # Both columns are matched because which one carries the launcher's name varies @@ -115,16 +222,24 @@ launcher=$(psql_pg -tAc \ "SELECT count(*) FROM pg_stat_activity WHERE backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%'") -# so_telegraf before the postgres state lands, postgres after. -cron_db=$(docker exec so-postgres psql -U postgres -tAc \ - "SELECT current_setting('cron.database_name', true)" | tr -d '[:space:]') last_run=never -if [[ -n "$cron_db" ]]; then - last_run=$(docker exec so-postgres psql -U postgres -d "$cron_db" -tAc \ - "SELECT coalesce(max(d.start_time)::text, 'never') +cron_installed=$(has_cron) +if [[ -n "$cron_installed" ]]; then + run=$(psql_cron -tAc \ + "SELECT d.status || ' at ' || d.start_time FROM cron.job_run_details d JOIN cron.job j USING (jobid) - WHERE j.jobname = '$JOB_NAME'" 2>/dev/null | tr -d '[:space:]' || echo unknown) - [[ -n "$last_run" ]] || last_run=never + WHERE j.jobname = '$JOB_NAME' + ORDER BY d.start_time DESC LIMIT 1") + if [[ -n "$run" ]]; then + message=$(psql_cron -tAc \ + "SELECT coalesce(d.return_message, '') + FROM cron.job_run_details d JOIN cron.job j USING (jobid) + WHERE j.jobname = '$JOB_NAME' + ORDER BY d.start_time DESC LIMIT 1") + message=$(sanitise <<<"$message") + message=${message//$'\n'/ } + last_run="$(sanitise <<<"$run"): ${message:0:200}" + fi fi # A grid that has never written a metric has nothing to recover, and an empty @@ -132,43 +247,67 @@ fi restart_needed=false [[ "$launcher" -eq 0 && "$parents" -gt 0 && -n "$cron_db" ]] && restart_needed=true +# group_role registers the job only in the postgres database. +job_inactive=false +if [[ "$cron_db" == postgres ]]; then + [[ -z "$job" || "${job##*|}" == f ]] && job_inactive=true +fi + repair_needed=false [[ "$stranded" -gt 0 ]] && repair_needed=true [[ "$behind" -gt 0 ]] && repair_needed=true [[ "$misconfigured" -gt 0 ]] && repair_needed=true $restart_needed && repair_needed=true +$job_inactive && repair_needed=true echo "Telegraf partition status:" -psql_tg -c "$REPORT" -echo "Rows stranded in default partitions: $stranded" -echo "pg_cron metadata database: ${cron_db:-unset}" +print_report +echo "Default partitions with rows: $stranded" +echo "pg_cron metadata database: $(sanitise <<<"${cron_db:-unset}")" +[[ "$cron_db" == postgres ]] \ + || echo "pg_cron metadata database is $(sanitise <<<"${cron_db:-unset}"); not checking the maintenance job." echo "pg_cron launcher running: $([[ "$launcher" -gt 0 ]] && echo yes || echo no)" echo "Last $JOB_NAME run: $last_run" echo if ! $repair_needed; then echo "Telegraf partitions are healthy. Nothing to do." - exit 0 + finish 0 fi if $CHECK_ONLY; then echo "Repair is needed:" - [[ "$stranded" -gt 0 ]] && echo " * $stranded row(s) stranded in default partitions" + [[ "$stranded" -gt 0 ]] && echo " * $stranded default partition(s) hold rows" [[ "$behind" -gt 0 ]] && echo " * $behind parent(s) with no partition for the current window" [[ "$misconfigured" -gt 0 ]] && echo " * $misconfigured parent(s) with stale partman settings" $restart_needed && echo " * pg_cron's launcher is dead; maintenance is not running at all" + $job_inactive && echo " * the maintenance job is missing or inactive" echo echo "Re-run without --check to repair." - exit 1 + finish 1 fi if [[ "$stranded" -gt 0 ]] && ! $ASSUME_YES; then - echo "This will permanently discard the $stranded stranded row(s) above." + echo "This will permanently discard the rows in the $stranded default partition(s) above." $restart_needed && ! $NO_RESTART && \ echo "so-postgres will also be restarted, which briefly interrupts SOC." - [[ -t 0 ]] || { echo "Not a terminal; re-run with --yes to confirm." >&2; exit 2; } + [[ -t 0 ]] || { echo "Not a terminal; re-run with --yes to confirm." >&2; finish 1; } read -r -p "Continue? [y/N] " answer - [[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 0; } + [[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; finish 0; } +fi + +ERR_STATUS=1 +ERR_PREFIX="Telegraf partition repair failed" + +if [[ "$cron_db" == postgres ]]; then + if [[ -z "$job" ]]; then + echo "Could not register the maintenance job; run a highstate and try again." >&2 + finish 3 + fi + job_id=${job%%|*} + [[ "${job##*|}" == t ]] && restore_active=true || restore_active=false + echo "Pausing the $JOB_NAME job during the repair." + psql_cron -tAc "SELECT cron.alter_job($job_id, active := false)" >/dev/null fi if [[ "$misconfigured" -gt 0 ]]; then @@ -178,7 +317,7 @@ if [[ "$misconfigured" -gt 0 ]]; then "UPDATE partman.part_config SET premake = GREATEST(premake, $PREMAKE), infinite_time_partitions = true - WHERE parent_table LIKE 'telegraf.%'" + WHERE $TELEGRAF_PARENTS" fi if [[ "$stranded" -gt 0 ]]; then @@ -186,27 +325,24 @@ if [[ "$stranded" -gt 0 ]]; then # One transaction: Telegraf is still writing, so a default emptied without # its partition in place is refilled before maintenance can attach one. psql_tg -v ON_ERROR_STOP=1 <<'EOSQL' +BEGIN; DO $$ DECLARE r record; BEGIN FOR r IN - SELECT pc.parent_table, - format('%I.%I', n.nspname, c.relname) AS default_table - FROM partman.part_config pc - JOIN pg_class p ON p.oid = pc.parent_table::regclass - JOIN pg_inherits i ON i.inhparent = p.oid - JOIN pg_class c ON c.oid = i.inhrelid - JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE pc.parent_table LIKE 'telegraf.%' - AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT' + SELECT pc.parent_table, d.default_part + FROM so_admin.telegraf_defaults() d + JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table + WHERE d.default_part IS NOT NULL LOOP - EXECUTE format('TRUNCATE TABLE %s', r.default_table); + EXECUTE format('TRUNCATE TABLE %s', r.default_part); PERFORM partman.create_partition_time( r.parent_table, ARRAY[date_trunc('day', now())]::timestamptz[]); END LOOP; END $$; +COMMIT; EOSQL fi @@ -222,7 +358,7 @@ if $restart_needed; then sleep 2 done docker exec so-postgres pg_isready -U postgres -q \ - || { echo "so-postgres did not come back; check 'docker logs so-postgres'." >&2; exit 1; } + || { echo "so-postgres did not come back; check 'docker logs so-postgres'." >&2; finish 1; } fi fi @@ -238,8 +374,14 @@ CALL partman.run_maintenance_proc(); \endif EOSQL +if [[ -n "$job_id" ]]; then + psql_cron -tAc "SELECT cron.alter_job($job_id, active := true)" >/dev/null + restore_active= + echo "The $JOB_NAME job is active." +fi + echo echo "Telegraf partition status after repair:" -psql_tg -c "$REPORT" +print_report echo "The $JOB_NAME job runs hourly at :17. Confirm it fired with:" echo " so-telegraf-repair --check" diff --git a/salt/soc/config.sls b/salt/soc/config.sls index be2887897..53ee7f40a 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -251,6 +251,16 @@ socsensoronirepos: - mode: 775 - makedirs: True +nsm_rules_dir: + file.directory: + - name: /nsm/rules + - user: socore + - group: socore + - recurse: + - user + - group + - show_changes: False + create_custom_local_yara_repo_template: git.present: - name: /nsm/rules/custom-local-repos/local-yara diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index ab51ac2a5..94bf0af81 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -9,6 +9,7 @@ {%- set MDENGINE = GLOBALS.md_engine %} {%- set LOGSTASH_ENABLED = LOGSTASH_MERGED.enabled %} {%- set TG_OUT = TELEGRAFMERGED.output | upper %} +{%- set PG_ONLY = TG_OUT == 'POSTGRES' %} {%- set PG_HOST = GLOBALS.manager_ip %} {#- Per-minion telegraf creds live in the grid-wide telegraf/creds.sls pillar, written by /usr/sbin/so-telegraf-cred on the manager. Each minion looks up @@ -138,6 +139,10 @@ # AGGREGATOR PLUGINS # ############################################################################### +[[aggregators.merge]] + drop_original = true + namepass = ["system", "swap", "kafka_partition"] + ############################################################################### # INPUT PLUGINS # ############################################################################### @@ -145,7 +150,12 @@ # Read metrics about cpu usage [[inputs.cpu]] ## Whether to report per-cpu stats or not +{%- if PG_ONLY %} + {#- SOC and so-stats-show read only cpu-total #} + percpu = false +{%- else %} percpu = true +{%- endif %} ## Whether to report total system cpu stats or not totalcpu = true ## If true, collect raw CPU time metrics. @@ -195,6 +205,11 @@ ## The typical use case is for LVM volumes, to get the VG/LV name instead of ## the near-meaningless DM-0 name. # name_templates = ["$ID_FS_LABEL","$DM_VG_NAME/$DM_LV_NAME"] +{%- if PG_ONLY %} + {#- Whole disks, md RAID included, already carry the I/O of their partitions and dm devices #} + [inputs.diskio.tagdrop] + name = ["sd*[0-9]", "vd*[0-9]", "xvd*[0-9]", "hd*[0-9]", "nvme*p[0-9]*", "mmcblk*p[0-9]*", "dm-*", "sr*", "loop*"] +{%- endif %} # Get kernel statistics from /proc/stat @@ -238,6 +253,12 @@ password = "{{ ES_PASS }}" insecure_skip_verify = true cluster_health = true +{%- if PG_ONLY %} + {#- SOC reads cluster status, indices and clusterstats_nodes ingest stats; alarms plan jvm and fs #} + cluster_health_level = "cluster" + node_stats = ["indices", "jvm", "fs"] + namedrop = ["elasticsearch_clusterstats_indices"] +{%- endif %} {%- elif grains['role'] in ['so-searchnode'] %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] @@ -245,6 +266,10 @@ username = "{{ ES_USER }}" password = "{{ ES_PASS }}" insecure_skip_verify = true +{%- if PG_ONLY %} + {#- SOC reads indices; alarms plan jvm and fs #} + node_stats = ["indices", "jvm", "fs"] +{%- endif %} {%- endif %} # ## Timeout for HTTP requests to the elastic search server(s) @@ -287,9 +312,13 @@ collect = ["pipelines"] username = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user') }}" password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}" +{%- if PG_ONLY %} + {#- SOC reads logstash_events only #} + namedrop = ["logstash_plugins"] +{%- endif %} {%- endif %} -{% if grains.role in ['so-manager','so-managersearch','so-standalone','so-receiver'] and GLOBALS.pipeline == "KAFKA" -%} +{% if grains.role in ['so-manager','so-managerhype','so-managersearch','so-standalone','so-receiver'] and GLOBALS.pipeline == "KAFKA" -%} [[inputs.jolokia2_agent]] name_prefix= "kafka_" urls = ["http://{{ NODEIP }}:8778/jolokia"] @@ -346,6 +375,26 @@ interval = "60s" {%- endif %} +{#- These report state that changes slowly; collecting them every interval only stores duplicate rows #} +{%- set SLOWSCRIPTS = [] %} +{%- for script in ['agentstatus.sh', 'os.sh', 'features.sh', 'raid.sh', 'zeekloss.sh'] %} +{%- if script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %} +{%- do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove(script) %} +{%- do SLOWSCRIPTS.append(script) %} +{%- endif %} +{%- endfor %} +{%- if SLOWSCRIPTS | length > 0 %} +[[inputs.exec]] + commands = [ +{%- for script in SLOWSCRIPTS %} + ["/scripts/{{script}}"]{% if not loop.last %},{% endif %} +{%- endfor %} + ] + data_format = "influx" + timeout = "15s" + interval = "5m" +{%- endif %} + {%- if TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] | length > 0 %} [[inputs.exec]] commands = [ @@ -358,12 +407,6 @@ timeout = "15s" {%- endif %} -{%- if salt['pillar.get']('healthcheck:enabled', False) %} -[[inputs.file]] - files = ["/host/nsm/zeek/logs/zeek_restart.log"] - data_format = "influx" -{%- endif %} - [[inputs.file]] files = ["/etc/telegraf/node_config.json"] name_override = "node_config" @@ -374,6 +417,12 @@ # # Read metrics about network interface usage [[inputs.net]] +{%- if PG_ONLY %} +{%- from 'docker/docker.map.jinja' import DOCKERMERGED %} + {#- Only virtual interfaces; every physical NIC, bond members included, is kept for SOC and alarms #} + [inputs.net.tagdrop] + interface = {{ (['veth*', 'docker*', 'lo'] + DOCKERMERGED.networks.keys() | list) | json }} +{%- endif %} # Scripts run every 30s||TELEGRAFMERGED.config.interval - ES index script doesn't need to run as frequently {%- if GLOBALS.is_manager or GLOBALS.role == 'so-heavynode' %} diff --git a/salt/telegraf/map.jinja b/salt/telegraf/map.jinja index fe2625bbc..511cd0b89 100644 --- a/salt/telegraf/map.jinja +++ b/salt/telegraf/map.jinja @@ -18,7 +18,7 @@ {% if GLOBALS.pipeline != 'REDIS' %} {# When global pipeline is not REDIS remove redis.sh script. KAFKA metrics are collected via jolokia agent. Config in telegraf.conf #} -{% if GLOBALS.role in ['so-standalone', 'so-manager', 'so-managersearch', 'so-receiver', 'so-heavynode'] %} +{% if GLOBALS.role in ['so-standalone', 'so-manager', 'so-managerhype', 'so-managersearch', 'so-receiver', 'so-heavynode'] %} {% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('redis.sh') %} {% endif %} {% endif %} diff --git a/salt/telegraf/scripts/checkfiles.sh b/salt/telegraf/scripts/checkfiles.sh index 6b2f1333d..bc7afb609 100644 --- a/salt/telegraf/scripts/checkfiles.sh +++ b/salt/telegraf/scripts/checkfiles.sh @@ -10,7 +10,7 @@ # if this script isn't already running if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then - FILES=$(ls -1x /host/nsm/strelka/unprocessed | wc -l) + FILES=$(find /host/nsm/strelka/unprocessed -maxdepth 1 -type f | wc -l) echo "faffiles files=$FILES" fi diff --git a/salt/telegraf/scripts/containers.sh b/salt/telegraf/scripts/containers.sh index 836d4a999..3ba991f78 100644 --- a/salt/telegraf/scripts/containers.sh +++ b/salt/telegraf/scripts/containers.sh @@ -12,8 +12,9 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then CONTAINERSLOG=/var/log/somon/containers.log # the collector rewrites this every minute; report nothing rather than repeating a stale - # file as if it were current, in case a run was skipped or the collector is wedged - MAXAGE=150 + # file as if it were current, in case a run was skipped or the collector is wedged. + # Kept under two collector cycles so a 60s telegraf interval can't emit the same snapshot twice. + MAXAGE=90 if [ -r "$CONTAINERSLOG" ]; then AGE=$(( $(date +%s) - $(stat -c %Y "$CONTAINERSLOG") )) diff --git a/salt/telegraf/scripts/eps.sh b/salt/telegraf/scripts/eps.sh index 10193cc38..fdaaa149b 100644 --- a/salt/telegraf/scripts/eps.sh +++ b/salt/telegraf/scripts/eps.sh @@ -11,27 +11,40 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then PREVCOUNTFILE='/tmp/eps.txt' - EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.events.in')" + STATS="$(curl -s localhost:9600/_node/stats)" - if [ ! -z "$EVENTCOUNTCURRENT" ]; then - - if [ -f "$PREVCOUNTFILE" ]; then - EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` - else - echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE - exit 0 - fi - - echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE - # the division by 30 is because the agent interval is 30 seconds - EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) - if [ "$EVENTS" -lt 0 ]; then - EVENTS=0 - fi - - echo "consumptioneps eps=${EVENTS%%.*}" + if echo "$STATS" | jq -e '.pipelines.search' > /dev/null 2>&1; then + STAGE=search + COUNT="$(echo "$STATS" | jq '.pipelines.search.events.out')" + elif echo "$STATS" | jq -e '.pipelines.receiver' > /dev/null 2>&1; then + STAGE=receiver + COUNT="$(echo "$STATS" | jq '.pipelines.receiver.events.in')" + else + exit 0 fi + if ! [[ "$COUNT" =~ ^[0-9]+$ ]]; then + exit 0 + fi + + NOW=$(date +%s) + read -r PREVEPOCH PREVCOUNT 2>/dev/null < "$PREVCOUNTFILE" + echo "${NOW} ${COUNT}" > "$PREVCOUNTFILE" + + if ! [[ "$PREVEPOCH" =~ ^[0-9]+$ && "$PREVCOUNT" =~ ^[0-9]+$ ]]; then + exit 0 + fi + + ELAPSED=$((NOW - PREVEPOCH)) + DELTA=$((COUNT - PREVCOUNT)) + if [ "$ELAPSED" -le 0 ] || [ "$DELTA" -lt 0 ]; then + EPS=0 + else + EPS=$((DELTA / ELAPSED)) + fi + + echo "consumptioneps,stage=${STAGE} eps=${EPS}" + fi exit 0 diff --git a/salt/telegraf/scripts/features.sh b/salt/telegraf/scripts/features.sh index 7c4fe6f52..bcc47ecb1 100644 --- a/salt/telegraf/scripts/features.sh +++ b/salt/telegraf/scripts/features.sh @@ -10,8 +10,7 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then FPS_ENABLED=$(cat /var/log/sostatus/fps_enabled) LKS_ENABLED=$(cat /var/log/sostatus/lks_enabled) - echo "features fps=$FPS_ENABLED" - echo "features lks=$LKS_ENABLED" + echo "features fps=$FPS_ENABLED,lks=$LKS_ENABLED" fi exit 0