Merge pull request #16389 from Security-Onion-Solutions/reyesj2-patch-1

update elastic-agent artifact repo directory

salt/common/tools/sbin/so-status

@@ -17,7 +17,6 @@ import time
 EXPECTED_CONTAINERS_FILE = "/opt/so/conf/so-status/so-status.conf"
 HIGHSTATE_COMPLETE_FILE = "/opt/so/log/salt/lasthighstate"
 UPTIME_FILE = "/proc/uptime"
-POSTGRES_MAINTENANCE_STATUS_FILE = "/opt/so/log/postgres/maintenance.status"
 
 def showUsage(options, args):
   print('Usage: {} [OPTIONS]'.format(sys.argv[0]))
@@ -150,20 +149,6 @@ def check_container_status(options, console):
   return code, container_list
 
 
-def check_postgres_maintenance(code, container_list):
-  try:
-    with open(POSTGRES_MAINTENANCE_STATUS_FILE, "r") as status_file:
-      status = status_file.readline().strip()
-  except OSError:
-    return code
-  if status.startswith("failed"):
-    reason = status[len("failed"):].lstrip(":").strip()
-    container_list.append({ "Name": "so-postgres-maintenance", "Status": "failed", "Details": reason })
-    if code == 0:
-      code = 1
-  return code
-
-
 def check_status(options, console):
   container_list = []
   code = check_installation_status(options, console)
@@ -175,7 +160,6 @@ def check_status(options, console):
   # on the Grid until a highstate completes.
   if code == 0 or code == 2:
     container_code, container_list = check_container_status(options, console)
-    container_code = check_postgres_maintenance(container_code, container_list)
     if code == 0:
       code = container_code
   output(options, console, code, container_list)

salt/manager/tools/sbin/soup

@@ -546,7 +546,6 @@ postupgrade_changes() {
     [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0
     [[ "$POSTVERSION" == "3.2.0" ]] && post_to_3.3.0
     [[ "$POSTVERSION" == "3.3.0" ]] && post_to_3.4.0
-    [[ "$POSTVERSION" == "3.4.0" ]] && post_to_3.5.0
     # All applicable post-upgrade steps completed; clear the resume marker.
     rm -f "$POSTVERSION_FILE"
     true
@@ -1070,14 +1069,11 @@ telegraf_repair() {
 
   echo "Checking Telegraf metric partitions."
   local status=0
-  local max_age
+  "$repair" --check >> "$SOUP_LOG" 2>&1 || status=$?
-  max_age=$(lookup_pillar maintenance_max_age_hours postgres:telegraf 2>/dev/null) || true
-  max_age=${max_age:-3}
-  MAINTENANCE_MAX_AGE_HOURS="$max_age" "$repair" --check >> "$SOUP_LOG" 2>&1 || status=$?
   case "$status" in
     0) echo "  Telegraf partitions are healthy; nothing to repair." ;;
     1) echo "  Repairing stalled Telegraf partitions."
-       MAINTENANCE_MAX_AGE_HOURS="$max_age" "$repair" --yes \
+       "$repair" --yes \
          || echo "  warning: so-telegraf-repair failed; run it manually" >&2 ;;
     *) echo "  Skipping; Telegraf is not storing metrics in Postgres on this host." ;;
   esac
@@ -1206,14 +1202,6 @@ add_detection_runtime_fields() {
 }
 ### 3.4.0 End ###
 
-### 3.5.0 Scripts ###
-post_to_3.5.0() {
-  telegraf_repair
-
-  set_postversion 3.5.0
-}
-### 3.5.0 End ###
-
 
 repo_sync() {
   echo "Sync the local repo."

salt/nginx/enabled.sls

@@ -54,9 +54,9 @@ so-nginx:
       - /opt/so/log/nginx/:/var/log/nginx:rw
       - /opt/so/tmp/nginx/:/var/lib/nginx:rw
       - /opt/so/tmp/nginx/:/run:rw
-      - /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages
+      - /nsm/elastic-fleet/artifacts/:/opt/socore/elastic-artifacts/artifacts:ro
-      - /nsm/elastic-fleet/artifacts/:/opt/socore/html/artifacts 
       {% if GLOBALS.is_manager %}
+      - /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages
       - /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro
       - /etc/pki/managerssl.key:/etc/pki/nginx/server.key:ro
       # ATT&CK Navigator binds

salt/nginx/etc/nginx.conf

@@ -4,6 +4,8 @@
 {%- set role = grains.id.split('_') | last %}
 {%- set influxpass = salt['pillar.get']('secrets:influx_pass') %}
 {%- set influxauth = ('so:' + influxpass) | base64_encode %}
+{%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %}
+{%- set full_server_name = all_names | unique | join(' ') %}
 
 worker_processes auto;
 error_log /var/log/nginx/error.log;
@@ -39,52 +41,31 @@ http {
 
 	include /etc/nginx/conf.d/*.conf;
 
-	{%- if role in ['fleet'] %}
-
-	server {
-		listen 8443;
-		server_name {{ GLOBALS.hostname }};
-		root /opt/socore/html;
-		location /artifacts/ {
-			try_files $uri =206;
-			proxy_read_timeout    90;
-			proxy_connect_timeout 90;
-			proxy_set_header      Host $host;
-			proxy_set_header      X-Real-IP $remote_addr;
-			proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
-			proxy_set_header      Proxy "";
-			proxy_set_header      X-Forwarded-Proto $scheme;
-		}
-	}
-
-    {%- endif %}
-
 	{%- if GLOBALS.is_manager %}
-	{%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %}
-	{%- set full_server_name = all_names | unique | join(' ') %}
-
 	server {
 		listen 80 default_server;
 		server_name _;
 		return 307 https://{{ GLOBALS.url_base }}$request_uri;
 	}
-
+  {%- endif %}
+  {%- if GLOBALS.is_manager or role == 'fleet' %}
 	server {
 		listen 8443;
+    {%- if GLOBALS.is_manager %}
 		server_name {{ full_server_name }};
-		root /opt/socore/html;
+    {%- else %}
+    server_name {{ GLOBALS.hostname }};
+    {%- endif %}
+		root /opt/socore/elastic-artifacts;
 		location /artifacts/ {
-			try_files $uri =206;
+			try_files $uri =404;
-			proxy_read_timeout    90;
+		}
-			proxy_connect_timeout 90;
+		location / {
-			proxy_set_header      Host $host;
+			return 404;
-			proxy_set_header      X-Real-IP $remote_addr;
-			proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
-			proxy_set_header      Proxy "";
-			proxy_set_header      X-Forwarded-Proto $scheme;
 		}
 	}
-
+  {%- endif %}
+  {%- if GLOBALS.is_manager %}
 	server {
 		listen 443 ssl default_server;
 		http2 on;

salt/postgres/defaults.yaml

@@ -2,7 +2,6 @@ postgres:
   enabled: True
   telegraf:
     retention_days: 14
-    maintenance_max_age_hours: 3
   config:
     listen_addresses: '*'
     port: 5432
@@ -18,6 +17,3 @@ postgres:
     log_min_messages: 'warning'
     shared_preload_libraries: pg_cron
     cron.database_name: postgres
-    # pg_hba.conf only trusts local socket connections; over TCP pg_cron has no
-    # password to send and every job fails to connect.
-    cron.host: /run/postgresql

salt/postgres/disabled.sls

@@ -24,15 +24,6 @@ so_postgres_backup:
     - identifier: so_postgres_backup
     - user: root
 
-so_postgres_maintenance_status:
-  cron.absent:
-    - identifier: so_postgres_maintenance_status
-    - user: root
-
-so_postgres_maintenance_status_file:
-  file.absent:
-    - name: /opt/so/log/postgres/maintenance.status
-
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/postgres/enabled.sls

@@ -8,7 +8,6 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
-{%   from 'telegraf/map.jinja' import TELEGRAFMERGED %}
 {%   set SO_POSTGRES_USER = salt['pillar.get']('postgres:auth:users:so_postgres_user:user', 'so_postgres') %}
 
 include:
@@ -123,29 +122,6 @@ so_postgres_backup:
     - month: '*'
     - dayweek: '*'
 
-{% if TELEGRAFMERGED.output | upper in ['POSTGRES', 'BOTH'] %}
-{%   set max_age = salt['pillar.get']('postgres:telegraf:maintenance_max_age_hours', 3) | int %}
-so_postgres_maintenance_status:
-  cron.present:
-    - name: MAINTENANCE_MAX_AGE_HOURS={{ max_age }} /usr/sbin/so-telegraf-postgres status > /dev/null 2>&1
-    - identifier: so_postgres_maintenance_status
-    - user: root
-    - minute: '*/15'
-    - hour: '*'
-    - daymonth: '*'
-    - month: '*'
-    - dayweek: '*'
-{% else %}
-so_postgres_maintenance_status:
-  cron.absent:
-    - identifier: so_postgres_maintenance_status
-    - user: root
-
-so_postgres_maintenance_status_file:
-  file.absent:
-    - name: /opt/so/log/postgres/maintenance.status
-{% endif %}
-
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/postgres/soc_postgres.yaml

@@ -9,11 +9,6 @@ postgres:
       description: Number of days of Telegraf metrics to keep in the so_telegraf database. Older partitions are dropped hourly by pg_partman.
       forcedType: int
       helpLink: postgres
-    maintenance_max_age_hours:
-      description: Number of hours without a successful run of the hourly pg_partman maintenance job before the Telegraf metrics database is reported as unhealthy in grid status and repaired by soup.
-      forcedType: int
-      advanced: True
-      helpLink: postgres
   config:
     max_connections:
       description: Maximum number of concurrent PostgreSQL connections.
@@ -92,8 +87,3 @@ postgres:
       global: True
       advanced: True
       helpLink: postgres
-    cron.host:
-      description: Unix socket directory pg_cron uses to run scheduled jobs. Must be a local socket directory so jobs authenticate with the local trust rule; a TCP host requires a password and jobs will fail.
-      global: True
-      advanced: True
-      helpLink: postgres

salt/postgres/tools/sbin/so-telegraf-postgres

@@ -12,11 +12,7 @@ set -e
 #   retention    Reconcile partman retention and premake on telegraf parents.
 #                Env: RETENTION_DAYS.
 #   maintenance  Drain default partitions and run partman maintenance.
-#   check        Report partition and maintenance health. Non-zero if any parent
+#   check        Report partition health. Non-zero if any parent is unhealthy.
-#                is unhealthy or the maintenance job has no successful run within
-#                the last MAINTENANCE_MAX_AGE_HOURS (default 3).
-#   status       Run check and write `ok` or `failed: <reason>` to
-#                /opt/so/log/postgres/maintenance.status for so-status.
 #
 # A default partition holding rows for a day blocks creating that day's child,
 # so maintenance drains defaults before calling partman. Use so-telegraf-repair
@@ -193,11 +189,6 @@ SELECT cron.schedule_in_database(
   'CALL so_admin.telegraf_maintenance()',
   'so_telegraf'
 );
--- pg_cron copies cron.host into nodename only when a job is first created, so
--- jobs registered before cron.host changed keep connecting to the old host.
-UPDATE cron.job SET nodename = current_setting('cron.host')
- WHERE jobname = 'telegraf-partman-maintenance'
-   AND nodename IS DISTINCT FROM current_setting('cron.host');
 \else
 \echo 'pg_cron metadata database is not `postgres` yet; skipping job registration.'
 \endif
@@ -287,80 +278,16 @@ LIMIT 5;
 \echo 'pg_cron is not installed in this database.'
 \endif
 EOSQL
-    max_age="${MAINTENANCE_MAX_AGE_HOURS:-3}"
-    [[ "$max_age" =~ ^[0-9]+$ ]] || { echo "MAINTENANCE_MAX_AGE_HOURS must be an integer" >&2; exit 1; }
-    failures=()
     unhealthy=$(docker exec so-postgres psql -U postgres -d so_telegraf -tAc \
       "SELECT count(*) FROM so_admin.telegraf_partition_status()
        WHERE coalesce(default_rows, 0) > 0 OR coalesce(days_ahead, -1) < 1")
     if [ "${unhealthy:-1}" != "0" ]; then
-      failures+=("$unhealthy telegraf parent(s) unhealthy")
+      echo "so-telegraf-postgres check: $unhealthy telegraf parent(s) unhealthy" >&2
-    fi
-    has_cron=$(docker exec so-postgres psql -U postgres -d postgres -tAc \
-      "SELECT to_regclass('cron.job_run_details') IS NOT NULL")
-    if [ "$has_cron" != "t" ]; then
-      failures+=("pg_cron is not installed in the postgres database")
-    else
-      runs=$(docker exec so-postgres psql -U postgres -d postgres -tA -F '|' -c \
-        "SELECT (SELECT count(*) FROM cron.job WHERE jobname = 'telegraf-partman-maintenance'),
-                (SELECT count(*) FROM cron.job WHERE jobname = 'telegraf-partman-maintenance' AND active),
-                (SELECT count(*) FROM pg_stat_activity
-                  WHERE backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%'),
-                (SELECT count(*) FROM pg_stat_activity
-                  WHERE (backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%')
-                    AND backend_start < now() - make_interval(hours => $max_age)),
-                count(*),
-                count(*) FILTER (WHERE d.status = 'succeeded'
-                                   AND d.start_time > now() - make_interval(hours => $max_age)),
-                coalesce((array_agg(format('last run %s at %s: %s', d.status, d.start_time,
-                                           regexp_replace(coalesce(d.return_message, ''), '\s+', ' ', 'g'))
-                                    ORDER BY d.start_time DESC))[1], '')
-           FROM cron.job j
-           JOIN cron.job_run_details d ON d.jobid = j.jobid
-          WHERE j.jobname = 'telegraf-partman-maintenance'") || runs=""
-      IFS='|' read -r jobs active launchers old_launchers run_count recent_ok last_run <<< "$runs"
-      # With no runs recorded, a live launcher younger than N hours means the
-      # job was only just scheduled (fresh install or a restart for a config
-      # change) and has not reached its first :17 yet. A launcher older than
-      # that should have fired it by now. Tradeoff: a job first scheduled long
-      # after the launcher started reports failed until its first run.
-      if [ -z "$runs" ]; then
-        failures+=("could not read telegraf-partman-maintenance runs from cron.job_run_details")
-      elif [ "$jobs" = "0" ]; then
-        failures+=("telegraf-partman-maintenance is not scheduled")
-      elif [ "$active" = "0" ]; then
-        failures+=("telegraf-partman-maintenance is inactive")
-      elif [ "$launchers" = "0" ]; then
-        failures+=("pg_cron launcher is not running; telegraf-partman-maintenance cannot run")
-      elif [ "$run_count" = "0" ]; then
-        if [ "$old_launchers" != "0" ]; then
-          failures+=("telegraf-partman-maintenance has never run although pg_cron has been up for more than $max_age hour(s)")
-        fi
-      elif [ "$recent_ok" = "0" ]; then
-        failures+=("no successful telegraf-partman-maintenance run in the last $max_age hour(s); $last_run")
-      fi
-    fi
-    if [ "${#failures[@]}" -gt 0 ]; then
-      msg=$(printf '%s; ' "${failures[@]}")
-      echo "so-telegraf-postgres check: ${msg%; }" >&2
       exit 1
     fi
     echo "so-telegraf-postgres check: all telegraf parents healthy"
     ;;
 
-  status)
-    status_file=/opt/so/log/postgres/maintenance.status
-    tmp=$(mktemp "$status_file.XXXXXX")
-    if reason=$("$0" check 2>&1 >/dev/null); then
-      echo ok > "$tmp"
-    else
-      reason=$(printf '%s\n' "$reason" | tail -n 1)
-      echo "failed: ${reason#so-telegraf-postgres check: }" > "$tmp"
-    fi
-    chmod 644 "$tmp"
-    mv -f "$tmp" "$status_file"
-    ;;
-
   *)
     echo "Unknown subcommand: $cmd" >&2
     exit 1

salt/postgres/tools/sbin/so-telegraf-repair

@@ -15,9 +15,6 @@
 #   --yes         Skip the confirmation prompt (for soup and other automation).
 #   --no-restart  Never restart so-postgres, even if pg_cron's launcher is dead.
 #
-# Env: MAINTENANCE_MAX_AGE_HOURS (default 3) is how long the maintenance job may
-# go without a successful run before repair is needed.
-#
 # Exit status:
 #   0  healthy, or repair completed
 #   1  repair is needed (--check only)
@@ -28,8 +25,6 @@ set -e
 # Matches p_premake in telegraf.conf's create_parent template.
 PREMAKE=7
 JOB_NAME=telegraf-partman-maintenance
-MAX_AGE_HOURS="${MAINTENANCE_MAX_AGE_HOURS:-3}"
-[[ "$MAX_AGE_HOURS" =~ ^[0-9]+$ ]] || { echo "MAINTENANCE_MAX_AGE_HOURS must be an integer" >&2; exit 2; }
 
 CHECK_ONLY=false
 ASSUME_YES=false
@@ -123,30 +118,14 @@ launcher=$(psql_pg -tAc \
 # so_telegraf before the postgres state lands, postgres after.
 cron_db=$(docker exec so-postgres psql -U postgres -tAc \
   "SELECT current_setting('cron.database_name', true)" | tr -d '[:space:]')
-run_count=0
-recent_ok=0
 last_run=never
-has_run_details=$([[ -n "$cron_db" ]] && docker exec so-postgres psql -U postgres -d "$cron_db" -tAc \
+if [[ -n "$cron_db" ]]; then
-  "SELECT to_regclass('cron.job_run_details') IS NOT NULL" 2>/dev/null || echo f)
+  last_run=$(docker exec so-postgres psql -U postgres -d "$cron_db" -tAc \
-if [[ "$has_run_details" == t ]]; then
+    "SELECT coalesce(max(d.start_time)::text, 'never')
-  runs=$(docker exec so-postgres psql -U postgres -d "$cron_db" -tA -F '|' -c \
-    "SELECT count(*),
-            count(*) FILTER (WHERE d.status = 'succeeded'
-                               AND d.start_time > now() - make_interval(hours => $MAX_AGE_HOURS)),
-            coalesce((array_agg(format('%s at %s', d.status, d.start_time)
-                                ORDER BY d.start_time DESC))[1], 'never')
        FROM cron.job_run_details d JOIN cron.job j USING (jobid)
-      WHERE j.jobname = '$JOB_NAME'" 2>/dev/null) \
+      WHERE j.jobname = '$JOB_NAME'" 2>/dev/null | tr -d '[:space:]' || echo unknown)
-    || { echo "Could not read $JOB_NAME runs from cron.job_run_details; cannot judge maintenance health." >&2; exit 2; }
+  [[ -n "$last_run" ]] || last_run=never
-  IFS='|' read -r run_count recent_ok last_run <<< "$runs"
 fi
-# An unreadable run history exits 2 rather than 1: soup repairs on 1, which
-# truncates the default partitions, and a transient query error must not do that.
-# No runs at all means a freshly scheduled job (or pg_cron not loaded, handled
-# by restart_needed); the partition checks catch a job that never fires. Runs
-# with no recent success mean the job fires but fails, e.g. cannot connect.
-maintenance_failing=false
-[[ "$run_count" -gt 0 && "$recent_ok" -eq 0 ]] && maintenance_failing=true
 
 # A grid that has never written a metric has nothing to recover, and an empty
 # cron_db means pg_cron is not loaded at all, which no restart fixes.
@@ -158,7 +137,6 @@ repair_needed=false
 [[ "$behind" -gt 0 ]] && repair_needed=true
 [[ "$misconfigured" -gt 0 ]] && repair_needed=true
 $restart_needed && repair_needed=true
-$maintenance_failing && repair_needed=true
 
 echo "Telegraf partition status:"
 psql_tg -c "$REPORT"
@@ -179,7 +157,6 @@ if $CHECK_ONLY; then
   [[ "$behind" -gt 0 ]]      && echo "  * $behind parent(s) with no partition for the current window"
   [[ "$misconfigured" -gt 0 ]] && echo "  * $misconfigured parent(s) with stale partman settings"
   $restart_needed            && echo "  * pg_cron's launcher is dead; maintenance is not running at all"
-  $maintenance_failing       && echo "  * no successful $JOB_NAME run in the last $MAX_AGE_HOURS hour(s)"
   echo
   echo "Re-run without --check to repair."
   exit 1
@@ -264,6 +241,5 @@ EOSQL
 echo
 echo "Telegraf partition status after repair:"
 psql_tg -c "$REPORT"
-echo "The $JOB_NAME job runs hourly at :17. Until its next run succeeds, grid status"
+echo "The $JOB_NAME job runs hourly at :17. Confirm it fired with:"
-echo "may still report so-postgres-maintenance as failed. Confirm it fired with:"
 echo "  so-telegraf-repair --check"

salt/telegraf/etc/telegraf.conf

@@ -138,6 +138,10 @@
 #                            AGGREGATOR PLUGINS                               #
 ###############################################################################
 
+[[aggregators.merge]]
+  drop_original = true
+  namepass = ["system", "swap", "kafka_partition"]
+
 ###############################################################################
 #                            INPUT PLUGINS                                    #
 ###############################################################################
@@ -289,7 +293,7 @@
   password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}"
 {%- endif %}
 
-{% if grains.role in ['so-manager','so-managersearch','so-standalone','so-receiver'] and GLOBALS.pipeline == "KAFKA" -%}
+{% if grains.role in ['so-manager','so-managerhype','so-managersearch','so-standalone','so-receiver'] and GLOBALS.pipeline == "KAFKA" -%}
 [[inputs.jolokia2_agent]]
   name_prefix= "kafka_"
   urls = ["http://{{ NODEIP }}:8778/jolokia"]
@@ -346,6 +350,26 @@
    interval = "60s"
 {%- endif %}
 
+{#- These report state that changes slowly; collecting them every interval only stores duplicate rows #}
+{%- set SLOWSCRIPTS = [] %}
+{%- for script in ['agentstatus.sh', 'os.sh', 'features.sh', 'raid.sh', 'zeekloss.sh'] %}
+{%-   if script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %}
+{%-     do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove(script) %}
+{%-     do SLOWSCRIPTS.append(script) %}
+{%-   endif %}
+{%- endfor %}
+{%- if SLOWSCRIPTS | length > 0 %}
+[[inputs.exec]]
+   commands = [
+{%-   for script in SLOWSCRIPTS %}
+      ["/scripts/{{script}}"]{% if not loop.last %},{% endif %}
+{%-   endfor %}
+   ]
+   data_format = "influx"
+   timeout = "15s"
+   interval = "5m"
+{%- endif %}
+
 {%- if TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] | length > 0 %}
 [[inputs.exec]]
    commands = [
@@ -358,12 +382,6 @@
   timeout = "15s"
 {%- endif %}
 
-{%- if salt['pillar.get']('healthcheck:enabled', False) %}
-[[inputs.file]]
-  files = ["/host/nsm/zeek/logs/zeek_restart.log"]
-  data_format = "influx"
-{%- endif %}
-
 [[inputs.file]]
   files = ["/etc/telegraf/node_config.json"]
   name_override = "node_config"

salt/telegraf/map.jinja

@@ -18,7 +18,7 @@
 
 {% if GLOBALS.pipeline != 'REDIS' %}
 {# When global pipeline is not REDIS remove redis.sh script. KAFKA metrics are collected via jolokia agent. Config in telegraf.conf #}
-{%   if GLOBALS.role in ['so-standalone', 'so-manager', 'so-managersearch', 'so-receiver', 'so-heavynode'] %}
+{%   if GLOBALS.role in ['so-standalone', 'so-manager', 'so-managerhype', 'so-managersearch', 'so-receiver', 'so-heavynode'] %}
 {%     do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('redis.sh') %}
 {%   endif %}
 {% endif %}

salt/telegraf/scripts/checkfiles.sh

@@ -10,7 +10,7 @@
 # if this script isn't already running
 if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
 
-    FILES=$(ls -1x /host/nsm/strelka/unprocessed | wc -l)
+    FILES=$(find /host/nsm/strelka/unprocessed -maxdepth 1 -type f | wc -l)
     echo "faffiles files=$FILES"
     
 fi

salt/telegraf/scripts/containers.sh

@@ -12,8 +12,9 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
 
     CONTAINERSLOG=/var/log/somon/containers.log
     # the collector rewrites this every minute; report nothing rather than repeating a stale
-    # file as if it were current, in case a run was skipped or the collector is wedged
+    # file as if it were current, in case a run was skipped or the collector is wedged.
-    MAXAGE=150
+    # Kept under two collector cycles so a 60s telegraf interval can't emit the same snapshot twice.
+    MAXAGE=90
 
     if [ -r "$CONTAINERSLOG" ]; then
         AGE=$(( $(date +%s) - $(stat -c %Y "$CONTAINERSLOG") ))

salt/telegraf/scripts/eps.sh

@@ -11,27 +11,40 @@
 if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
 
     PREVCOUNTFILE='/tmp/eps.txt'
-    EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.events.in')"
+    STATS="$(curl -s localhost:9600/_node/stats)"
 
-    if [ ! -z "$EVENTCOUNTCURRENT" ]; then
+    if echo "$STATS" | jq -e '.pipelines.search' > /dev/null 2>&1; then
-
+      STAGE=search
-      if [ -f "$PREVCOUNTFILE" ]; then
+      COUNT="$(echo "$STATS" | jq '.pipelines.search.events.out')"
-        EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE`
+    elif echo "$STATS" | jq -e '.pipelines.receiver' > /dev/null 2>&1; then
+      STAGE=receiver
+      COUNT="$(echo "$STATS" | jq '.pipelines.receiver.events.in')"
     else
-        echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE
       exit 0
     fi
 
-      echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE
+    if ! [[ "$COUNT" =~ ^[0-9]+$ ]]; then
-      # the division by 30 is because the agent interval is 30 seconds
+      exit 0
-      EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30))
-      if [ "$EVENTS" -lt 0 ]; then
-        EVENTS=0
     fi
 
-      echo "consumptioneps eps=${EVENTS%%.*}"
+    NOW=$(date +%s)
+    read -r PREVEPOCH PREVCOUNT 2>/dev/null < "$PREVCOUNTFILE"
+    echo "${NOW} ${COUNT}" > "$PREVCOUNTFILE"
+
+    if ! [[ "$PREVEPOCH" =~ ^[0-9]+$ && "$PREVCOUNT" =~ ^[0-9]+$ ]]; then
+      exit 0
     fi
 
+    ELAPSED=$((NOW - PREVEPOCH))
+    DELTA=$((COUNT - PREVCOUNT))
+    if [ "$ELAPSED" -le 0 ] || [ "$DELTA" -lt 0 ]; then
+      EPS=0
+    else
+      EPS=$((DELTA / ELAPSED))
+    fi
+
+    echo "consumptioneps,stage=${STAGE} eps=${EPS}"
+
 fi
 
 exit 0

salt/telegraf/scripts/features.sh

@@ -10,8 +10,7 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
     FPS_ENABLED=$(cat /var/log/sostatus/fps_enabled)
     LKS_ENABLED=$(cat /var/log/sostatus/lks_enabled)
 
-    echo "features fps=$FPS_ENABLED"
+    echo "features fps=$FPS_ENABLED,lks=$LKS_ENABLED"
-    echo "features lks=$LKS_ENABLED"
 fi
 
 exit 0