Postgres: run pg_partman maintenance over the local socket and detect failed runs

salt/common/tools/sbin/so-status

@@ -17,6 +17,7 @@ import time
 EXPECTED_CONTAINERS_FILE = "/opt/so/conf/so-status/so-status.conf"
 HIGHSTATE_COMPLETE_FILE = "/opt/so/log/salt/lasthighstate"
 UPTIME_FILE = "/proc/uptime"
+POSTGRES_MAINTENANCE_STATUS_FILE = "/opt/so/log/postgres/maintenance.status"
 
 def showUsage(options, args):
   print('Usage: {} [OPTIONS]'.format(sys.argv[0]))
@@ -149,6 +150,20 @@ def check_container_status(options, console):
   return code, container_list
 
 
+def check_postgres_maintenance(code, container_list):
+  try:
+    with open(POSTGRES_MAINTENANCE_STATUS_FILE, "r") as status_file:
+      status = status_file.readline().strip()
+  except OSError:
+    return code
+  if status.startswith("failed"):
+    reason = status[len("failed"):].lstrip(":").strip()
+    container_list.append({ "Name": "so-postgres-maintenance", "Status": "failed", "Details": reason })
+    if code == 0:
+      code = 1
+  return code
+
+
 def check_status(options, console):
   container_list = []
   code = check_installation_status(options, console)
@@ -160,6 +175,7 @@ def check_status(options, console):
   # on the Grid until a highstate completes.
   if code == 0 or code == 2:
     container_code, container_list = check_container_status(options, console)
+    container_code = check_postgres_maintenance(container_code, container_list)
     if code == 0:
       code = container_code
   output(options, console, code, container_list)

salt/manager/tools/sbin/soup

@@ -546,6 +546,7 @@ postupgrade_changes() {
     [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0
     [[ "$POSTVERSION" == "3.2.0" ]] && post_to_3.3.0
     [[ "$POSTVERSION" == "3.3.0" ]] && post_to_3.4.0
+    [[ "$POSTVERSION" == "3.4.0" ]] && post_to_3.5.0
     # All applicable post-upgrade steps completed; clear the resume marker.
     rm -f "$POSTVERSION_FILE"
     true
@@ -1069,11 +1070,14 @@ telegraf_repair() {
 
   echo "Checking Telegraf metric partitions."
   local status=0
-  "$repair" --check >> "$SOUP_LOG" 2>&1 || status=$?
+  local max_age
+  max_age=$(lookup_pillar maintenance_max_age_hours postgres:telegraf 2>/dev/null) || true
+  max_age=${max_age:-3}
+  MAINTENANCE_MAX_AGE_HOURS="$max_age" "$repair" --check >> "$SOUP_LOG" 2>&1 || status=$?
   case "$status" in
     0) echo "  Telegraf partitions are healthy; nothing to repair." ;;
     1) echo "  Repairing stalled Telegraf partitions."
-       "$repair" --yes \
+       MAINTENANCE_MAX_AGE_HOURS="$max_age" "$repair" --yes \
          || echo "  warning: so-telegraf-repair failed; run it manually" >&2 ;;
     *) echo "  Skipping; Telegraf is not storing metrics in Postgres on this host." ;;
   esac
@@ -1202,6 +1206,14 @@ add_detection_runtime_fields() {
 }
 ### 3.4.0 End ###
 
+### 3.5.0 Scripts ###
+post_to_3.5.0() {
+  telegraf_repair
+
+  set_postversion 3.5.0
+}
+### 3.5.0 End ###
+
 
 repo_sync() {
   echo "Sync the local repo."

salt/nginx/enabled.sls

@@ -16,12 +16,6 @@ include:
   - nginx.config
   - nginx.sostatus
 
-fleet_beats_dir_nginx:
-  file.directory:
-    - name: /nsm/elastic-fleet/artifacts/beats
-    - makedirs: True
-    - show_changes: False
-
 {%   if GLOBALS.role != 'so-fleet' %}
 {%     set container_config = 'so-nginx' %}
 make-rule-dir-nginx:
@@ -60,9 +54,9 @@ so-nginx:
       - /opt/so/log/nginx/:/var/log/nginx:rw
       - /opt/so/tmp/nginx/:/var/lib/nginx:rw
       - /opt/so/tmp/nginx/:/run:rw
-      - /nsm/elastic-fleet/artifacts/beats/:/opt/socore/elastic-artifacts/artifacts/beats:ro
-      {% if GLOBALS.is_manager %}
       - /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages
+      - /nsm/elastic-fleet/artifacts/:/opt/socore/html/artifacts 
+      {% if GLOBALS.is_manager %}
       - /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro
       - /etc/pki/managerssl.key:/etc/pki/nginx/server.key:ro
       # ATT&CK Navigator binds
@@ -112,7 +106,6 @@ so-nginx:
       {% endif %}
     - require:
       - file: nginxconf
-      - file: fleet_beats_dir_nginx
       {% if GLOBALS.is_manager %}
       {%   if NGINXMERGED.ssl.replace_cert %}
       - file: managerssl_key

salt/nginx/etc/nginx.conf

@@ -4,8 +4,6 @@
 {%- set role = grains.id.split('_') | last %}
 {%- set influxpass = salt['pillar.get']('secrets:influx_pass') %}
 {%- set influxauth = ('so:' + influxpass) | base64_encode %}
-{%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %}
-{%- set full_server_name = all_names | unique | join(' ') %}
 
 worker_processes auto;
 error_log /var/log/nginx/error.log;
@@ -41,31 +39,52 @@ http {
 
 	include /etc/nginx/conf.d/*.conf;
 
+	{%- if role in ['fleet'] %}
+
+	server {
+		listen 8443;
+		server_name {{ GLOBALS.hostname }};
+		root /opt/socore/html;
+		location /artifacts/ {
+			try_files $uri =206;
+			proxy_read_timeout    90;
+			proxy_connect_timeout 90;
+			proxy_set_header      Host $host;
+			proxy_set_header      X-Real-IP $remote_addr;
+			proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
+			proxy_set_header      Proxy "";
+			proxy_set_header      X-Forwarded-Proto $scheme;
+		}
+	}
+
+    {%- endif %}
+
 	{%- if GLOBALS.is_manager %}
+	{%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %}
+	{%- set full_server_name = all_names | unique | join(' ') %}
+
 	server {
 		listen 80 default_server;
 		server_name _;
 		return 307 https://{{ GLOBALS.url_base }}$request_uri;
 	}
-  {%- endif %}
-  {%- if GLOBALS.is_manager or role == 'fleet' %}
+
 	server {
 		listen 8443;
-    {%- if GLOBALS.is_manager %}
 		server_name {{ full_server_name }};
-    {%- else %}
-    server_name {{ GLOBALS.hostname }};
-    {%- endif %}
-		root /opt/socore/elastic-artifacts;
-		location /artifacts/beats/ {
-			try_files $uri =404;
-		}
-		location / {
-			return 404;
+		root /opt/socore/html;
+		location /artifacts/ {
+			try_files $uri =206;
+			proxy_read_timeout    90;
+			proxy_connect_timeout 90;
+			proxy_set_header      Host $host;
+			proxy_set_header      X-Real-IP $remote_addr;
+			proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
+			proxy_set_header      Proxy "";
+			proxy_set_header      X-Forwarded-Proto $scheme;
 		}
 	}
-  {%- endif %}
-  {%- if GLOBALS.is_manager %}
+
 	server {
 		listen 443 ssl default_server;
 		http2 on;

salt/postgres/defaults.yaml

@@ -2,6 +2,7 @@ postgres:
   enabled: True
   telegraf:
     retention_days: 14
+    maintenance_max_age_hours: 3
   config:
     listen_addresses: '*'
     port: 5432
@@ -17,3 +18,6 @@ postgres:
     log_min_messages: 'warning'
     shared_preload_libraries: pg_cron
     cron.database_name: postgres
+    # pg_hba.conf only trusts local socket connections; over TCP pg_cron has no
+    # password to send and every job fails to connect.
+    cron.host: /run/postgresql

salt/postgres/disabled.sls

@@ -24,6 +24,15 @@ so_postgres_backup:
     - identifier: so_postgres_backup
     - user: root
 
+so_postgres_maintenance_status:
+  cron.absent:
+    - identifier: so_postgres_maintenance_status
+    - user: root
+
+so_postgres_maintenance_status_file:
+  file.absent:
+    - name: /opt/so/log/postgres/maintenance.status
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/postgres/enabled.sls

@@ -8,6 +8,7 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'ca/trust.map.jinja' import CATRUST %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
+{%   from 'telegraf/map.jinja' import TELEGRAFMERGED %}
 {%   set SO_POSTGRES_USER = salt['pillar.get']('postgres:auth:users:so_postgres_user:user', 'so_postgres') %}
 
 include:
@@ -122,6 +123,29 @@ so_postgres_backup:
     - month: '*'
     - dayweek: '*'
 
+{% if TELEGRAFMERGED.output | upper in ['POSTGRES', 'BOTH'] %}
+{%   set max_age = salt['pillar.get']('postgres:telegraf:maintenance_max_age_hours', 3) | int %}
+so_postgres_maintenance_status:
+  cron.present:
+    - name: MAINTENANCE_MAX_AGE_HOURS={{ max_age }} /usr/sbin/so-telegraf-postgres status > /dev/null 2>&1
+    - identifier: so_postgres_maintenance_status
+    - user: root
+    - minute: '*/15'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+{% else %}
+so_postgres_maintenance_status:
+  cron.absent:
+    - identifier: so_postgres_maintenance_status
+    - user: root
+
+so_postgres_maintenance_status_file:
+  file.absent:
+    - name: /opt/so/log/postgres/maintenance.status
+{% endif %}
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/postgres/soc_postgres.yaml

@@ -9,6 +9,11 @@ postgres:
       description: Number of days of Telegraf metrics to keep in the so_telegraf database. Older partitions are dropped hourly by pg_partman.
       forcedType: int
       helpLink: postgres
+    maintenance_max_age_hours:
+      description: Number of hours without a successful run of the hourly pg_partman maintenance job before the Telegraf metrics database is reported as unhealthy in grid status and repaired by soup.
+      forcedType: int
+      advanced: True
+      helpLink: postgres
   config:
     max_connections:
       description: Maximum number of concurrent PostgreSQL connections.
@@ -87,3 +92,8 @@ postgres:
       global: True
       advanced: True
       helpLink: postgres
+    cron.host:
+      description: Unix socket directory pg_cron uses to run scheduled jobs. Must be a local socket directory so jobs authenticate with the local trust rule; a TCP host requires a password and jobs will fail.
+      global: True
+      advanced: True
+      helpLink: postgres

salt/postgres/tools/sbin/so-telegraf-postgres

@@ -12,7 +12,11 @@ set -e
 #   retention    Reconcile partman retention and premake on telegraf parents.
 #                Env: RETENTION_DAYS.
 #   maintenance  Drain default partitions and run partman maintenance.
-#   check        Report partition health. Non-zero if any parent is unhealthy.
+#   check        Report partition and maintenance health. Non-zero if any parent
+#                is unhealthy or the maintenance job has no successful run within
+#                the last MAINTENANCE_MAX_AGE_HOURS (default 3).
+#   status       Run check and write `ok` or `failed: <reason>` to
+#                /opt/so/log/postgres/maintenance.status for so-status.
 #
 # A default partition holding rows for a day blocks creating that day's child,
 # so maintenance drains defaults before calling partman. Use so-telegraf-repair
@@ -189,6 +193,11 @@ SELECT cron.schedule_in_database(
   'CALL so_admin.telegraf_maintenance()',
   'so_telegraf'
 );
+-- pg_cron copies cron.host into nodename only when a job is first created, so
+-- jobs registered before cron.host changed keep connecting to the old host.
+UPDATE cron.job SET nodename = current_setting('cron.host')
+ WHERE jobname = 'telegraf-partman-maintenance'
+   AND nodename IS DISTINCT FROM current_setting('cron.host');
 \else
 \echo 'pg_cron metadata database is not `postgres` yet; skipping job registration.'
 \endif
@@ -278,16 +287,80 @@ LIMIT 5;
 \echo 'pg_cron is not installed in this database.'
 \endif
 EOSQL
+    max_age="${MAINTENANCE_MAX_AGE_HOURS:-3}"
+    [[ "$max_age" =~ ^[0-9]+$ ]] || { echo "MAINTENANCE_MAX_AGE_HOURS must be an integer" >&2; exit 1; }
+    failures=()
     unhealthy=$(docker exec so-postgres psql -U postgres -d so_telegraf -tAc \
       "SELECT count(*) FROM so_admin.telegraf_partition_status()
        WHERE coalesce(default_rows, 0) > 0 OR coalesce(days_ahead, -1) < 1")
     if [ "${unhealthy:-1}" != "0" ]; then
-      echo "so-telegraf-postgres check: $unhealthy telegraf parent(s) unhealthy" >&2
+      failures+=("$unhealthy telegraf parent(s) unhealthy")
+    fi
+    has_cron=$(docker exec so-postgres psql -U postgres -d postgres -tAc \
+      "SELECT to_regclass('cron.job_run_details') IS NOT NULL")
+    if [ "$has_cron" != "t" ]; then
+      failures+=("pg_cron is not installed in the postgres database")
+    else
+      runs=$(docker exec so-postgres psql -U postgres -d postgres -tA -F '|' -c \
+        "SELECT (SELECT count(*) FROM cron.job WHERE jobname = 'telegraf-partman-maintenance'),
+                (SELECT count(*) FROM cron.job WHERE jobname = 'telegraf-partman-maintenance' AND active),
+                (SELECT count(*) FROM pg_stat_activity
+                  WHERE backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%'),
+                (SELECT count(*) FROM pg_stat_activity
+                  WHERE (backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%')
+                    AND backend_start < now() - make_interval(hours => $max_age)),
+                count(*),
+                count(*) FILTER (WHERE d.status = 'succeeded'
+                                   AND d.start_time > now() - make_interval(hours => $max_age)),
+                coalesce((array_agg(format('last run %s at %s: %s', d.status, d.start_time,
+                                           regexp_replace(coalesce(d.return_message, ''), '\s+', ' ', 'g'))
+                                    ORDER BY d.start_time DESC))[1], '')
+           FROM cron.job j
+           JOIN cron.job_run_details d ON d.jobid = j.jobid
+          WHERE j.jobname = 'telegraf-partman-maintenance'") || runs=""
+      IFS='|' read -r jobs active launchers old_launchers run_count recent_ok last_run <<< "$runs"
+      # With no runs recorded, a live launcher younger than N hours means the
+      # job was only just scheduled (fresh install or a restart for a config
+      # change) and has not reached its first :17 yet. A launcher older than
+      # that should have fired it by now. Tradeoff: a job first scheduled long
+      # after the launcher started reports failed until its first run.
+      if [ -z "$runs" ]; then
+        failures+=("could not read telegraf-partman-maintenance runs from cron.job_run_details")
+      elif [ "$jobs" = "0" ]; then
+        failures+=("telegraf-partman-maintenance is not scheduled")
+      elif [ "$active" = "0" ]; then
+        failures+=("telegraf-partman-maintenance is inactive")
+      elif [ "$launchers" = "0" ]; then
+        failures+=("pg_cron launcher is not running; telegraf-partman-maintenance cannot run")
+      elif [ "$run_count" = "0" ]; then
+        if [ "$old_launchers" != "0" ]; then
+          failures+=("telegraf-partman-maintenance has never run although pg_cron has been up for more than $max_age hour(s)")
+        fi
+      elif [ "$recent_ok" = "0" ]; then
+        failures+=("no successful telegraf-partman-maintenance run in the last $max_age hour(s); $last_run")
+      fi
+    fi
+    if [ "${#failures[@]}" -gt 0 ]; then
+      msg=$(printf '%s; ' "${failures[@]}")
+      echo "so-telegraf-postgres check: ${msg%; }" >&2
       exit 1
     fi
     echo "so-telegraf-postgres check: all telegraf parents healthy"
     ;;
 
+  status)
+    status_file=/opt/so/log/postgres/maintenance.status
+    tmp=$(mktemp "$status_file.XXXXXX")
+    if reason=$("$0" check 2>&1 >/dev/null); then
+      echo ok > "$tmp"
+    else
+      reason=$(printf '%s\n' "$reason" | tail -n 1)
+      echo "failed: ${reason#so-telegraf-postgres check: }" > "$tmp"
+    fi
+    chmod 644 "$tmp"
+    mv -f "$tmp" "$status_file"
+    ;;
+
   *)
     echo "Unknown subcommand: $cmd" >&2
     exit 1

salt/postgres/tools/sbin/so-telegraf-repair

@@ -15,6 +15,9 @@
 #   --yes         Skip the confirmation prompt (for soup and other automation).
 #   --no-restart  Never restart so-postgres, even if pg_cron's launcher is dead.
 #
+# Env: MAINTENANCE_MAX_AGE_HOURS (default 3) is how long the maintenance job may
+# go without a successful run before repair is needed.
+#
 # Exit status:
 #   0  healthy, or repair completed
 #   1  repair is needed (--check only)
@@ -25,6 +28,8 @@ set -e
 # Matches p_premake in telegraf.conf's create_parent template.
 PREMAKE=7
 JOB_NAME=telegraf-partman-maintenance
+MAX_AGE_HOURS="${MAINTENANCE_MAX_AGE_HOURS:-3}"
+[[ "$MAX_AGE_HOURS" =~ ^[0-9]+$ ]] || { echo "MAINTENANCE_MAX_AGE_HOURS must be an integer" >&2; exit 2; }
 
 CHECK_ONLY=false
 ASSUME_YES=false
@@ -118,14 +123,30 @@ launcher=$(psql_pg -tAc \
 # so_telegraf before the postgres state lands, postgres after.
 cron_db=$(docker exec so-postgres psql -U postgres -tAc \
   "SELECT current_setting('cron.database_name', true)" | tr -d '[:space:]')
+run_count=0
+recent_ok=0
 last_run=never
-if [[ -n "$cron_db" ]]; then
-  last_run=$(docker exec so-postgres psql -U postgres -d "$cron_db" -tAc \
-    "SELECT coalesce(max(d.start_time)::text, 'never')
+has_run_details=$([[ -n "$cron_db" ]] && docker exec so-postgres psql -U postgres -d "$cron_db" -tAc \
+  "SELECT to_regclass('cron.job_run_details') IS NOT NULL" 2>/dev/null || echo f)
+if [[ "$has_run_details" == t ]]; then
+  runs=$(docker exec so-postgres psql -U postgres -d "$cron_db" -tA -F '|' -c \
+    "SELECT count(*),
+            count(*) FILTER (WHERE d.status = 'succeeded'
+                               AND d.start_time > now() - make_interval(hours => $MAX_AGE_HOURS)),
+            coalesce((array_agg(format('%s at %s', d.status, d.start_time)
+                                ORDER BY d.start_time DESC))[1], 'never')
        FROM cron.job_run_details d JOIN cron.job j USING (jobid)
-      WHERE j.jobname = '$JOB_NAME'" 2>/dev/null | tr -d '[:space:]' || echo unknown)
-  [[ -n "$last_run" ]] || last_run=never
+      WHERE j.jobname = '$JOB_NAME'" 2>/dev/null) \
+    || { echo "Could not read $JOB_NAME runs from cron.job_run_details; cannot judge maintenance health." >&2; exit 2; }
+  IFS='|' read -r run_count recent_ok last_run <<< "$runs"
 fi
+# An unreadable run history exits 2 rather than 1: soup repairs on 1, which
+# truncates the default partitions, and a transient query error must not do that.
+# No runs at all means a freshly scheduled job (or pg_cron not loaded, handled
+# by restart_needed); the partition checks catch a job that never fires. Runs
+# with no recent success mean the job fires but fails, e.g. cannot connect.
+maintenance_failing=false
+[[ "$run_count" -gt 0 && "$recent_ok" -eq 0 ]] && maintenance_failing=true
 
 # A grid that has never written a metric has nothing to recover, and an empty
 # cron_db means pg_cron is not loaded at all, which no restart fixes.
@@ -137,6 +158,7 @@ repair_needed=false
 [[ "$behind" -gt 0 ]] && repair_needed=true
 [[ "$misconfigured" -gt 0 ]] && repair_needed=true
 $restart_needed && repair_needed=true
+$maintenance_failing && repair_needed=true
 
 echo "Telegraf partition status:"
 psql_tg -c "$REPORT"
@@ -157,6 +179,7 @@ if $CHECK_ONLY; then
   [[ "$behind" -gt 0 ]]      && echo "  * $behind parent(s) with no partition for the current window"
   [[ "$misconfigured" -gt 0 ]] && echo "  * $misconfigured parent(s) with stale partman settings"
   $restart_needed            && echo "  * pg_cron's launcher is dead; maintenance is not running at all"
+  $maintenance_failing       && echo "  * no successful $JOB_NAME run in the last $MAX_AGE_HOURS hour(s)"
   echo
   echo "Re-run without --check to repair."
   exit 1
@@ -241,5 +264,6 @@ EOSQL
 echo
 echo "Telegraf partition status after repair:"
 psql_tg -c "$REPORT"
-echo "The $JOB_NAME job runs hourly at :17. Confirm it fired with:"
+echo "The $JOB_NAME job runs hourly at :17. Until its next run succeeds, grid status"
+echo "may still report so-postgres-maintenance as failed. Confirm it fired with:"
 echo "  so-telegraf-repair --check"

salt/telegraf/etc/telegraf.conf

@@ -138,10 +138,6 @@
 #                            AGGREGATOR PLUGINS                               #
 ###############################################################################
 
-[[aggregators.merge]]
-  drop_original = true
-  namepass = ["system", "swap", "kafka_partition"]
-
 ###############################################################################
 #                            INPUT PLUGINS                                    #
 ###############################################################################
@@ -293,7 +289,7 @@
   password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}"
 {%- endif %}
 
-{% if grains.role in ['so-manager','so-managerhype','so-managersearch','so-standalone','so-receiver'] and GLOBALS.pipeline == "KAFKA" -%}
+{% if grains.role in ['so-manager','so-managersearch','so-standalone','so-receiver'] and GLOBALS.pipeline == "KAFKA" -%}
 [[inputs.jolokia2_agent]]
   name_prefix= "kafka_"
   urls = ["http://{{ NODEIP }}:8778/jolokia"]
@@ -350,26 +346,6 @@
    interval = "60s"
 {%- endif %}
 
-{#- These report state that changes slowly; collecting them every interval only stores duplicate rows #}
-{%- set SLOWSCRIPTS = [] %}
-{%- for script in ['agentstatus.sh', 'os.sh', 'features.sh', 'raid.sh', 'zeekloss.sh'] %}
-{%-   if script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %}
-{%-     do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove(script) %}
-{%-     do SLOWSCRIPTS.append(script) %}
-{%-   endif %}
-{%- endfor %}
-{%- if SLOWSCRIPTS | length > 0 %}
-[[inputs.exec]]
-   commands = [
-{%-   for script in SLOWSCRIPTS %}
-      ["/scripts/{{script}}"]{% if not loop.last %},{% endif %}
-{%-   endfor %}
-   ]
-   data_format = "influx"
-   timeout = "15s"
-   interval = "5m"
-{%- endif %}
-
 {%- if TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] | length > 0 %}
 [[inputs.exec]]
    commands = [
@@ -382,6 +358,12 @@
   timeout = "15s"
 {%- endif %}
 
+{%- if salt['pillar.get']('healthcheck:enabled', False) %}
+[[inputs.file]]
+  files = ["/host/nsm/zeek/logs/zeek_restart.log"]
+  data_format = "influx"
+{%- endif %}
+
 [[inputs.file]]
   files = ["/etc/telegraf/node_config.json"]
   name_override = "node_config"

salt/telegraf/map.jinja

@@ -18,7 +18,7 @@
 
 {% if GLOBALS.pipeline != 'REDIS' %}
 {# When global pipeline is not REDIS remove redis.sh script. KAFKA metrics are collected via jolokia agent. Config in telegraf.conf #}
-{%   if GLOBALS.role in ['so-standalone', 'so-manager', 'so-managerhype', 'so-managersearch', 'so-receiver', 'so-heavynode'] %}
+{%   if GLOBALS.role in ['so-standalone', 'so-manager', 'so-managersearch', 'so-receiver', 'so-heavynode'] %}
 {%     do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('redis.sh') %}
 {%   endif %}
 {% endif %}

salt/telegraf/scripts/checkfiles.sh

@@ -10,7 +10,7 @@
 # if this script isn't already running
 if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
 
-    FILES=$(find /host/nsm/strelka/unprocessed -maxdepth 1 -type f | wc -l)
+    FILES=$(ls -1x /host/nsm/strelka/unprocessed | wc -l)
     echo "faffiles files=$FILES"
     
 fi

salt/telegraf/scripts/containers.sh

@@ -12,9 +12,8 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
 
     CONTAINERSLOG=/var/log/somon/containers.log
     # the collector rewrites this every minute; report nothing rather than repeating a stale
-    # file as if it were current, in case a run was skipped or the collector is wedged.
-    # Kept under two collector cycles so a 60s telegraf interval can't emit the same snapshot twice.
-    MAXAGE=90
+    # file as if it were current, in case a run was skipped or the collector is wedged
+    MAXAGE=150
 
     if [ -r "$CONTAINERSLOG" ]; then
         AGE=$(( $(date +%s) - $(stat -c %Y "$CONTAINERSLOG") ))

salt/telegraf/scripts/eps.sh

@@ -11,40 +11,27 @@
 if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
 
     PREVCOUNTFILE='/tmp/eps.txt'
-    STATS="$(curl -s localhost:9600/_node/stats)"
+    EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.events.in')"
 
-    if echo "$STATS" | jq -e '.pipelines.search' > /dev/null 2>&1; then
-      STAGE=search
-      COUNT="$(echo "$STATS" | jq '.pipelines.search.events.out')"
-    elif echo "$STATS" | jq -e '.pipelines.receiver' > /dev/null 2>&1; then
-      STAGE=receiver
-      COUNT="$(echo "$STATS" | jq '.pipelines.receiver.events.in')"
-    else
-      exit 0
+    if [ ! -z "$EVENTCOUNTCURRENT" ]; then
+
+      if [ -f "$PREVCOUNTFILE" ]; then
+        EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE`
+      else
+        echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE
+        exit 0
+      fi
+
+      echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE
+      # the division by 30 is because the agent interval is 30 seconds
+      EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30))
+      if [ "$EVENTS" -lt 0 ]; then
+        EVENTS=0
+      fi
+
+      echo "consumptioneps eps=${EVENTS%%.*}"
     fi
 
-    if ! [[ "$COUNT" =~ ^[0-9]+$ ]]; then
-      exit 0
-    fi
-
-    NOW=$(date +%s)
-    read -r PREVEPOCH PREVCOUNT 2>/dev/null < "$PREVCOUNTFILE"
-    echo "${NOW} ${COUNT}" > "$PREVCOUNTFILE"
-
-    if ! [[ "$PREVEPOCH" =~ ^[0-9]+$ && "$PREVCOUNT" =~ ^[0-9]+$ ]]; then
-      exit 0
-    fi
-
-    ELAPSED=$((NOW - PREVEPOCH))
-    DELTA=$((COUNT - PREVCOUNT))
-    if [ "$ELAPSED" -le 0 ] || [ "$DELTA" -lt 0 ]; then
-      EPS=0
-    else
-      EPS=$((DELTA / ELAPSED))
-    fi
-
-    echo "consumptioneps,stage=${STAGE} eps=${EPS}"
-
 fi
 
 exit 0

salt/telegraf/scripts/features.sh

@@ -10,7 +10,8 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
     FPS_ENABLED=$(cat /var/log/sostatus/fps_enabled)
     LKS_ENABLED=$(cat /var/log/sostatus/lks_enabled)
 
-    echo "features fps=$FPS_ENABLED,lks=$LKS_ENABLED"
+    echo "features fps=$FPS_ENABLED"
+    echo "features lks=$LKS_ENABLED"
 fi
 
 exit 0