Merge pull request #3871 from Security-Onion-Solutions/beatstats

Beatstats

salt/filebeat/etc/filebeat.yml

@@ -493,12 +493,13 @@ setup.template.enabled: false
 # append ?pretty to the URL.
 
 # Defines if the HTTP endpoint is enabled.
-#http.enabled: false
+http.enabled: true
 
 # The HTTP endpoint will bind to this hostname or IP address. It is recommended to use only localhost.
-#http.host: localhost
+http.host: 0.0.0.0
 
 # Port on which the HTTP endpoint will bind. Default is 5066.
+http.port: 5066
 
 queue.mem.events: {{ FBMEMEVENTS }}
 queue.mem.flush.min_events: {{ FBMEMFLUSHMINEVENTS }}

salt/filebeat/init.sls

@@ -74,6 +74,7 @@ so-filebeat:
     - port_bindings:
         - 0.0.0.0:514:514/udp
         - 0.0.0.0:514:514/tcp
+        - 0.0.0.0:5066:5066/tcp
     - watch:
       - file: /opt/so/conf/filebeat/etc/filebeat.yml
 

salt/firewall/portgroups.yaml

@@ -18,6 +18,9 @@ firewall:
       beats_5644:
         tcp:
           - 5644
+      beats_5066:
+        tcp:
+          - 5066
       cortex:
         tcp:
           - 9001

salt/telegraf/etc/telegraf.conf

@@ -679,7 +679,8 @@
       "/scripts/redis.sh",
       "/scripts/influxdbsize.sh",
       "/scripts/eps.sh",
-      "/scripts/raid.sh"
+      "/scripts/raid.sh",
+      "/beatseps.sh"
    ]
   data_format = "influx"
   ## Timeout for each command to complete.
@@ -695,7 +696,8 @@
       "/scripts/zeekcaptureloss.sh",
   {% endif %}
       "/scripts/oldpcap.sh",
-      "/scripts/raid.sh"
+      "/scripts/raid.sh",
+      "/beatseps.sh"
    ]
   data_format = "influx"
   timeout = "15s"
@@ -711,7 +713,8 @@
   {% endif %}
       "/scripts/oldpcap.sh",
       "/scripts/eps.sh",
-      "/scripts/raid.sh"
+      "/scripts/raid.sh",
+      "/beatseps.sh"
    ]
   data_format = "influx"
   timeout = "15s"
@@ -729,7 +732,8 @@
   {% endif %}
       "/scripts/oldpcap.sh",
       "/scripts/eps.sh",
-      "/scripts/raid.sh"
+      "/scripts/raid.sh",
+      "/beatseps.sh"
    ]
   data_format = "influx"
   timeout = "15s"
@@ -746,7 +750,8 @@
   {% endif %}
       "/scripts/oldpcap.sh",
       "/scripts/influxdbsize.sh",
-      "/scripts/raid.sh"
+      "/scripts/raid.sh",
+      "/beatseps.sh"
    ]
   data_format = "influx"
   timeout = "15s"

salt/telegraf/scripts/beatseps.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+APP=beatseps
+lf=/tmp/$APP-pidLockFile
+# create empty lock file if none exists
+cat /dev/null >> $lf
+read lastPID < $lf
+# if lastPID is not null and a process with that pid exists , exit
+[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
+echo $$ > $lf
+
+PREVCOUNTFILE='/tmp/beatseps.txt'
+EVENTCOUNTCURRENT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.acked')"
+FAILEDEVENTCOUNT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.failed')"
+
+if [ ! -z "$EVENTCOUNTCURRENT" ]; then
+
+  if [ -f "$PREVCOUNTFILE" ]; then
+    EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE`
+  else
+    echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE
+    exit 0
+  fi
+
+  echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE
+  # the division by 30 is because the agent interval is 30 seconds
+  EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30))
+  if [ "$EVENTS" -lt 0 ]; then
+    EVENTS=0
+  fi
+
+  echo "fbstats eps=${EVENTS%%.*},failed=$FAILEDEVENTCOUNT"
+
+fi
+
+exit 0