From 31f193c397c26cc2cdf7d088f1c30795048d602a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 19 Apr 2021 12:36:46 -0400 Subject: [PATCH 1/5] Change EPS for Telegraf --- salt/telegraf/etc/telegraf.conf | 41 ++++++++++++++++-- salt/telegraf/scripts/beatseps.sh | 50 ++++++++++------------ salt/telegraf/scripts/checkfiles.sh | 18 ++++---- salt/telegraf/scripts/eps.sh | 50 ++++++++++------------ salt/telegraf/scripts/helixeps.sh | 48 ++++++++++----------- salt/telegraf/scripts/influxdbsize.sh | 18 ++++---- salt/telegraf/scripts/oldpcap.sh | 24 +++++------ salt/telegraf/scripts/raid.sh | 20 ++++----- salt/telegraf/scripts/redis.sh | 20 ++++----- salt/telegraf/scripts/sostatus.sh | 24 +++++------ salt/telegraf/scripts/stenoloss.sh | 50 +++++++++++----------- salt/telegraf/scripts/suriloss.sh | 49 ++++++++++------------ salt/telegraf/scripts/zeekcaptureloss.sh | 53 +++++++++++------------- salt/telegraf/scripts/zeekloss.sh | 53 +++++++++++------------- 14 files changed, 255 insertions(+), 263 deletions(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 0798fc920..dac9bf60e 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -17,6 +17,7 @@ {% set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') %} {% set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %} {% set UNIQUEID = salt['pillar.get']('sensor:uniqueid', '') %} +{%- set TRUE_CLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %} # Global tags can be specified here in key="value" format. [global_tags] @@ -614,18 +615,29 @@ # ## Use TLS but skip chain & host verification # # insecure_skip_verify = false +{% if TRUE_CLUSTER %} + {% if grains.role == 'so-manager' %} +[[inputs.elasticsearch]] + servers = ["https://{{ MANAGER }}:9200"] + insecure_skip_verify = true + local = false + cluster_health = true + cluster_stats = true + {% endif %} + +{% else %} # # Read stats from one or more Elasticsearch servers or clusters -{% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %} + {% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %} [[inputs.elasticsearch]] servers = ["https://{{ MANAGER }}:9200"] insecure_skip_verify = true -{% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %} + {% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] insecure_skip_verify = true + {% endif %} {% endif %} - # # ## Timeout for HTTP requests to the elastic search server(s) # http_timeout = "5s" @@ -673,11 +685,32 @@ # ## Commands array -{% if grains['role'] in ['so-manager', 'so-managersearch'] %} +{% if grains['role'] in ['so-manager'] %} [[inputs.exec]] commands = [ "/scripts/redis.sh", "/scripts/influxdbsize.sh", + "/scripts/raid.sh", + "/scripts/beatseps.sh" + ] + data_format = "influx" + ## Timeout for each command to complete. + timeout = "15s" +{% elif grains['role'] in ['so-managersearch'] %} +[[inputs.exec]] + commands = [ + "/scripts/redis.sh", + "/scripts/influxdbsize.sh", + "/scripts/eps.sh", + "/scripts/raid.sh", + "/scripts/beatseps.sh" + ] + data_format = "influx" + ## Timeout for each command to complete. + timeout = "15s" +{% elif grains['role'] in ['so-node'] %} +[[inputs.exec]] + commands = [ "/scripts/eps.sh", "/scripts/raid.sh", "/scripts/beatseps.sh" diff --git a/salt/telegraf/scripts/beatseps.sh b/salt/telegraf/scripts/beatseps.sh index faba0fabc..1226c42e4 100644 --- a/salt/telegraf/scripts/beatseps.sh +++ b/salt/telegraf/scripts/beatseps.sh @@ -15,37 +15,33 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=beatseps -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -PREVCOUNTFILE='/tmp/beatseps.txt' -EVENTCOUNTCURRENT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.acked')" -FAILEDEVENTCOUNT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.failed')" +if [ ! $THEGREP ]; then -if [ ! -z "$EVENTCOUNTCURRENT" ]; then + PREVCOUNTFILE='/tmp/beatseps.txt' + EVENTCOUNTCURRENT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.acked')" + FAILEDEVENTCOUNT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.failed')" + + if [ ! -z "$EVENTCOUNTCURRENT" ]; then + + if [ -f "$PREVCOUNTFILE" ]; then + EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` + else + echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + exit 0 + fi - if [ -f "$PREVCOUNTFILE" ]; then - EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` - else echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + # the division by 30 is because the agent interval is 30 seconds + EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) + if [ "$EVENTS" -lt 0 ]; then + EVENTS=0 + fi + + echo "fbstats eps=${EVENTS%%.*},failed=$FAILEDEVENTCOUNT" + +else exit 0 - fi - - echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE - # the division by 30 is because the agent interval is 30 seconds - EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) - if [ "$EVENTS" -lt 0 ]; then - EVENTS=0 - fi - - echo "fbstats eps=${EVENTS%%.*},failed=$FAILEDEVENTCOUNT" - fi -exit 0 diff --git a/salt/telegraf/scripts/checkfiles.sh b/salt/telegraf/scripts/checkfiles.sh index c84b6bec9..12cf3ece6 100644 --- a/salt/telegraf/scripts/checkfiles.sh +++ b/salt/telegraf/scripts/checkfiles.sh @@ -15,15 +15,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=checkfiles -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -FILES=$(ls -1x /host/nsm/faf/complete/ | wc -l) +if [ ! $THEGREP ]; then -echo "faffiles files=$FILES" + FILES=$(ls -1x /host/nsm/faf/complete/ | wc -l) + + echo "faffiles files=$FILES" +else + exit 0 +fi \ No newline at end of file diff --git a/salt/telegraf/scripts/eps.sh b/salt/telegraf/scripts/eps.sh index dcc4b9051..99f001552 100644 --- a/salt/telegraf/scripts/eps.sh +++ b/salt/telegraf/scripts/eps.sh @@ -15,36 +15,32 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=eps -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -PREVCOUNTFILE='/tmp/eps.txt' -EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.events.in')" +if [ ! $THEGREP ]; then -if [ ! -z "$EVENTCOUNTCURRENT" ]; then + PREVCOUNTFILE='/tmp/eps.txt' + EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.events.in')" - if [ -f "$PREVCOUNTFILE" ]; then - EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` - else - echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + if [ ! -z "$EVENTCOUNTCURRENT" ]; then + + if [ -f "$PREVCOUNTFILE" ]; then + EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` + else + echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + exit 0 + fi + + echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + # the division by 30 is because the agent interval is 30 seconds + EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) + if [ "$EVENTS" -lt 0 ]; then + EVENTS=0 + fi + + echo "consumptioneps eps=${EVENTS%%.*}" + +else exit 0 - fi - - echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE - # the division by 30 is because the agent interval is 30 seconds - EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) - if [ "$EVENTS" -lt 0 ]; then - EVENTS=0 - fi - - echo "esteps eps=${EVENTS%%.*}" - fi -exit 0 diff --git a/salt/telegraf/scripts/helixeps.sh b/salt/telegraf/scripts/helixeps.sh index be5aaa1d2..7922a7ab6 100644 --- a/salt/telegraf/scripts/helixeps.sh +++ b/salt/telegraf/scripts/helixeps.sh @@ -15,35 +15,29 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=helixeps -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -PREVCOUNTFILE='/tmp/helixevents.txt' -EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.pipelines.helix.events.out')" +if [ ! $THEGREP ]; then -if [ ! -z "$EVENTCOUNTCURRENT" ]; then + PREVCOUNTFILE='/tmp/helixevents.txt' + EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.pipelines.helix.events.out')" + + if [ ! -z "$EVENTCOUNTCURRENT" ]; then + + if [ -f "$PREVCOUNTFILE" ]; then + EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` + else + echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + exit 0 + fi - if [ -f "$PREVCOUNTFILE" ]; then - EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` - else echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) + if [ "$EVENTS" -lt 0 ]; then + EVENTS=0 + fi + + echo "helixeps eps=${EVENTS%%.*}" +else exit 0 - fi - - echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE - EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) - if [ "$EVENTS" -lt 0 ]; then - EVENTS=0 - fi - - echo "helixeps eps=${EVENTS%%.*}" - -fi - -exit 0 +fi \ No newline at end of file diff --git a/salt/telegraf/scripts/influxdbsize.sh b/salt/telegraf/scripts/influxdbsize.sh index 9bab7815b..f2ed41f35 100644 --- a/salt/telegraf/scripts/influxdbsize.sh +++ b/salt/telegraf/scripts/influxdbsize.sh @@ -15,15 +15,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=influxsize -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -INFLUXSIZE=$(du -s -k /host/nsm/influxdb | awk {'print $1'}) +if [ ! $THEGREP ]; then -echo "influxsize kbytes=$INFLUXSIZE" + INFLUXSIZE=$(du -s -k /host/nsm/influxdb | awk {'print $1'}) + + echo "influxsize kbytes=$INFLUXSIZE" +else + exit 0 +fi \ No newline at end of file diff --git a/salt/telegraf/scripts/oldpcap.sh b/salt/telegraf/scripts/oldpcap.sh index 0557137e7..d43f16d14 100644 --- a/salt/telegraf/scripts/oldpcap.sh +++ b/salt/telegraf/scripts/oldpcap.sh @@ -15,18 +15,16 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=oldpcap -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -# Get the data -OLDPCAP=$(find /host/nsm/pcap -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'}) -DATE=$(date +%s) -AGE=$(($DATE - $OLDPCAP)) +if [ ! $THEGREP ]; then -echo "pcapage seconds=$AGE" + # Get the data + OLDPCAP=$(find /host/nsm/pcap -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'}) + DATE=$(date +%s) + AGE=$(($DATE - $OLDPCAP)) + + echo "pcapage seconds=$AGE" +else + exit 0 +fi \ No newline at end of file diff --git a/salt/telegraf/scripts/raid.sh b/salt/telegraf/scripts/raid.sh index 0938bb658..c0aabe75f 100644 --- a/salt/telegraf/scripts/raid.sh +++ b/salt/telegraf/scripts/raid.sh @@ -15,19 +15,15 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=raid -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf -RAIDLOG=/var/log/raid/status.log -RAIDSTATUS=$(cat /var/log/raid/status.log) +THEGREP=$(ps -ef | grep $0 | grep -v grep) -if [ -f "$RAIDLOG" ]; then - echo "raid $RAIDSTATUS" +if [ ! $THEGREP ]; then + + if [ -f "$RAIDLOG" ]; then + echo "raid $RAIDSTATUS" + else + exit 0 + fi else exit 0 fi diff --git a/salt/telegraf/scripts/redis.sh b/salt/telegraf/scripts/redis.sh index 04079c63b..613e6cdea 100644 --- a/salt/telegraf/scripts/redis.sh +++ b/salt/telegraf/scripts/redis.sh @@ -16,16 +16,14 @@ # along with this program. If not, see . -APP=redis -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -UNPARSED=$(redis-cli llen logstash:unparsed | awk '{print $1}') -PARSED=$(redis-cli llen logstash:parsed | awk '{print $1}') +if [ ! $THEGREP ]; then -echo "redisqueue unparsed=$UNPARSED,parsed=$PARSED" + UNPARSED=$(redis-cli llen logstash:unparsed | awk '{print $1}') + PARSED=$(redis-cli llen logstash:parsed | awk '{print $1}') + + echo "redisqueue unparsed=$UNPARSED,parsed=$PARSED" +else + exit 0 +fi diff --git a/salt/telegraf/scripts/sostatus.sh b/salt/telegraf/scripts/sostatus.sh index 23096d903..1baf38d2b 100644 --- a/salt/telegraf/scripts/sostatus.sh +++ b/salt/telegraf/scripts/sostatus.sh @@ -14,20 +14,18 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . +THEGREP=$(ps -ef | grep $0 | grep -v grep) -APP=sostatus -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf -SOSTATUSLOG=/var/log/sostatus/status.log -SOSTATUSSTATUS=$(cat /var/log/sostatus/status.log) +if [ ! $THEGREP ]; then -if [ -f "$SOSTATUSLOG" ]; then - echo "sostatus status=$SOSTATUSSTATUS" -else + SOSTATUSLOG=/var/log/sostatus/status.log + SOSTATUSSTATUS=$(cat /var/log/sostatus/status.log) + + if [ -f "$SOSTATUSLOG" ]; then + echo "sostatus status=$SOSTATUSSTATUS" + else + exit 0 + fi +else exit 0 fi diff --git a/salt/telegraf/scripts/stenoloss.sh b/salt/telegraf/scripts/stenoloss.sh index ad88ccc8d..a5c974a73 100644 --- a/salt/telegraf/scripts/stenoloss.sh +++ b/salt/telegraf/scripts/stenoloss.sh @@ -15,31 +15,29 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -APP=stenoloss -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -TSFILE=/var/log/telegraf/laststenodrop.log -if [ -f "$TSFILE" ]; then - LASTTS=$(cat $TSFILE) +if [ ! $THEGREP ]; then + + TSFILE=/var/log/telegraf/laststenodrop.log + if [ -f "$TSFILE" ]; then + LASTTS=$(cat $TSFILE) + else + LASTTS=0 + fi + + # Get the data + LOGLINE=$(tac /var/log/stenographer/stenographer.log | grep -m1 drop) + CURRENTTS=$(echo $LOGLINE | awk '{print $1}') + + if [[ "$CURRENTTS" != "$LASTTS" ]]; then + DROP=$(echo $LOGLINE | awk '{print $14}' | awk -F "=" '{print $2}') + echo $CURRENTTS > $TSFILE + else + DROP=0 + fi + + echo "stenodrop drop=$DROP" else - LASTTS=0 -fi - -# Get the data -LOGLINE=$(tac /var/log/stenographer/stenographer.log | grep -m1 drop) -CURRENTTS=$(echo $LOGLINE | awk '{print $1}') - -if [[ "$CURRENTTS" != "$LASTTS" ]]; then - DROP=$(echo $LOGLINE | awk '{print $14}' | awk -F "=" '{print $2}') - echo $CURRENTTS > $TSFILE -else - DROP=0 -fi - -echo "stenodrop drop=$DROP" \ No newline at end of file + exit 0 +fi \ No newline at end of file diff --git a/salt/telegraf/scripts/suriloss.sh b/salt/telegraf/scripts/suriloss.sh index 08f8c23eb..0ae867b29 100644 --- a/salt/telegraf/scripts/suriloss.sh +++ b/salt/telegraf/scripts/suriloss.sh @@ -16,37 +16,32 @@ # along with this program. If not, see . -APP=suriloss -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -SURILOG=$(tac /var/log/suricata/stats.log | grep kernel | head -4) -CHECKIT=$(echo $SURILOG | grep -o 'drop' | wc -l) +if [ ! $THEGREP ]; then -if [ $CHECKIT == 2 ]; then - declare RESULT=($SURILOG) + SURILOG=$(tac /var/log/suricata/stats.log | grep kernel | head -4) + CHECKIT=$(echo $SURILOG | grep -o 'drop' | wc -l) - CURRENTDROP=${RESULT[4]} - PASTDROP=${RESULT[14]} - DROPPED=$((CURRENTDROP - PASTDROP)) - if [ $DROPPED == 0 ]; then - LOSS=0 - echo "suridrop drop=0" - else - CURRENTPACKETS=${RESULT[9]} - PASTPACKETS=${RESULT[19]} - TOTALCURRENT=$((CURRENTPACKETS + CURRENTDROP)) - TOTALPAST=$((PASTPACKETS + PASTDROP)) - TOTAL=$((TOTALCURRENT - TOTALPAST)) + if [ $CHECKIT == 2 ]; then + declare RESULT=($SURILOG) - LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc) - echo "suridrop drop=$LOSS" - fi + CURRENTDROP=${RESULT[4]} + PASTDROP=${RESULT[14]} + DROPPED=$((CURRENTDROP - PASTDROP)) + if [ $DROPPED == 0 ]; then + LOSS=0 + echo "suridrop drop=0" + else + CURRENTPACKETS=${RESULT[9]} + PASTPACKETS=${RESULT[19]} + TOTALCURRENT=$((CURRENTPACKETS + CURRENTDROP)) + TOTALPAST=$((PASTPACKETS + PASTDROP)) + TOTAL=$((TOTALCURRENT - TOTALPAST)) + + LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc) + echo "suridrop drop=$LOSS" + fi else echo "suridrop drop=0" fi \ No newline at end of file diff --git a/salt/telegraf/scripts/zeekcaptureloss.sh b/salt/telegraf/scripts/zeekcaptureloss.sh index aa8a222a3..995971b18 100644 --- a/salt/telegraf/scripts/zeekcaptureloss.sh +++ b/salt/telegraf/scripts/zeekcaptureloss.sh @@ -18,35 +18,32 @@ # This script returns the average of all the workers average capture loss to telegraf / influxdb in influx format include nanosecond precision timestamp -APP=zeekcaploss -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -if [ -d "/host/nsm/zeek/spool/logger" ]; then - WORKERS={{ salt['pillar.get']('sensor:zeek_lbprocs', salt['pillar.get']('sensor:zeek_pins') | length) }} - ZEEKLOG=/host/nsm/zeek/spool/logger/capture_loss.log -elif [ -d "/host/nsm/zeek/spool/zeeksa" ]; then - WORKERS=1 - ZEEKLOG=/host/nsm/zeek/spool/zeeksa/capture_loss.log -else - echo 'Zeek capture_loss.log not found' >/dev/stderr - exit 2 -fi +if [ ! $THEGREP ]; then -LASTCAPTURELOSSLOG=/var/log/telegraf/lastcaptureloss.txt -if [ -f "$ZEEKLOG" ]; then - CURRENTTS=$(tail -1 $ZEEKLOG | jq .ts | sed 's/"//g') - if [ -f "$LASTCAPTURELOSSLOG" ]; then - LASTTS=$(cat $LASTCAPTURELOSSLOG) - if [[ "$LASTTS" != "$CURRENTTS" ]]; then - LOSS=$(tail -$WORKERS $ZEEKLOG | awk -F, '{print $NF}' | sed 's/}//' | awk -v WORKERS=$WORKERS -F: '{LOSS += $2 / WORKERS} END { print LOSS}') - echo "zeekcaptureloss loss=$LOSS" + if [ -d "/host/nsm/zeek/spool/logger" ]; then + WORKERS={{ salt['pillar.get']('sensor:zeek_lbprocs', salt['pillar.get']('sensor:zeek_pins') | length) }} + ZEEKLOG=/host/nsm/zeek/spool/logger/capture_loss.log + elif [ -d "/host/nsm/zeek/spool/zeeksa" ]; then + WORKERS=1 + ZEEKLOG=/host/nsm/zeek/spool/zeeksa/capture_loss.log + else + echo 'Zeek capture_loss.log not found' >/dev/stderr + exit 2 fi - fi - echo "$CURRENTTS" > $LASTCAPTURELOSSLOG + + LASTCAPTURELOSSLOG=/var/log/telegraf/lastcaptureloss.txt + if [ -f "$ZEEKLOG" ]; then + CURRENTTS=$(tail -1 $ZEEKLOG | jq .ts | sed 's/"//g') + if [ -f "$LASTCAPTURELOSSLOG" ]; then + LASTTS=$(cat $LASTCAPTURELOSSLOG) + if [[ "$LASTTS" != "$CURRENTTS" ]]; then + LOSS=$(tail -$WORKERS $ZEEKLOG | awk -F, '{print $NF}' | sed 's/}//' | awk -v WORKERS=$WORKERS -F: '{LOSS += $2 / WORKERS} END { print LOSS}') + echo "zeekcaptureloss loss=$LOSS" + fi + fi + echo "$CURRENTTS" > $LASTCAPTURELOSSLOG +else + exit 0 fi diff --git a/salt/telegraf/scripts/zeekloss.sh b/salt/telegraf/scripts/zeekloss.sh index 0c1a714ba..559c6b15f 100644 --- a/salt/telegraf/scripts/zeekloss.sh +++ b/salt/telegraf/scripts/zeekloss.sh @@ -17,34 +17,31 @@ # This script returns the packets dropped by Zeek, but it isn't a percentage. $LOSS * 100 would be the percentage -APP=zeekloss -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf +THEGREP=$(ps -ef | grep $0 | grep -v grep) -ZEEKLOG=$(tac /host/nsm/zeek/logs/packetloss.log | head -2) -declare RESULT=($ZEEKLOG) -CURRENTDROP=${RESULT[3]} -# zeek likely not running if this is true -if [[ $CURRENTDROP == "rcvd:" ]]; then - CURRENTDROP=0 - PASTDROP=0 - DROPPED=0 +if [ ! $THEGREP ]; then + + ZEEKLOG=$(tac /host/nsm/zeek/logs/packetloss.log | head -2) + declare RESULT=($ZEEKLOG) + CURRENTDROP=${RESULT[3]} + # zeek likely not running if this is true + if [[ $CURRENTDROP == "rcvd:" ]]; then + CURRENTDROP=0 + PASTDROP=0 + DROPPED=0 + else + PASTDROP=${RESULT[9]} + DROPPED=$((CURRENTDROP - PASTDROP)) + fi + if [[ "$DROPPED" -le 0 ]]; then + LOSS=0 + echo "zeekdrop drop=0" + else + CURRENTPACKETS=${RESULT[5]} + PASTPACKETS=${RESULT[11]} + TOTAL=$((CURRENTPACKETS - PASTPACKETS)) + LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc) + echo "zeekdrop drop=$LOSS" else - PASTDROP=${RESULT[9]} - DROPPED=$((CURRENTDROP - PASTDROP)) -fi -if [[ "$DROPPED" -le 0 ]]; then - LOSS=0 - echo "zeekdrop drop=0" -else - CURRENTPACKETS=${RESULT[5]} - PASTPACKETS=${RESULT[11]} - TOTAL=$((CURRENTPACKETS - PASTPACKETS)) - LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc) - echo "zeekdrop drop=$LOSS" + exit 0 fi \ No newline at end of file From f5b04117728780797bdd1d7c50c9375fd6f10f27 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 19 Apr 2021 13:11:19 -0400 Subject: [PATCH 2/5] Change EPS for Telegraf --- salt/telegraf/scripts/beatseps.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/telegraf/scripts/beatseps.sh b/salt/telegraf/scripts/beatseps.sh index 1226c42e4..50cfeff91 100644 --- a/salt/telegraf/scripts/beatseps.sh +++ b/salt/telegraf/scripts/beatseps.sh @@ -40,6 +40,7 @@ if [ ! $THEGREP ]; then fi echo "fbstats eps=${EVENTS%%.*},failed=$FAILEDEVENTCOUNT" + fi else exit 0 From 54322f5e9d608d0f8feade99bd8b23ee29322a50 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 19 Apr 2021 13:17:02 -0400 Subject: [PATCH 3/5] Change EPS for Telegraf --- salt/telegraf/scripts/beatseps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/telegraf/scripts/beatseps.sh b/salt/telegraf/scripts/beatseps.sh index 50cfeff91..22189e43a 100644 --- a/salt/telegraf/scripts/beatseps.sh +++ b/salt/telegraf/scripts/beatseps.sh @@ -17,7 +17,7 @@ THEGREP=$(ps -ef | grep $0 | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then PREVCOUNTFILE='/tmp/beatseps.txt' EVENTCOUNTCURRENT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.acked')" From 0c0edbaac8168c240937571459a8fda19b73bccb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 19 Apr 2021 13:29:46 -0400 Subject: [PATCH 4/5] Change EPS for Telegraf --- salt/telegraf/scripts/beatseps.sh | 2 +- salt/telegraf/scripts/checkfiles.sh | 4 ++-- salt/telegraf/scripts/eps.sh | 4 ++-- salt/telegraf/scripts/helixeps.sh | 4 ++-- salt/telegraf/scripts/influxdbsize.sh | 4 ++-- salt/telegraf/scripts/oldpcap.sh | 4 ++-- salt/telegraf/scripts/raid.sh | 4 ++-- salt/telegraf/scripts/redis.sh | 5 ++--- salt/telegraf/scripts/sostatus.sh | 4 ++-- salt/telegraf/scripts/stenoloss.sh | 4 ++-- salt/telegraf/scripts/suriloss.sh | 4 ++-- salt/telegraf/scripts/zeekcaptureloss.sh | 4 ++-- salt/telegraf/scripts/zeekloss.sh | 4 ++-- 13 files changed, 25 insertions(+), 26 deletions(-) diff --git a/salt/telegraf/scripts/beatseps.sh b/salt/telegraf/scripts/beatseps.sh index 22189e43a..5e8256c22 100644 --- a/salt/telegraf/scripts/beatseps.sh +++ b/salt/telegraf/scripts/beatseps.sh @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) if [ ! "$THEGREP" ]; then diff --git a/salt/telegraf/scripts/checkfiles.sh b/salt/telegraf/scripts/checkfiles.sh index 12cf3ece6..1d7a44382 100644 --- a/salt/telegraf/scripts/checkfiles.sh +++ b/salt/telegraf/scripts/checkfiles.sh @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then FILES=$(ls -1x /host/nsm/faf/complete/ | wc -l) diff --git a/salt/telegraf/scripts/eps.sh b/salt/telegraf/scripts/eps.sh index 99f001552..25332e94a 100644 --- a/salt/telegraf/scripts/eps.sh +++ b/salt/telegraf/scripts/eps.sh @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then PREVCOUNTFILE='/tmp/eps.txt' EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.events.in')" diff --git a/salt/telegraf/scripts/helixeps.sh b/salt/telegraf/scripts/helixeps.sh index 7922a7ab6..d24f1d1e7 100644 --- a/salt/telegraf/scripts/helixeps.sh +++ b/salt/telegraf/scripts/helixeps.sh @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then PREVCOUNTFILE='/tmp/helixevents.txt' EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.pipelines.helix.events.out')" diff --git a/salt/telegraf/scripts/influxdbsize.sh b/salt/telegraf/scripts/influxdbsize.sh index f2ed41f35..46e230a8a 100644 --- a/salt/telegraf/scripts/influxdbsize.sh +++ b/salt/telegraf/scripts/influxdbsize.sh @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then INFLUXSIZE=$(du -s -k /host/nsm/influxdb | awk {'print $1'}) diff --git a/salt/telegraf/scripts/oldpcap.sh b/salt/telegraf/scripts/oldpcap.sh index d43f16d14..f23c0c83f 100644 --- a/salt/telegraf/scripts/oldpcap.sh +++ b/salt/telegraf/scripts/oldpcap.sh @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then # Get the data OLDPCAP=$(find /host/nsm/pcap -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'}) diff --git a/salt/telegraf/scripts/raid.sh b/salt/telegraf/scripts/raid.sh index c0aabe75f..03e309c38 100644 --- a/salt/telegraf/scripts/raid.sh +++ b/salt/telegraf/scripts/raid.sh @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then if [ -f "$RAIDLOG" ]; then echo "raid $RAIDSTATUS" diff --git a/salt/telegraf/scripts/redis.sh b/salt/telegraf/scripts/redis.sh index 613e6cdea..b448bba2d 100644 --- a/salt/telegraf/scripts/redis.sh +++ b/salt/telegraf/scripts/redis.sh @@ -15,10 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -THEGREP=$(ps -ef | grep $0 | grep -v grep) - -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then UNPARSED=$(redis-cli llen logstash:unparsed | awk '{print $1}') PARSED=$(redis-cli llen logstash:parsed | awk '{print $1}') diff --git a/salt/telegraf/scripts/sostatus.sh b/salt/telegraf/scripts/sostatus.sh index 1baf38d2b..a7222b67d 100644 --- a/salt/telegraf/scripts/sostatus.sh +++ b/salt/telegraf/scripts/sostatus.sh @@ -14,9 +14,9 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then SOSTATUSLOG=/var/log/sostatus/status.log SOSTATUSSTATUS=$(cat /var/log/sostatus/status.log) diff --git a/salt/telegraf/scripts/stenoloss.sh b/salt/telegraf/scripts/stenoloss.sh index a5c974a73..028637e16 100644 --- a/salt/telegraf/scripts/stenoloss.sh +++ b/salt/telegraf/scripts/stenoloss.sh @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then TSFILE=/var/log/telegraf/laststenodrop.log if [ -f "$TSFILE" ]; then diff --git a/salt/telegraf/scripts/suriloss.sh b/salt/telegraf/scripts/suriloss.sh index 0ae867b29..1f43fbaf8 100644 --- a/salt/telegraf/scripts/suriloss.sh +++ b/salt/telegraf/scripts/suriloss.sh @@ -16,9 +16,9 @@ # along with this program. If not, see . -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then SURILOG=$(tac /var/log/suricata/stats.log | grep kernel | head -4) CHECKIT=$(echo $SURILOG | grep -o 'drop' | wc -l) diff --git a/salt/telegraf/scripts/zeekcaptureloss.sh b/salt/telegraf/scripts/zeekcaptureloss.sh index 995971b18..03c9188ea 100644 --- a/salt/telegraf/scripts/zeekcaptureloss.sh +++ b/salt/telegraf/scripts/zeekcaptureloss.sh @@ -18,9 +18,9 @@ # This script returns the average of all the workers average capture loss to telegraf / influxdb in influx format include nanosecond precision timestamp -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then if [ -d "/host/nsm/zeek/spool/logger" ]; then WORKERS={{ salt['pillar.get']('sensor:zeek_lbprocs', salt['pillar.get']('sensor:zeek_pins') | length) }} diff --git a/salt/telegraf/scripts/zeekloss.sh b/salt/telegraf/scripts/zeekloss.sh index 559c6b15f..c9bc843cf 100644 --- a/salt/telegraf/scripts/zeekloss.sh +++ b/salt/telegraf/scripts/zeekloss.sh @@ -17,9 +17,9 @@ # This script returns the packets dropped by Zeek, but it isn't a percentage. $LOSS * 100 would be the percentage -THEGREP=$(ps -ef | grep $0 | grep -v grep) +THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) -if [ ! $THEGREP ]; then +if [ ! "$THEGREP" ]; then ZEEKLOG=$(tac /host/nsm/zeek/logs/packetloss.log | head -2) declare RESULT=($ZEEKLOG) From be6933e8fb80536be70bbf12697bbfeb958c6085 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 19 Apr 2021 14:20:00 -0400 Subject: [PATCH 5/5] Change EPS for Telegraf --- salt/telegraf/scripts/checkfiles.sh | 2 +- salt/telegraf/scripts/eps.sh | 2 +- salt/telegraf/scripts/helixeps.sh | 1 + salt/telegraf/scripts/suriloss.sh | 1 + salt/telegraf/scripts/zeekcaptureloss.sh | 1 + salt/telegraf/scripts/zeekloss.sh | 1 + 6 files changed, 6 insertions(+), 2 deletions(-) diff --git a/salt/telegraf/scripts/checkfiles.sh b/salt/telegraf/scripts/checkfiles.sh index 1d7a44382..5c6ab56c1 100644 --- a/salt/telegraf/scripts/checkfiles.sh +++ b/salt/telegraf/scripts/checkfiles.sh @@ -19,7 +19,7 @@ THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) if [ ! "$THEGREP" ]; then - FILES=$(ls -1x /host/nsm/faf/complete/ | wc -l) + FILES=$(ls -1x /host/nsm/strelka/unprocessed | wc -l) echo "faffiles files=$FILES" else diff --git a/salt/telegraf/scripts/eps.sh b/salt/telegraf/scripts/eps.sh index 25332e94a..b497c2519 100644 --- a/salt/telegraf/scripts/eps.sh +++ b/salt/telegraf/scripts/eps.sh @@ -39,7 +39,7 @@ if [ ! "$THEGREP" ]; then fi echo "consumptioneps eps=${EVENTS%%.*}" - + fi else exit 0 fi diff --git a/salt/telegraf/scripts/helixeps.sh b/salt/telegraf/scripts/helixeps.sh index d24f1d1e7..1411cc40b 100644 --- a/salt/telegraf/scripts/helixeps.sh +++ b/salt/telegraf/scripts/helixeps.sh @@ -38,6 +38,7 @@ if [ ! "$THEGREP" ]; then fi echo "helixeps eps=${EVENTS%%.*}" + fi else exit 0 fi \ No newline at end of file diff --git a/salt/telegraf/scripts/suriloss.sh b/salt/telegraf/scripts/suriloss.sh index 1f43fbaf8..2d0a56106 100644 --- a/salt/telegraf/scripts/suriloss.sh +++ b/salt/telegraf/scripts/suriloss.sh @@ -42,6 +42,7 @@ if [ ! "$THEGREP" ]; then LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc) echo "suridrop drop=$LOSS" fi + fi else echo "suridrop drop=0" fi \ No newline at end of file diff --git a/salt/telegraf/scripts/zeekcaptureloss.sh b/salt/telegraf/scripts/zeekcaptureloss.sh index 03c9188ea..6cb2dd2e2 100644 --- a/salt/telegraf/scripts/zeekcaptureloss.sh +++ b/salt/telegraf/scripts/zeekcaptureloss.sh @@ -44,6 +44,7 @@ if [ ! "$THEGREP" ]; then fi fi echo "$CURRENTTS" > $LASTCAPTURELOSSLOG + fi else exit 0 fi diff --git a/salt/telegraf/scripts/zeekloss.sh b/salt/telegraf/scripts/zeekloss.sh index c9bc843cf..3dbd42833 100644 --- a/salt/telegraf/scripts/zeekloss.sh +++ b/salt/telegraf/scripts/zeekloss.sh @@ -42,6 +42,7 @@ if [ ! "$THEGREP" ]; then TOTAL=$((CURRENTPACKETS - PASTPACKETS)) LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc) echo "zeekdrop drop=$LOSS" + fi else exit 0 fi \ No newline at end of file