Merge pull request #3912 from Security-Onion-Solutions/telefix1

Change telegraf scripts to new method of process detection

salt/telegraf/etc/telegraf.conf

@@ -17,6 +17,7 @@
 {% set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') %}
 {% set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %}
 {% set UNIQUEID = salt['pillar.get']('sensor:uniqueid', '') %}
+{%- set TRUE_CLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False)  %}
 
 # Global tags can be specified here in key="value" format.
 [global_tags]
@@ -614,18 +615,29 @@
 #   ## Use TLS but skip chain & host verification
 #   # insecure_skip_verify = false
 
+{% if TRUE_CLUSTER %}
+  {% if grains.role == 'so-manager' %}
+[[inputs.elasticsearch]]
+   servers = ["https://{{ MANAGER }}:9200"]
+   insecure_skip_verify = true
+   local = false
+   cluster_health = true
+   cluster_stats = true
+  {% endif %}
+  
+{% else %}
 
 # # Read stats from one or more Elasticsearch servers or clusters
-{% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone']   %}
+  {% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone']   %}
  [[inputs.elasticsearch]]
    servers = ["https://{{ MANAGER }}:9200"]
    insecure_skip_verify = true
-{% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode']  %}
+  {% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode']  %}
  [[inputs.elasticsearch]]
    servers = ["https://{{ NODEIP }}:9200"]
    insecure_skip_verify = true
+  {% endif %}
 {% endif %}
-
 #
 #   ## Timeout for HTTP requests to the elastic search server(s)
 #   http_timeout = "5s"
@@ -673,11 +685,32 @@
 
 
 #   ## Commands array
-{% if grains['role'] in ['so-manager', 'so-managersearch'] %}
+{% if grains['role'] in ['so-manager'] %}
 [[inputs.exec]]
    commands = [
       "/scripts/redis.sh",
       "/scripts/influxdbsize.sh",
+      "/scripts/raid.sh",
+      "/scripts/beatseps.sh"
+   ]
+  data_format = "influx"
+  ## Timeout for each command to complete.
+  timeout = "15s"
+{% elif grains['role'] in ['so-managersearch'] %}
+[[inputs.exec]]
+   commands = [
+      "/scripts/redis.sh",
+      "/scripts/influxdbsize.sh",
+      "/scripts/eps.sh",
+      "/scripts/raid.sh",
+      "/scripts/beatseps.sh"
+   ]
+  data_format = "influx"
+  ## Timeout for each command to complete.
+  timeout = "15s"
+{% elif grains['role'] in ['so-node'] %}
+[[inputs.exec]]
+   commands = [
       "/scripts/eps.sh",
       "/scripts/raid.sh",
       "/scripts/beatseps.sh"

salt/telegraf/scripts/beatseps.sh

@@ -15,20 +15,15 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-APP=beatseps
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
 
-PREVCOUNTFILE='/tmp/beatseps.txt'
+if [ ! "$THEGREP" ]; then
-EVENTCOUNTCURRENT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.acked')"
-FAILEDEVENTCOUNT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.failed')"
 
-if [ ! -z "$EVENTCOUNTCURRENT" ]; then
+  PREVCOUNTFILE='/tmp/beatseps.txt'
+  EVENTCOUNTCURRENT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.acked')"
+  FAILEDEVENTCOUNT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.failed')"
+
+  if [ ! -z "$EVENTCOUNTCURRENT" ]; then
 
     if [ -f "$PREVCOUNTFILE" ]; then
       EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE`
@@ -45,7 +40,9 @@ if [ ! -z "$EVENTCOUNTCURRENT" ]; then
     fi
 
     echo "fbstats eps=${EVENTS%%.*},failed=$FAILEDEVENTCOUNT"
+  fi
 
+else
+    exit 0
 fi
 
-exit 0

salt/telegraf/scripts/checkfiles.sh

@@ -15,15 +15,13 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-APP=checkfiles
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
 
-FILES=$(ls -1x /host/nsm/faf/complete/ | wc -l)
+if [ ! "$THEGREP" ]; then
 
-echo "faffiles files=$FILES"
+    FILES=$(ls -1x /host/nsm/strelka/unprocessed | wc -l)
+
+    echo "faffiles files=$FILES"
+else
+    exit 0
+fi

salt/telegraf/scripts/eps.sh

@@ -15,19 +15,14 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-APP=eps
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
 
-PREVCOUNTFILE='/tmp/eps.txt'
+if [ ! "$THEGREP" ]; then
-EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.events.in')"
 
-if [ ! -z "$EVENTCOUNTCURRENT" ]; then
+    PREVCOUNTFILE='/tmp/eps.txt'
+    EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.events.in')"
+
+    if [ ! -z "$EVENTCOUNTCURRENT" ]; then
 
       if [ -f "$PREVCOUNTFILE" ]; then
         EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE`
@@ -43,8 +38,9 @@ if [ ! -z "$EVENTCOUNTCURRENT" ]; then
         EVENTS=0
       fi
 
-  echo "esteps eps=${EVENTS%%.*}"
+      echo "consumptioneps eps=${EVENTS%%.*}"
-
+    fi
+else
+    exit 0
 fi
 
-exit 0

salt/telegraf/scripts/helixeps.sh

@@ -15,19 +15,14 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-APP=helixeps
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
 
-PREVCOUNTFILE='/tmp/helixevents.txt'
+if [ ! "$THEGREP" ]; then
-EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.pipelines.helix.events.out')"
 
-if [ ! -z "$EVENTCOUNTCURRENT" ]; then
+  PREVCOUNTFILE='/tmp/helixevents.txt'
+  EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.pipelines.helix.events.out')"
+
+  if [ ! -z "$EVENTCOUNTCURRENT" ]; then
 
     if [ -f "$PREVCOUNTFILE" ]; then
       EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE`
@@ -43,7 +38,7 @@ if [ ! -z "$EVENTCOUNTCURRENT" ]; then
     fi
 
     echo "helixeps eps=${EVENTS%%.*}"
-
+  fi
+else
+    exit 0
 fi
-
-exit 0

salt/telegraf/scripts/influxdbsize.sh

@@ -15,15 +15,13 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-APP=influxsize
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
 
-INFLUXSIZE=$(du -s -k /host/nsm/influxdb | awk {'print $1'})
+if [ ! "$THEGREP" ]; then
 
-echo "influxsize kbytes=$INFLUXSIZE"
+    INFLUXSIZE=$(du -s -k /host/nsm/influxdb | awk {'print $1'})
+
+    echo "influxsize kbytes=$INFLUXSIZE"
+else
+    exit 0 
+fi

salt/telegraf/scripts/oldpcap.sh

@@ -15,18 +15,16 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-APP=oldpcap
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
 
-# Get the data
+if [ ! "$THEGREP" ]; then
-OLDPCAP=$(find /host/nsm/pcap -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'})
-DATE=$(date +%s)
-AGE=$(($DATE - $OLDPCAP))
 
-echo "pcapage seconds=$AGE"
+    # Get the data
+    OLDPCAP=$(find /host/nsm/pcap -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'})
+    DATE=$(date +%s)
+    AGE=$(($DATE - $OLDPCAP))
+
+    echo "pcapage seconds=$AGE"
+else
+    exit 0
+fi

salt/telegraf/scripts/raid.sh

@@ -15,19 +15,15 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-APP=raid
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
-RAIDLOG=/var/log/raid/status.log
-RAIDSTATUS=$(cat /var/log/raid/status.log)
 
-if [ -f "$RAIDLOG" ]; then
+if [ ! "$THEGREP" ]; then
+
+    if [ -f "$RAIDLOG" ]; then
         echo "raid $RAIDSTATUS"
+    else
+        exit 0
+    fi
 else
     exit 0
 fi

salt/telegraf/scripts/redis.sh

@@ -15,17 +15,14 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
 
-APP=redis
+if [ ! "$THEGREP" ]; then
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
 
-UNPARSED=$(redis-cli llen logstash:unparsed | awk '{print $1}')
+    UNPARSED=$(redis-cli llen logstash:unparsed | awk '{print $1}')
-PARSED=$(redis-cli llen logstash:parsed | awk '{print $1}')
+    PARSED=$(redis-cli llen logstash:parsed | awk '{print $1}')
 
-echo "redisqueue unparsed=$UNPARSED,parsed=$PARSED"
+    echo "redisqueue unparsed=$UNPARSED,parsed=$PARSED"
+else
+   exit 0
+fi

salt/telegraf/scripts/sostatus.sh

@@ -14,20 +14,18 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
 
-APP=sostatus
+if [ ! "$THEGREP" ]; then
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
-SOSTATUSLOG=/var/log/sostatus/status.log
-SOSTATUSSTATUS=$(cat /var/log/sostatus/status.log)
 
-if [ -f "$SOSTATUSLOG" ]; then
+    SOSTATUSLOG=/var/log/sostatus/status.log
+    SOSTATUSSTATUS=$(cat /var/log/sostatus/status.log)
+
+    if [ -f "$SOSTATUSLOG" ]; then
         echo "sostatus status=$SOSTATUSSTATUS"
+    else
+        exit 0
+    fi
 else 
     exit 0
 fi

salt/telegraf/scripts/stenoloss.sh

@@ -15,31 +15,29 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-APP=stenoloss
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
 
-TSFILE=/var/log/telegraf/laststenodrop.log
+if [ ! "$THEGREP" ]; then
-if [ -f "$TSFILE" ]; then
+
+    TSFILE=/var/log/telegraf/laststenodrop.log
+    if [ -f "$TSFILE" ]; then
         LASTTS=$(cat $TSFILE)
-else
+    else
         LASTTS=0
-fi
+    fi
 
-# Get the data
+    # Get the data
-LOGLINE=$(tac /var/log/stenographer/stenographer.log | grep -m1 drop)
+    LOGLINE=$(tac /var/log/stenographer/stenographer.log | grep -m1 drop)
-CURRENTTS=$(echo $LOGLINE | awk '{print $1}')
+    CURRENTTS=$(echo $LOGLINE | awk '{print $1}')
 
-if [[ "$CURRENTTS" != "$LASTTS" ]]; then
+    if [[ "$CURRENTTS" != "$LASTTS" ]]; then
       DROP=$(echo $LOGLINE | awk '{print $14}' | awk -F "=" '{print $2}')
       echo $CURRENTTS > $TSFILE
-else
+    else
       DROP=0
-fi
+    fi
 
-echo "stenodrop drop=$DROP"
+    echo "stenodrop drop=$DROP"
+else
+    exit 0
+fi

salt/telegraf/scripts/suriloss.sh

@@ -16,19 +16,14 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
-APP=suriloss
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
 
-SURILOG=$(tac /var/log/suricata/stats.log | grep kernel | head -4)
+if [ ! "$THEGREP" ]; then
-CHECKIT=$(echo $SURILOG | grep -o 'drop' | wc -l)
 
-if [ $CHECKIT == 2 ]; then
+    SURILOG=$(tac /var/log/suricata/stats.log | grep kernel | head -4)
+    CHECKIT=$(echo $SURILOG | grep -o 'drop' | wc -l)
+
+    if [ $CHECKIT == 2 ]; then
       declare RESULT=($SURILOG)
 
       CURRENTDROP=${RESULT[4]}
@@ -47,6 +42,7 @@ if [ $CHECKIT == 2 ]; then
         LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc)
         echo "suridrop drop=$LOSS"
       fi
+    fi
 else
   echo "suridrop drop=0"
 fi

salt/telegraf/scripts/zeekcaptureloss.sh

@@ -18,28 +18,23 @@
 
 # This script returns the average of all the workers average capture loss to telegraf / influxdb in influx format include nanosecond precision timestamp
 
-APP=zeekcaploss
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
 
-if [ -d "/host/nsm/zeek/spool/logger" ]; then
+if [ ! "$THEGREP" ]; then
+
+    if [ -d "/host/nsm/zeek/spool/logger" ]; then
       WORKERS={{ salt['pillar.get']('sensor:zeek_lbprocs', salt['pillar.get']('sensor:zeek_pins') | length) }}
       ZEEKLOG=/host/nsm/zeek/spool/logger/capture_loss.log
-elif [ -d "/host/nsm/zeek/spool/zeeksa" ]; then
+    elif [ -d "/host/nsm/zeek/spool/zeeksa" ]; then
       WORKERS=1
       ZEEKLOG=/host/nsm/zeek/spool/zeeksa/capture_loss.log
-else
+    else
       echo 'Zeek capture_loss.log not found' >/dev/stderr
       exit 2
-fi
+    fi
 
-LASTCAPTURELOSSLOG=/var/log/telegraf/lastcaptureloss.txt
+    LASTCAPTURELOSSLOG=/var/log/telegraf/lastcaptureloss.txt
-if [ -f "$ZEEKLOG" ]; then
+    if [ -f "$ZEEKLOG" ]; then
       CURRENTTS=$(tail -1 $ZEEKLOG | jq .ts | sed 's/"//g')
       if [ -f "$LASTCAPTURELOSSLOG" ]; then
         LASTTS=$(cat $LASTCAPTURELOSSLOG)
@@ -49,4 +44,7 @@ if [ -f "$ZEEKLOG" ]; then
         fi
       fi
       echo "$CURRENTTS" > $LASTCAPTURELOSSLOG
+    fi
+else
+    exit 0
 fi

salt/telegraf/scripts/zeekloss.sh

@@ -17,34 +17,32 @@
 
 # This script returns the packets dropped by Zeek, but it isn't a percentage. $LOSS * 100 would be the percentage
 
-APP=zeekloss
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
 
-ZEEKLOG=$(tac /host/nsm/zeek/logs/packetloss.log | head -2)
+if [ ! "$THEGREP" ]; then
-declare RESULT=($ZEEKLOG)
+
-CURRENTDROP=${RESULT[3]}
+  ZEEKLOG=$(tac /host/nsm/zeek/logs/packetloss.log | head -2)
-# zeek likely not running if this is true
+  declare RESULT=($ZEEKLOG)
-if [[ $CURRENTDROP == "rcvd:" ]]; then
+  CURRENTDROP=${RESULT[3]}
+  # zeek likely not running if this is true
+  if [[ $CURRENTDROP == "rcvd:" ]]; then
     CURRENTDROP=0
     PASTDROP=0
     DROPPED=0
-else
+  else
     PASTDROP=${RESULT[9]}
     DROPPED=$((CURRENTDROP - PASTDROP))
-fi
+  fi
-if [[ "$DROPPED" -le 0 ]]; then
+  if [[ "$DROPPED" -le 0 ]]; then
     LOSS=0
     echo "zeekdrop drop=0"
-else
+  else
     CURRENTPACKETS=${RESULT[5]}
     PASTPACKETS=${RESULT[11]}
     TOTAL=$((CURRENTPACKETS - PASTPACKETS))
     LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc)
     echo "zeekdrop drop=$LOSS"
+  fi
+else
+  exit 0
 fi