Merge pull request #3912 from Security-Onion-Solutions/telefix1

Change telegraf scripts to new method of process detection
2025-12-07 17:52:46 +01:00 · 2021-04-19 14:40:06 -04:00
parent a70b631b2c be6933e8fb
commit e9b86388da
14 changed files with 259 additions and 263 deletions
--- a/salt/telegraf/etc/telegraf.conf
+++ b/salt/telegraf/etc/telegraf.conf
@@ -17,6 +17,7 @@
 {% set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') %}
 {% set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %}
 {% set UNIQUEID = salt['pillar.get']('sensor:uniqueid', '') %}
+{%- set TRUE_CLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False)  %}

 # Global tags can be specified here in key="value" format.
 [global_tags]
@@ -614,6 +615,17 @@
 #   ## Use TLS but skip chain & host verification
 #   # insecure_skip_verify = false

+{% if TRUE_CLUSTER %}
+  {% if grains.role == 'so-manager' %}
+[[inputs.elasticsearch]]
+   servers = ["https://{{ MANAGER }}:9200"]
+   insecure_skip_verify = true
+   local = false
+   cluster_health = true
+   cluster_stats = true
+  {% endif %}
+  
+{% else %}

 # # Read stats from one or more Elasticsearch servers or clusters
  {% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone']   %}
@@ -625,7 +637,7 @@
   servers = ["https://{{ NODEIP }}:9200"]
   insecure_skip_verify = true
  {% endif %}
-
+{% endif %}
 #
 #   ## Timeout for HTTP requests to the elastic search server(s)
 #   http_timeout = "5s"
@@ -673,11 +685,32 @@


 #   ## Commands array
-{% if grains['role'] in ['so-manager', 'so-managersearch'] %}
+{% if grains['role'] in ['so-manager'] %}
 [[inputs.exec]]
   commands = [
      "/scripts/redis.sh",
      "/scripts/influxdbsize.sh",
+      "/scripts/raid.sh",
+      "/scripts/beatseps.sh"
+   ]
+  data_format = "influx"
+  ## Timeout for each command to complete.
+  timeout = "15s"
+{% elif grains['role'] in ['so-managersearch'] %}
+[[inputs.exec]]
+   commands = [
+      "/scripts/redis.sh",
+      "/scripts/influxdbsize.sh",
+      "/scripts/eps.sh",
+      "/scripts/raid.sh",
+      "/scripts/beatseps.sh"
+   ]
+  data_format = "influx"
+  ## Timeout for each command to complete.
+  timeout = "15s"
+{% elif grains['role'] in ['so-node'] %}
+[[inputs.exec]]
+   commands = [
      "/scripts/eps.sh",
      "/scripts/raid.sh",
      "/scripts/beatseps.sh"
--- a/salt/telegraf/scripts/beatseps.sh
+++ b/salt/telegraf/scripts/beatseps.sh
@@ -15,14 +15,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.

-APP=beatseps
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
+
+if [ ! "$THEGREP" ]; then

  PREVCOUNTFILE='/tmp/beatseps.txt'
  EVENTCOUNTCURRENT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.acked')"
@@ -45,7 +40,9 @@ if [ ! -z "$EVENTCOUNTCURRENT" ]; then
    fi

    echo "fbstats eps=${EVENTS%%.*},failed=$FAILEDEVENTCOUNT"
-
  fi

+else
    exit 0
+fi
+
--- a/salt/telegraf/scripts/checkfiles.sh
+++ b/salt/telegraf/scripts/checkfiles.sh
@@ -15,15 +15,13 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.

-APP=checkfiles
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)

-FILES=$(ls -1x /host/nsm/faf/complete/ | wc -l)
+if [ ! "$THEGREP" ]; then
+
+    FILES=$(ls -1x /host/nsm/strelka/unprocessed | wc -l)

    echo "faffiles files=$FILES"
+else
+    exit 0
+fi
--- a/salt/telegraf/scripts/eps.sh
+++ b/salt/telegraf/scripts/eps.sh
@@ -15,14 +15,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.

-APP=eps
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
+
+if [ ! "$THEGREP" ]; then

    PREVCOUNTFILE='/tmp/eps.txt'
    EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.events.in')"
@@ -43,8 +38,9 @@ if [ ! -z "$EVENTCOUNTCURRENT" ]; then
        EVENTS=0
      fi

-  echo "esteps eps=${EVENTS%%.*}"
-
+      echo "consumptioneps eps=${EVENTS%%.*}"
+    fi
+else
+    exit 0
 fi

-exit 0
--- a/salt/telegraf/scripts/helixeps.sh
+++ b/salt/telegraf/scripts/helixeps.sh
@@ -15,14 +15,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.

-APP=helixeps
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
+
+if [ ! "$THEGREP" ]; then

  PREVCOUNTFILE='/tmp/helixevents.txt'
  EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.pipelines.helix.events.out')"
@@ -43,7 +38,7 @@ if [ ! -z "$EVENTCOUNTCURRENT" ]; then
    fi

    echo "helixeps eps=${EVENTS%%.*}"
-
  fi
-
+else
    exit 0
+fi
--- a/salt/telegraf/scripts/influxdbsize.sh
+++ b/salt/telegraf/scripts/influxdbsize.sh
@@ -15,15 +15,13 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.

-APP=influxsize
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
+
+if [ ! "$THEGREP" ]; then

    INFLUXSIZE=$(du -s -k /host/nsm/influxdb | awk {'print $1'})

    echo "influxsize kbytes=$INFLUXSIZE"
+else
+    exit 0 
+fi
--- a/salt/telegraf/scripts/oldpcap.sh
+++ b/salt/telegraf/scripts/oldpcap.sh
@@ -15,14 +15,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.

-APP=oldpcap
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
+
+if [ ! "$THEGREP" ]; then

    # Get the data
    OLDPCAP=$(find /host/nsm/pcap -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'})
@@ -30,3 +25,6 @@ DATE=$(date +%s)
    AGE=$(($DATE - $OLDPCAP))

    echo "pcapage seconds=$AGE"
+else
+    exit 0
+fi
--- a/salt/telegraf/scripts/raid.sh
+++ b/salt/telegraf/scripts/raid.sh
@@ -15,19 +15,15 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.

-APP=raid
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
-RAIDLOG=/var/log/raid/status.log
-RAIDSTATUS=$(cat /var/log/raid/status.log)
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
+
+if [ ! "$THEGREP" ]; then

    if [ -f "$RAIDLOG" ]; then
        echo "raid $RAIDSTATUS"
    else
        exit 0
    fi
+else
+    exit 0
+fi
--- a/salt/telegraf/scripts/redis.sh
+++ b/salt/telegraf/scripts/redis.sh
@@ -15,17 +15,14 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.

+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)

-APP=redis
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
+if [ ! "$THEGREP" ]; then

    UNPARSED=$(redis-cli llen logstash:unparsed | awk '{print $1}')
    PARSED=$(redis-cli llen logstash:parsed | awk '{print $1}')

    echo "redisqueue unparsed=$UNPARSED,parsed=$PARSED"
+else
+   exit 0
+fi
--- a/salt/telegraf/scripts/sostatus.sh
+++ b/salt/telegraf/scripts/sostatus.sh
@@ -14,15 +14,10 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
+
+if [ ! "$THEGREP" ]; then

-APP=sostatus
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
    SOSTATUSLOG=/var/log/sostatus/status.log
    SOSTATUSSTATUS=$(cat /var/log/sostatus/status.log)

@@ -31,3 +26,6 @@ if [ -f "$SOSTATUSLOG" ]; then
    else
        exit 0
    fi
+else 
+    exit 0
+fi
--- a/salt/telegraf/scripts/stenoloss.sh
+++ b/salt/telegraf/scripts/stenoloss.sh
@@ -15,14 +15,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.

-APP=stenoloss
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
+
+if [ ! "$THEGREP" ]; then

    TSFILE=/var/log/telegraf/laststenodrop.log
    if [ -f "$TSFILE" ]; then
@@ -43,3 +38,6 @@ else
    fi

    echo "stenodrop drop=$DROP"
+else
+    exit 0
+fi
--- a/salt/telegraf/scripts/suriloss.sh
+++ b/salt/telegraf/scripts/suriloss.sh
@@ -16,14 +16,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.


-APP=suriloss
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
+
+if [ ! "$THEGREP" ]; then

    SURILOG=$(tac /var/log/suricata/stats.log | grep kernel | head -4)
    CHECKIT=$(echo $SURILOG | grep -o 'drop' | wc -l)
@@ -47,6 +42,7 @@ if [ $CHECKIT == 2 ]; then
        LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc)
        echo "suridrop drop=$LOSS"
      fi
+    fi
 else
  echo "suridrop drop=0"
 fi
--- a/salt/telegraf/scripts/zeekcaptureloss.sh
+++ b/salt/telegraf/scripts/zeekcaptureloss.sh
@@ -18,14 +18,9 @@

 # This script returns the average of all the workers average capture loss to telegraf / influxdb in influx format include nanosecond precision timestamp

-APP=zeekcaploss
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
+
+if [ ! "$THEGREP" ]; then

    if [ -d "/host/nsm/zeek/spool/logger" ]; then
      WORKERS={{ salt['pillar.get']('sensor:zeek_lbprocs', salt['pillar.get']('sensor:zeek_pins') | length) }}
@@ -50,3 +45,6 @@ if [ -f "$ZEEKLOG" ]; then
      fi
      echo "$CURRENTTS" > $LASTCAPTURELOSSLOG
    fi
+else
+    exit 0
+fi
--- a/salt/telegraf/scripts/zeekloss.sh
+++ b/salt/telegraf/scripts/zeekloss.sh
@@ -17,14 +17,9 @@

 # This script returns the packets dropped by Zeek, but it isn't a percentage. $LOSS * 100 would be the percentage

-APP=zeekloss
-lf=/tmp/$APP-pidLockFile
-# create empty lock file if none exists
-cat /dev/null >> $lf
-read lastPID < $lf
-# if lastPID is not null and a process with that pid exists , exit
-[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
-echo $$ > $lf
+THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep)
+
+if [ ! "$THEGREP" ]; then

  ZEEKLOG=$(tac /host/nsm/zeek/logs/packetloss.log | head -2)
  declare RESULT=($ZEEKLOG)
@@ -48,3 +43,6 @@ else
    LOSS=$(echo 4 k $DROPPED $TOTAL / p | dc)
    echo "zeekdrop drop=$LOSS"
  fi
+else
+  exit 0
+fi