Move zeek_clean to so-sensor-clean

salt/common/init.sls

@@ -131,3 +131,13 @@ utilsyncscripts:
     - file_mode: 755
     - template: jinja
     - source: salt://common/tools/sbin
+
+# Add sensor cleanup
+/usr/sbin/so-sensor-clean:
+  cron.present:
+    - user: root
+    - minute: '*'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'

salt/common/tools/sbin/so-sensor-clean

@@ -0,0 +1,121 @@
+#!/bin/bash
+
+# Delete Zeek Logs based on defined CRIT_DISK_USAGE value
+
+# Copyright 2014,2015,2016,2017,2018, 2019 Security Onion Solutions, LLC
+
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+SENSOR_DIR='/nsm'
+CRIT_DISK_USAGE=30
+CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
+LOG="/opt/so/log/sensor_clean.log"
+TODAY=$(date -u "+%Y-%m-%d")
+
+clean () {
+                ## find the oldest Zeek logs directory
+                OLDEST_DIR=$(ls /nsm/zeek/logs/ | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
+                if [ -z "$OLDEST_DIR" -o "$OLDEST_DIR" == ".." -o "$OLDEST_DIR" == "." ]
+                then
+                        echo "$(date) - No old Zeek logs available to clean up in /nsm/zeek/logs/" >> $LOG
+                        #exit 0
+                else
+                        echo "$(date) - Removing directory: /nsm/zeek/logs/$OLDEST_DIR" >> $LOG
+                        rm -rf /nsm/zeek/logs/"$OLDEST_DIR"
+                fi
+
+
+                ## Remarking for now, as we are moving extracted files to /nsm/strelka/processed
+                ## find oldest files in extracted directory and exclude today
+                #OLDEST_EXTRACT=$(find /nsm/zeek/extracted/complete -type f -printf '%T+ %p\n' 2>/dev/null | sort | grep -v $TODAY | head -n 1)
+                #if [ -z "$OLDEST_EXTRACT" -o "$OLDEST_EXTRACT" == ".." -o "$OLDEST_EXTRACT" == "." ]
+                #then
+                #        echo "$(date) - No old extracted files available to clean up in /nsm/zeek/extracted/complete" >> $LOG
+                #else
+                #        OLDEST_EXTRACT_DATE=`echo $OLDEST_EXTRACT | awk '{print $1}' | cut -d+ -f1`
+                #        OLDEST_EXTRACT_FILE=`echo $OLDEST_EXTRACT | awk '{print $2}'`
+                #        echo "$(date) - Removing extracted files for $OLDEST_EXTRACT_DATE" >> $LOG
+                #        find /nsm/zeek/extracted/complete -type f -printf '%T+ %p\n' | grep $OLDEST_EXTRACT_DATE | awk '{print $2}' |while read FILE
+                #        do
+                #                echo "$(date) - Removing extracted file: $FILE" >> $LOG
+                #                rm -f "$FILE"
+                #        done
+                #fi
+ 
+                ## Clean up Zeek extracted files processed by Strelka
+                STRELKA_FILES='/nsm/strelka/processed'
+                OLDEST_STRELKA=$(find $STRELKA_FILES -type f -printf '%T+ %p\n' | sort -n | head -n 1 )
+                if [ -z "$OLDEST_STRELKA" -o "$OLDEST_STRELKA" == ".." -o "$OLDEST_STRELKA" == "." ]
+                then
+                        echo "$(date) - No old files available to clean up in $STRELKA_FILES" >> $LOG
+                else
+                        OLDEST_STRELKA_DATE=`echo $OLDEST_STRELKA | awk '{print $1}' | cut -d+ -f1`
+                        OLDEST_STRELKA_FILE=`echo $OLDEST_STRELKA | awk '{print $2}'`
+                        echo "$(date) - Removing extracted files for $OLDEST_STRELKA_DATE" >> $LOG
+                        find $STRELKA_FILES -type f -printf '%T+ %p\n' | grep $OLDEST_STRELKA_DATE | awk '{print $2}' |while read FILE
+                        do
+                                echo "$(date) - Removing file: $FILE" >> $LOG
+                                rm -f "$FILE"
+                        done
+                fi
+
+                ## Clean up Suricata log files
+                SURICATA_LOGS='/nsm/suricata'
+                OLDEST_SURICATA=$(find $STRELKA_FILES -type f -printf '%T+ %p\n' | sort -n | head -n 1)
+                if [ -z "$OLDEST_SURICATA" -o "$OLDEST_SURICATA" == ".." -o "$OLDEST_SURICATA" == "." ]
+                then
+                        echo "$(date) - No old files available to clean up in $SURICATA_LOGS" >> $LOG
+                else
+                        OLDEST_SURICATA_DATE=`echo $OLDEST_SURICATA | awk '{print $1}' | cut -d+ -f1`
+                        OLDEST_SURICATA_FILE=`echo $OLDEST_SURICATA | awk '{print $2}'`
+                        echo "$(date) - Removing logs for $OLDEST_SURICATA_DATE" >> $LOG
+                        find $SURICATA_LOGS -type f -printf '%T+ %p\n' | grep $OLDEST_SURICATA_DATE | awk '{print $2}' |while read FILE
+                        do
+                                echo "$(date) - Removing file: $FILE" >> $LOG
+                                rm -f "$FILE"
+                        done
+                fi
+
+                ## Clean up extracted pcaps from Steno
+                PCAPS='/nsm/pcapout'
+                OLDEST_PCAP=$(find $PCAPS -type f -printf '%T+ %p\n' | sort -n | head -n 1 )
+                if [ -z "$OLDEST_PCAP" -o "$OLDEST_PCAP" == ".." -o "$OLDEST_PCAP" == "." ]
+                then
+                        echo "$(date) - No old files available to clean up in $PCAPS" >> $LOG
+                else
+                        OLDEST_PCAP_DATE=`echo $OLDEST_PCAP | awk '{print $1}' | cut -d+ -f1`
+                        OLDEST_PCAP_FILE=`echo $OLDEST_PCAP | awk '{print $2}'`
+                        echo "$(date) - Removing extracted files for $OLDEST_PCAP_DATE" >> $LOG
+                        find $PCAPS -type f -printf '%T+ %p\n' | grep $OLDEST_PCAP_DATE | awk '{print $2}' |while read FILE
+                        do
+                                echo "$(date) - Removing file: $FILE" >> $LOG
+                                rm -f "$FILE"
+                        done
+                fi
+}
+
+# Check to see if we are already running
+IS_RUNNING=$(ps aux | grep "sensor_clean" | grep -v grep | wc -l)
+[ "$IS_RUNNING" -gt 2 ] && echo "$(date) - $IS_RUNNING sensor clean script processes running...exiting." >> $LOG && exit 0
+
+if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then
+    while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ];
+        do
+         clean
+         CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
+    done
+else
+    echo "$(date) - Current usage value of $CUR_USAGE not greater than CRIT_DISK_USAGE value of $CRIT_DISK_USAGE..." >> $LOG
+fi
+

salt/zeek/init.sls

@@ -86,20 +86,20 @@ nodecfgsync:
     - group: 939
     - template: jinja
 
-zeekcleanscript:
+#zeekcleanscript:
-  file.managed:
+#  file.managed:
-    - name: /usr/local/bin/zeek_clean
+#    - name: /usr/local/bin/zeek_clean
-    - source: salt://zeek/cron/zeek_clean
+#    - source: salt://zeek/cron/zeek_clean
-    - mode: 755
+#    - mode: 755
 
-/usr/local/bin/zeek_clean:
+#/usr/local/bin/zeek_clean:
-  cron.present:
+#  cron.present:
-    - user: root
+#    - user: root
-    - minute: '*'
+#    - minute: '*'
-    - hour: '*'
+#    - hour: '*'
-    - daymonth: '*'
+#    - daymonth: '*'
-    - month: '*'
+#    - month: '*'
-    - dayweek: '*'
+#    - dayweek: '*'
 
 plcronscript:
   file.managed: