mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-09-19 14:19:49 +02:00
Merge pull request #16243 from Security-Onion-Solutions/fix/so-sensor-clean
FIX: prevent so-sensor-clean runaway loop and concurrent instances
This commit is contained in:
Josh Patterson
@@ -8,21 +8,37 @@
|
||||
# Elastic License 2.0.
|
||||
|
||||
|
||||
SENSOR_DIR='/nsm'
|
||||
SENSOR_DIR="${SENSOR_DIR:-/nsm}"
|
||||
CRIT_DISK_USAGE=90
|
||||
CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
|
||||
LOG="/opt/so/log/sensor_clean.log"
|
||||
TODAY=$(date -u "+%Y-%m-%d")
|
||||
LOG="${LOG:-/opt/so/log/sensor_clean.log}"
|
||||
LOCK="${LOCK:-/var/tmp/so-sensor-clean.lock}"
|
||||
MAX_PASSES=100
|
||||
|
||||
ZEEK_LOGS="$SENSOR_DIR/zeek/logs"
|
||||
STRELKA_FILES="$SENSOR_DIR/strelka/processed"
|
||||
SURICATA_LOGS="$SENSOR_DIR/suricata"
|
||||
PCAPS="$SENSOR_DIR/pcapout"
|
||||
|
||||
log() {
|
||||
echo "$(date) - $*" >>"$LOG"
|
||||
}
|
||||
|
||||
disk_usage() {
|
||||
df -P "$SENSOR_DIR" | tail -1 | awk '{print $5}' | tr -d %
|
||||
}
|
||||
|
||||
disk_avail() {
|
||||
df -P "$SENSOR_DIR" | tail -1 | awk '{print $4}'
|
||||
}
|
||||
|
||||
# sets REMOVED=1 if anything was actually deleted
|
||||
clean() {
|
||||
## find the oldest Zeek logs directory
|
||||
OLDEST_DIR=$(ls /nsm/zeek/logs/ | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
|
||||
if [ -z "$OLDEST_DIR" -o "$OLDEST_DIR" == ".." -o "$OLDEST_DIR" == "." ]; then
|
||||
echo "$(date) - No old Zeek logs available to clean up in /nsm/zeek/logs/" >>$LOG
|
||||
#exit 0
|
||||
else
|
||||
echo "$(date) - Removing directory: /nsm/zeek/logs/$OLDEST_DIR" >>$LOG
|
||||
rm -rf /nsm/zeek/logs/"$OLDEST_DIR"
|
||||
OLDEST_DIR=$(ls "$ZEEK_LOGS" 2>/dev/null | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
|
||||
if [ -n "$OLDEST_DIR" ]; then
|
||||
log "Removing directory: $ZEEK_LOGS/$OLDEST_DIR"
|
||||
rm -rf "$ZEEK_LOGS/$OLDEST_DIR"
|
||||
REMOVED=1
|
||||
fi
|
||||
|
||||
## Remarking for now, as we are moving extracted files to /nsm/strelka/processed
|
||||
@@ -43,58 +59,73 @@ clean() {
|
||||
#fi
|
||||
|
||||
## Clean up Zeek extracted files processed by Strelka
|
||||
STRELKA_FILES='/nsm/strelka/processed'
|
||||
OLDEST_STRELKA=$(find $STRELKA_FILES -type f -printf '%T+ %p\n' | sort -n | head -n 1)
|
||||
if [ -z "$OLDEST_STRELKA" -o "$OLDEST_STRELKA" == ".." -o "$OLDEST_STRELKA" == "." ]; then
|
||||
echo "$(date) - No old files available to clean up in $STRELKA_FILES" >>$LOG
|
||||
else
|
||||
OLDEST_STRELKA=$(find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
|
||||
if [ -n "$OLDEST_STRELKA" ]; then
|
||||
OLDEST_STRELKA_DATE=$(echo $OLDEST_STRELKA | awk '{print $1}' | cut -d+ -f1)
|
||||
OLDEST_STRELKA_FILE=$(echo $OLDEST_STRELKA | awk '{print $2}')
|
||||
echo "$(date) - Removing extracted files for $OLDEST_STRELKA_DATE" >>$LOG
|
||||
find $STRELKA_FILES -type f -printf '%T+ %p\n' | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
|
||||
echo "$(date) - Removing file: $FILE" >>$LOG
|
||||
log "Removing extracted files for $OLDEST_STRELKA_DATE"
|
||||
REMOVED=1
|
||||
find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
|
||||
log "Removing file: $FILE"
|
||||
rm -f "$FILE"
|
||||
done
|
||||
fi
|
||||
|
||||
## Clean up Suricata log files
|
||||
SURICATA_LOGS='/nsm/suricata'
|
||||
OLDEST_SURICATA=$(find $SURICATA_LOGS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
|
||||
if [[ -z "$OLDEST_SURICATA" ]] || [[ "$OLDEST_SURICATA" == ".." ]] || [[ "$OLDEST_SURICATA" == "." ]]; then
|
||||
echo "$(date) - No old files available to clean up in $SURICATA_LOGS" >>$LOG
|
||||
else
|
||||
OLDEST_SURICATA=$(find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
|
||||
if [ -n "$OLDEST_SURICATA" ]; then
|
||||
OLDEST_SURICATA_DATE=$(echo $OLDEST_SURICATA | awk '{print $1}' | cut -d+ -f1)
|
||||
OLDEST_SURICATA_FILE=$(echo $OLDEST_SURICATA | awk '{print $2}')
|
||||
echo "$(date) - Removing logs for $OLDEST_SURICATA_DATE" >>$LOG
|
||||
find $SURICATA_LOGS -type f -printf '%T+ %p\n' | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
|
||||
echo "$(date) - Removing file: $FILE" >>$LOG
|
||||
log "Removing logs for $OLDEST_SURICATA_DATE"
|
||||
REMOVED=1
|
||||
find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
|
||||
log "Removing file: $FILE"
|
||||
rm -f "$FILE"
|
||||
done
|
||||
fi
|
||||
|
||||
## Clean up extracted pcaps
|
||||
PCAPS='/nsm/pcapout'
|
||||
OLDEST_PCAP=$(find $PCAPS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
|
||||
if [ -z "$OLDEST_PCAP" -o "$OLDEST_PCAP" == ".." -o "$OLDEST_PCAP" == "." ]; then
|
||||
echo "$(date) - No old files available to clean up in $PCAPS" >>$LOG
|
||||
else
|
||||
OLDEST_PCAP=$(find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
|
||||
if [ -n "$OLDEST_PCAP" ]; then
|
||||
OLDEST_PCAP_DATE=$(echo $OLDEST_PCAP | awk '{print $1}' | cut -d+ -f1)
|
||||
OLDEST_PCAP_FILE=$(echo $OLDEST_PCAP | awk '{print $2}')
|
||||
echo "$(date) - Removing extracted files for $OLDEST_PCAP_DATE" >>$LOG
|
||||
find $PCAPS -type f -printf '%T+ %p\n' | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
|
||||
echo "$(date) - Removing file: $FILE" >>$LOG
|
||||
log "Removing extracted files for $OLDEST_PCAP_DATE"
|
||||
REMOVED=1
|
||||
find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
|
||||
log "Removing file: $FILE"
|
||||
rm -f "$FILE"
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
# Check to see if we are already running
|
||||
NUM_RUNNING=$(pgrep -cf "/bin/bash /usr/sbin/so-sensor-clean")
|
||||
[ "$NUM_RUNNING" -gt 1 ] && echo "$(date) - $NUM_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0
|
||||
|
||||
if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then
|
||||
while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
|
||||
clean
|
||||
CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
|
||||
done
|
||||
# Only one instance at a time; the lock is the fd, so it releases on any exit
|
||||
exec 9>"$LOCK" || exit 1
|
||||
if ! flock -n 9; then
|
||||
log "another so-sensor-clean is already running (lock $LOCK held); exiting"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
CUR_USAGE=$(disk_usage)
|
||||
[ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ] || exit 0
|
||||
|
||||
log "$SENSOR_DIR at ${CUR_USAGE}% (threshold ${CRIT_DISK_USAGE}%); starting cleanup"
|
||||
|
||||
PASS=0
|
||||
while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
|
||||
PASS=$((PASS + 1))
|
||||
if [ "$PASS" -gt "$MAX_PASSES" ]; then
|
||||
log "stopping after $MAX_PASSES passes; $SENSOR_DIR still at ${CUR_USAGE}%"
|
||||
break
|
||||
fi
|
||||
|
||||
REMOVED=0
|
||||
BEFORE=$(disk_avail)
|
||||
clean
|
||||
CUR_USAGE=$(disk_usage)
|
||||
|
||||
if [ "$REMOVED" -eq 0 ]; then
|
||||
log "nothing left to remove in $ZEEK_LOGS, $STRELKA_FILES, $SURICATA_LOGS, $PCAPS; $SENSOR_DIR still at ${CUR_USAGE}% - space is consumed outside of NSM cleanup scope"
|
||||
break
|
||||
fi
|
||||
if [ "$(disk_avail)" -le "$BEFORE" ]; then
|
||||
log "pass $PASS freed no space; $SENSOR_DIR still at ${CUR_USAGE}% - stopping until next run"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
Reference in New Issue
Block a user