mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-09-20 22:54:26 +02:00
Josh Patterson
84cd966736
The cleanup loop had no progress check or pass limit, so once /nsm was over threshold with nothing left to reclaim it spun at full speed, writing 1.1 GB / 12.5M lines to sensor_clean.log in five hours. Stop when a pass removes nothing, when a pass frees no space, or at MAX_PASSES, and drop the per-pass "no old files" logging in favor of one actionable line. Replace the pgrep guard with flock -n. A find|while read subshell inherits the parent's argv, so pgrep -cf counted one instance as hundreds; it was also check-then-act, which let cron stack up overlapping runs. Paths now derive from SENSOR_DIR with env-overridable LOG/LOCK so the over-threshold path can be tested against a scratch filesystem.
132 lines
4.7 KiB
Bash
Executable File
132 lines
4.7 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Delete Zeek Logs based on defined CRIT_DISK_USAGE value
|
|
|
|
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
|
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
|
# https://securityonion.net/license; you may not use this file except in compliance with the
|
|
# Elastic License 2.0.
|
|
|
|
|
|
SENSOR_DIR="${SENSOR_DIR:-/nsm}"
|
|
CRIT_DISK_USAGE=90
|
|
LOG="${LOG:-/opt/so/log/sensor_clean.log}"
|
|
LOCK="${LOCK:-/var/tmp/so-sensor-clean.lock}"
|
|
MAX_PASSES=100
|
|
|
|
ZEEK_LOGS="$SENSOR_DIR/zeek/logs"
|
|
STRELKA_FILES="$SENSOR_DIR/strelka/processed"
|
|
SURICATA_LOGS="$SENSOR_DIR/suricata"
|
|
PCAPS="$SENSOR_DIR/pcapout"
|
|
|
|
log() {
|
|
echo "$(date) - $*" >>"$LOG"
|
|
}
|
|
|
|
disk_usage() {
|
|
df -P "$SENSOR_DIR" | tail -1 | awk '{print $5}' | tr -d %
|
|
}
|
|
|
|
disk_avail() {
|
|
df -P "$SENSOR_DIR" | tail -1 | awk '{print $4}'
|
|
}
|
|
|
|
# sets REMOVED=1 if anything was actually deleted
|
|
clean() {
|
|
## find the oldest Zeek logs directory
|
|
OLDEST_DIR=$(ls "$ZEEK_LOGS" 2>/dev/null | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
|
|
if [ -n "$OLDEST_DIR" ]; then
|
|
log "Removing directory: $ZEEK_LOGS/$OLDEST_DIR"
|
|
rm -rf "$ZEEK_LOGS/$OLDEST_DIR"
|
|
REMOVED=1
|
|
fi
|
|
|
|
## Remarking for now, as we are moving extracted files to /nsm/strelka/processed
|
|
## find oldest files in extracted directory and exclude today
|
|
#OLDEST_EXTRACT=$(find /nsm/zeek/extracted/complete -type f -printf '%T+ %p\n' 2>/dev/null | sort | grep -v $TODAY | head -n 1)
|
|
#if [ -z "$OLDEST_EXTRACT" -o "$OLDEST_EXTRACT" == ".." -o "$OLDEST_EXTRACT" == "." ]
|
|
#then
|
|
# echo "$(date) - No old extracted files available to clean up in /nsm/zeek/extracted/complete" >> $LOG
|
|
#else
|
|
# OLDEST_EXTRACT_DATE=`echo $OLDEST_EXTRACT | awk '{print $1}' | cut -d+ -f1`
|
|
# OLDEST_EXTRACT_FILE=`echo $OLDEST_EXTRACT | awk '{print $2}'`
|
|
# echo "$(date) - Removing extracted files for $OLDEST_EXTRACT_DATE" >> $LOG
|
|
# find /nsm/zeek/extracted/complete -type f -printf '%T+ %p\n' | grep $OLDEST_EXTRACT_DATE | awk '{print $2}' |while read FILE
|
|
# do
|
|
# echo "$(date) - Removing extracted file: $FILE" >> $LOG
|
|
# rm -f "$FILE"
|
|
# done
|
|
#fi
|
|
|
|
## Clean up Zeek extracted files processed by Strelka
|
|
OLDEST_STRELKA=$(find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
|
|
if [ -n "$OLDEST_STRELKA" ]; then
|
|
OLDEST_STRELKA_DATE=$(echo $OLDEST_STRELKA | awk '{print $1}' | cut -d+ -f1)
|
|
log "Removing extracted files for $OLDEST_STRELKA_DATE"
|
|
REMOVED=1
|
|
find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
|
|
log "Removing file: $FILE"
|
|
rm -f "$FILE"
|
|
done
|
|
fi
|
|
|
|
## Clean up Suricata log files
|
|
OLDEST_SURICATA=$(find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
|
|
if [ -n "$OLDEST_SURICATA" ]; then
|
|
OLDEST_SURICATA_DATE=$(echo $OLDEST_SURICATA | awk '{print $1}' | cut -d+ -f1)
|
|
log "Removing logs for $OLDEST_SURICATA_DATE"
|
|
REMOVED=1
|
|
find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
|
|
log "Removing file: $FILE"
|
|
rm -f "$FILE"
|
|
done
|
|
fi
|
|
|
|
## Clean up extracted pcaps
|
|
OLDEST_PCAP=$(find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
|
|
if [ -n "$OLDEST_PCAP" ]; then
|
|
OLDEST_PCAP_DATE=$(echo $OLDEST_PCAP | awk '{print $1}' | cut -d+ -f1)
|
|
log "Removing extracted files for $OLDEST_PCAP_DATE"
|
|
REMOVED=1
|
|
find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
|
|
log "Removing file: $FILE"
|
|
rm -f "$FILE"
|
|
done
|
|
fi
|
|
}
|
|
|
|
# Only one instance at a time; the lock is the fd, so it releases on any exit
|
|
exec 9>"$LOCK" || exit 1
|
|
if ! flock -n 9; then
|
|
log "another so-sensor-clean is already running (lock $LOCK held); exiting"
|
|
exit 0
|
|
fi
|
|
|
|
CUR_USAGE=$(disk_usage)
|
|
[ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ] || exit 0
|
|
|
|
log "$SENSOR_DIR at ${CUR_USAGE}% (threshold ${CRIT_DISK_USAGE}%); starting cleanup"
|
|
|
|
PASS=0
|
|
while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
|
|
PASS=$((PASS + 1))
|
|
if [ "$PASS" -gt "$MAX_PASSES" ]; then
|
|
log "stopping after $MAX_PASSES passes; $SENSOR_DIR still at ${CUR_USAGE}%"
|
|
break
|
|
fi
|
|
|
|
REMOVED=0
|
|
BEFORE=$(disk_avail)
|
|
clean
|
|
CUR_USAGE=$(disk_usage)
|
|
|
|
if [ "$REMOVED" -eq 0 ]; then
|
|
log "nothing left to remove in $ZEEK_LOGS, $STRELKA_FILES, $SURICATA_LOGS, $PCAPS; $SENSOR_DIR still at ${CUR_USAGE}% - space is consumed outside of NSM cleanup scope"
|
|
break
|
|
fi
|
|
if [ "$(disk_avail)" -le "$BEFORE" ]; then
|
|
log "pass $PASS freed no space; $SENSOR_DIR still at ${CUR_USAGE}% - stopping until next run"
|
|
break
|
|
fi
|
|
done
|