#!/bin/bash

# Delete Zeek Logs based on defined CRIT_DISK_USAGE value

# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.


SENSOR_DIR="${SENSOR_DIR:-/nsm}"
CRIT_DISK_USAGE=90
LOG="${LOG:-/opt/so/log/sensor_clean.log}"
LOCK="${LOCK:-/var/tmp/so-sensor-clean.lock}"
MAX_PASSES=100

ZEEK_LOGS="$SENSOR_DIR/zeek/logs"
STRELKA_FILES="$SENSOR_DIR/strelka/processed"
SURICATA_LOGS="$SENSOR_DIR/suricata"
PCAPS="$SENSOR_DIR/pcapout"

log() {
	echo "$(date) - $*" >>"$LOG"
}

disk_usage() {
	df -P "$SENSOR_DIR" | tail -1 | awk '{print $5}' | tr -d %
}

disk_avail() {
	df -P "$SENSOR_DIR" | tail -1 | awk '{print $4}'
}

# sets REMOVED=1 if anything was actually deleted
clean() {
	## find the oldest Zeek logs directory
	OLDEST_DIR=$(ls "$ZEEK_LOGS" 2>/dev/null | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
	if [ -n "$OLDEST_DIR" ]; then
		log "Removing directory: $ZEEK_LOGS/$OLDEST_DIR"
		rm -rf "$ZEEK_LOGS/$OLDEST_DIR"
		REMOVED=1
	fi

	## Remarking for now, as we are moving extracted files to /nsm/strelka/processed
	## find oldest files in extracted directory and exclude today
	#OLDEST_EXTRACT=$(find /nsm/zeek/extracted/complete -type f -printf '%T+ %p\n' 2>/dev/null | sort | grep -v $TODAY | head -n 1)
	#if [ -z "$OLDEST_EXTRACT" -o "$OLDEST_EXTRACT" == ".." -o "$OLDEST_EXTRACT" == "." ]
	#then
	#        echo "$(date) - No old extracted files available to clean up in /nsm/zeek/extracted/complete" >> $LOG
	#else
	#        OLDEST_EXTRACT_DATE=`echo $OLDEST_EXTRACT | awk '{print $1}' | cut -d+ -f1`
	#        OLDEST_EXTRACT_FILE=`echo $OLDEST_EXTRACT | awk '{print $2}'`
	#        echo "$(date) - Removing extracted files for $OLDEST_EXTRACT_DATE" >> $LOG
	#        find /nsm/zeek/extracted/complete -type f -printf '%T+ %p\n' | grep $OLDEST_EXTRACT_DATE | awk '{print $2}' |while read FILE
	#        do
	#                echo "$(date) - Removing extracted file: $FILE" >> $LOG
	#                rm -f "$FILE"
	#        done
	#fi

	## Clean up Zeek extracted files processed by Strelka
	OLDEST_STRELKA=$(find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
	if [ -n "$OLDEST_STRELKA" ]; then
		OLDEST_STRELKA_DATE=$(echo $OLDEST_STRELKA | awk '{print $1}' | cut -d+ -f1)
		log "Removing extracted files for $OLDEST_STRELKA_DATE"
		REMOVED=1
		find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
			log "Removing file: $FILE"
			rm -f "$FILE"
		done
	fi

	## Clean up Suricata log files
	OLDEST_SURICATA=$(find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
	if [ -n "$OLDEST_SURICATA" ]; then
		OLDEST_SURICATA_DATE=$(echo $OLDEST_SURICATA | awk '{print $1}' | cut -d+ -f1)
		log "Removing logs for $OLDEST_SURICATA_DATE"
		REMOVED=1
		find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
			log "Removing file: $FILE"
			rm -f "$FILE"
		done
	fi

	## Clean up extracted pcaps
	OLDEST_PCAP=$(find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
	if [ -n "$OLDEST_PCAP" ]; then
		OLDEST_PCAP_DATE=$(echo $OLDEST_PCAP | awk '{print $1}' | cut -d+ -f1)
		log "Removing extracted files for $OLDEST_PCAP_DATE"
		REMOVED=1
		find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
			log "Removing file: $FILE"
			rm -f "$FILE"
		done
	fi
}

# Only one instance at a time; the lock is the fd, so it releases on any exit
exec 9>"$LOCK" || exit 1
if ! flock -n 9; then
	log "another so-sensor-clean is already running (lock $LOCK held); exiting"
	exit 0
fi

CUR_USAGE=$(disk_usage)
[ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ] || exit 0

log "$SENSOR_DIR at ${CUR_USAGE}% (threshold ${CRIT_DISK_USAGE}%); starting cleanup"

PASS=0
while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
	PASS=$((PASS + 1))
	if [ "$PASS" -gt "$MAX_PASSES" ]; then
		log "stopping after $MAX_PASSES passes; $SENSOR_DIR still at ${CUR_USAGE}%"
		break
	fi

	REMOVED=0
	BEFORE=$(disk_avail)
	clean
	CUR_USAGE=$(disk_usage)

	if [ "$REMOVED" -eq 0 ]; then
		log "nothing left to remove in $ZEEK_LOGS, $STRELKA_FILES, $SURICATA_LOGS, $PCAPS; $SENSOR_DIR still at ${CUR_USAGE}% - space is consumed outside of NSM cleanup scope"
		break
	fi
	if [ "$(disk_avail)" -le "$BEFORE" ]; then
		log "pass $PASS freed no space; $SENSOR_DIR still at ${CUR_USAGE}% - stopping until next run"
		break
	fi
done