From 9c854a13ccf44b56163ed90d9ae8e26d163a0ff2 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Tue, 26 Sep 2023 21:41:44 -0400
Subject: [PATCH] skip zeek spool logs due to test data false positives

---
 salt/common/tools/sbin/so-log-check | 32 ++++++++++++++++++++++-------
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check
index 752a6d51e..6169e9720 100755
--- a/salt/common/tools/sbin/so-log-check
+++ b/salt/common/tools/sbin/so-log-check
@@ -38,7 +38,7 @@ while [[ $# -gt 0 ]]; do
             echo "where options are:"
             echo "  --recent-log-lines N            looks at the most recent N log lines per file or container; defaults to 200"
             echo "  --exclude-connection-errors     exclude errors caused by a recent server or container restart"
-            echo "  --exclude-false-positives       exclude logs that are not actual errors but contain the error string"
+            echo "  --exclude-false-positives       exclude logs that are known false positives"
             echo "  --exclude-known-errors          exclude errors that are known and non-critical issues"
             echo "  --unknown                       exclude everything mentioned above; only show unknown errors"
             echo ""
@@ -76,7 +76,8 @@ function exclude_container() {
 function exclude_log() {
     name=$1
 
-    LOG_FILES=$(echo "$LOG_FILES" | sed -e "s/$name//g")
+    cat /tmp/log_check_files | grep -v $name > /tmp/log_check_files.new
+    mv /tmp/log_check_files.new /tmp/log_check_files
 }
 
 function check_for_errors() {
@@ -97,8 +98,10 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|request.py"                   # server not yet ready (python stack output)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|httperror"                    # server not yet ready
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|servfail"                     # server not yet ready
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|connection refused"           # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|connect"                      # server not yet ready
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing shards"               # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to send metrics"       # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|influxsize kbytes"            # server not yet ready (telegraf)
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
@@ -107,11 +110,15 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error: '0'"                   # false positive
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|errors_index"                 # false positive
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|noerror"                      # false positive
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|deprecated"                   # false positive (playbook)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|could cause errors"           # false positive (playbook)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|id.orig_h"                    # false positive (zeek test data)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|emerging-all.rules"           # false positive (error in rulename)
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|eof"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|forbidden"                    # playbook
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|_ml"                          # Elastic ML errors 
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration"
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets"
@@ -161,14 +168,25 @@ for container_id in $CONTAINER_IDS; do
 done
 
 # Check Security Onion related log files
-LOG_FILES=$(find /opt/so/log/ /nsm -name \*.log)
-exclude_log "\s?.*kibana.log"
-LOG_FILES="$LOG_FILES /var/log/cron"
+find /opt/so/log/ /nsm -name \*.log > /tmp/log_check_files
+echo "/var/log/cron" >> /tmp/log_check_files
+exclude_log "kibana.log"
+exclude_log "spool"
 
-for log_file in $LOG_FILES; do
+for log_file in $(cat /tmp/log_check_files); do
     status "Checking log file $log_file"
     tail -n $RECENT_LOG_LINES $log_file > /tmp/log_check
     check_for_errors
 done
 
+# Cleanup temp files
+rm -f /tmp/log_check_files
+rm -f /tmp/log_check
+
+if [[ $RESULT -eq 0 ]]; then
+    echo -e "\nResult: No errors found"
+else
+    echo -e "\nResult: One or more errors found"
+fi
+
 exit $RESULT
\ No newline at end of file