From 48801da44e9df1c589a165fa42f7778bfed26b93 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 26 Sep 2023 18:12:20 -0400 Subject: [PATCH 1/2] log check tool initial --- salt/common/tools/sbin/so-log-check | 174 ++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100755 salt/common/tools/sbin/so-log-check diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check new file mode 100755 index 000000000..6a3ca9876 --- /dev/null +++ b/salt/common/tools/sbin/so-log-check @@ -0,0 +1,174 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +RECENT_LOG_LINES=200 +EXCLUDE_STARTUP_ERRORS=N +EXCLUDE_FALSE_POSITIVE_ERRORS=N +EXCLUDE_KNOWN_ERRORS=N + +while [[ $# -gt 0 ]]; do + case $1 in + --exclude-connection-errors) + EXCLUDE_STARTUP_ERRORS=Y + ;; + --exclude-false-positives) + EXCLUDE_FALSE_POSITIVE_ERRORS=Y + ;; + --exclude-known-errors) + EXCLUDE_KNOWN_ERRORS=Y + ;; + --unknown) + EXCLUDE_STARTUP_ERRORS=Y + EXCLUDE_FALSE_POSITIVE_ERRORS=Y + EXCLUDE_KNOWN_ERRORS=Y + ;; + --recent-log-lines) + shift + RECENT_LOG_LINES=$1 + ;; + *) + echo "Usage: $0 [options]" + echo "" + echo "where options are:" + echo " --recent-log-lines N looks at the most recent N log lines per file or container; defaults to 200" + echo " --exclude-connection-errors exclude errors caused by a recent server or container restart" + echo " --exclude-false-positives exclude logs that are not actual errors but contain the error string" + echo " --exclude-known-errors exclude errors that are known and non-critical issues" + echo " --unknown exclude everthing mentioned above; only show unknown errors" + echo "" + echo "A non-zero return value indicates errors were found" + exit 1 + ;; + esac + shift +done + +echo "Security Onion Log Check - $(date)" +echo "-------------------------------------------" +echo "" +echo "- RECENT_LOG_LINES: $RECENT_LOG_LINES" +echo "- EXCLUDE_STARTUP_ERRORS: $EXCLUDE_STARTUP_ERRORS" +echo "- EXCLUDE_FALSE_POSITIVE_ERRORS: $EXCLUDE_FALSE_POSITIVE_ERRORS" +echo "- EXCLUDE_KNOWN_ERRORS: $EXCLUDE_KNOWN_ERRORS" +echo "" + +function status() { + header "$1" +} + +function exclude_container() { + name=$1 + + exclude_id=$(docker ps | grep "$name" | awk '{print $1}') + if [[ -n "$exclude_id" ]]; then + CONTAINER_IDS=$(echo $CONTAINER_IDS | sed -e "s/$exclude_id//g") + return $? + fi + return $? +} + +function exclude_log() { + name=$1 + + LOG_FILES=$(echo "$LOG_FILES" | sed -e "s/$name//g") +} + +function check_for_errors() { + if cat /tmp/log_check | grep -i error | grep -vEi "$EXCLUDED_ERRORS"; then + RESULT=1 + fi +} + +EXCLUDED_ERRORS="__LOG_CHECK_PLACEHOLDER_EXCLUSION__" + +if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|database is locked" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|econnreset" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unreachable" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no route to host" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|not running" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unavailable" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|request.py" # server not yet ready (python stack output) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|httperror" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|servfail" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|connection refused" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing shards" # server not yet ready +fi + +if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|elastalert_status_error" # false positive + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|elastalert_error.json" # false positive + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error: '0'" # false positive + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|errors_index" # false positive + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|noerror" # false positive + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|emerging-all.rules" # false positive (error in rulename) +fi + +if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|eof" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|_ml" # Elastic ML errors + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|bookkeeper" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|noindices" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to start transient scope" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200" # request successful, contained error string in content + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|so-user.lock exists" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|systemd-run" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|retcode: 1" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|telemetry-task" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|redisqueue" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|fleet_detail_query" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|num errors=0" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|provisioning/alerting" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|provisioning/notifiers" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|provisoning/plugins" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|active-responses.log" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|scanentropy" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integration policy" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|blob unknown" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|token required" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|zeekcaptureloss" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unable to create detection" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error installing new prebuilt rules" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|parent.error" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|req.LocalMeta.host.ip" # known issue + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sendmail" # zeek + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|example" # example test data + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|so_long_term" # setup in progress, influxdb not yet setup + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|stats.log" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context deadline exceeded" +fi + +RESULT=0 + +# Check Security Onion container stdout/stderr logs +CONTAINER_IDS=$(docker ps -q) +exclude_container so-kibana +exclude_container so-idstools + +for container_id in $CONTAINER_IDS; do + status "Checking container $container_id" + docker logs -n $RECENT_LOG_LINES $container_id > /tmp/log_check 2>&1 + check_for_errors +done + +# Check Security Onion related log files +LOG_FILES=$(find /opt/so/log/ /nsm -name \*.log) +exclude_log "\s?.*kibana.log" +LOG_FILES="$LOG_FILES /var/log/cron" + +for log_file in $LOG_FILES; do + status "Checking log file $log_file" + tail -n $RECENT_LOG_LINES $log_file > /tmp/log_check + check_for_errors +done + +exit $RESULT \ No newline at end of file From 2c8d413f168fe2dedcf9e7eb91dfc806377ee3b5 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 26 Sep 2023 18:14:37 -0400 Subject: [PATCH 2/2] log check tool initial --- salt/common/tools/sbin/so-log-check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 6a3ca9876..752a6d51e 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -40,7 +40,7 @@ while [[ $# -gt 0 ]]; do echo " --exclude-connection-errors exclude errors caused by a recent server or container restart" echo " --exclude-false-positives exclude logs that are not actual errors but contain the error string" echo " --exclude-known-errors exclude errors that are known and non-critical issues" - echo " --unknown exclude everthing mentioned above; only show unknown errors" + echo " --unknown exclude everything mentioned above; only show unknown errors" echo "" echo "A non-zero return value indicates errors were found" exit 1