mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2025-12-06 09:12:45 +01:00
Merge remote-tracking branch 'origin/2.4/dev' into issue/11390
This commit is contained in:
m0duspwnens
207
salt/common/tools/sbin/so-log-check
Executable file
207
salt/common/tools/sbin/so-log-check
Executable file
@@ -0,0 +1,207 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
. /usr/sbin/so-common
|
||||
|
||||
RECENT_LOG_LINES=200
|
||||
EXCLUDE_STARTUP_ERRORS=N
|
||||
EXCLUDE_FALSE_POSITIVE_ERRORS=N
|
||||
EXCLUDE_KNOWN_ERRORS=N
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--exclude-connection-errors)
|
||||
EXCLUDE_STARTUP_ERRORS=Y
|
||||
;;
|
||||
--exclude-false-positives)
|
||||
EXCLUDE_FALSE_POSITIVE_ERRORS=Y
|
||||
;;
|
||||
--exclude-known-errors)
|
||||
EXCLUDE_KNOWN_ERRORS=Y
|
||||
;;
|
||||
--unknown)
|
||||
EXCLUDE_STARTUP_ERRORS=Y
|
||||
EXCLUDE_FALSE_POSITIVE_ERRORS=Y
|
||||
EXCLUDE_KNOWN_ERRORS=Y
|
||||
;;
|
||||
--recent-log-lines)
|
||||
shift
|
||||
RECENT_LOG_LINES=$1
|
||||
;;
|
||||
*)
|
||||
echo "Usage: $0 [options]"
|
||||
echo ""
|
||||
echo "where options are:"
|
||||
echo " --recent-log-lines N looks at the most recent N log lines per file or container; defaults to 200"
|
||||
echo " --exclude-connection-errors exclude errors caused by a recent server or container restart"
|
||||
echo " --exclude-false-positives exclude logs that are known false positives"
|
||||
echo " --exclude-known-errors exclude errors that are known and non-critical issues"
|
||||
echo " --unknown exclude everything mentioned above; only show unknown errors"
|
||||
echo ""
|
||||
echo "A non-zero return value indicates errors were found"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
echo "Security Onion Log Check - $(date)"
|
||||
echo "-------------------------------------------"
|
||||
echo ""
|
||||
echo "- RECENT_LOG_LINES: $RECENT_LOG_LINES"
|
||||
echo "- EXCLUDE_STARTUP_ERRORS: $EXCLUDE_STARTUP_ERRORS"
|
||||
echo "- EXCLUDE_FALSE_POSITIVE_ERRORS: $EXCLUDE_FALSE_POSITIVE_ERRORS"
|
||||
echo "- EXCLUDE_KNOWN_ERRORS: $EXCLUDE_KNOWN_ERRORS"
|
||||
echo ""
|
||||
|
||||
function status() {
|
||||
header "$1"
|
||||
}
|
||||
|
||||
function exclude_container() {
|
||||
name=$1
|
||||
|
||||
exclude_id=$(docker ps | grep "$name" | awk '{print $1}')
|
||||
if [[ -n "$exclude_id" ]]; then
|
||||
CONTAINER_IDS=$(echo $CONTAINER_IDS | sed -e "s/$exclude_id//g")
|
||||
return $?
|
||||
fi
|
||||
return $?
|
||||
}
|
||||
|
||||
function exclude_log() {
|
||||
name=$1
|
||||
|
||||
cat /tmp/log_check_files | grep -v $name > /tmp/log_check_files.new
|
||||
mv /tmp/log_check_files.new /tmp/log_check_files
|
||||
}
|
||||
|
||||
function check_for_errors() {
|
||||
if cat /tmp/log_check | grep -i error | grep -vEi "$EXCLUDED_ERRORS"; then
|
||||
RESULT=1
|
||||
fi
|
||||
}
|
||||
|
||||
EXCLUDED_ERRORS="__LOG_CHECK_PLACEHOLDER_EXCLUSION__"
|
||||
|
||||
if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|database is locked" # server not yet ready
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|econnreset" # server not yet ready
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unreachable" # server not yet ready
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no route to host" # server not yet ready
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|not running" # server not yet ready
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unavailable" # server not yet ready
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|request.py" # server not yet ready (python stack output)
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|httperror" # server not yet ready
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|servfail" # server not yet ready
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|connect" # server not yet ready
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing shards" # server not yet ready
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to send metrics" # server not yet ready
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|influxsize kbytes" # server not yet ready (telegraf waiting on influx)
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|expected field at" # server not yet ready (telegraf waiting on health data)
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|cached the public key" # server not yet ready (salt minion waiting on key acceptance)
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no ingest nodes" # server not yet ready (logstash waiting on elastic)
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to poll" # server not yet ready (sensoroni waiting on soc)
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|minions returned with non" # server not yet ready (salt waiting on minions)
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|so_long_term" # server not yet ready (influxdb not yet setup)
|
||||
fi
|
||||
|
||||
if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|elastalert_status_error" # false positive
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|elastalert_error.json" # false positive
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error: '0'" # false positive
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|errors_index" # false positive
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|noerror" # false positive
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error-template" # false positive (elastic templates)
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|deprecated" # false positive (playbook)
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|could cause errors" # false positive (playbook)
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|id.orig_h" # false positive (zeek test data)
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|emerging-all.rules" # false positive (error in rulename)
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|invalid query input" # false positive (Invalid user input in hunt query)
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|example" # false positive (example test data)
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200" # false positive (request successful, contained error string in content)
|
||||
fi
|
||||
|
||||
if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|eof"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|raise error" # redis/python generic stack line, rely on other lines for actual error
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|fail\\(error\\)" # redis/python generic stack line, rely on other lines for actual error
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|urlerror" # idstools connection timeout
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeouterror" # idstools connection timeout
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|forbidden" # playbook
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|_ml" # Elastic ML errors
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context canceled" # elastic agent during shutdown
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|bookkeeper"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|noindices"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to start transient scope"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|so-user.lock exists"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|systemd-run"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|retcode: 1"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|telemetry-task"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|redisqueue"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|fleet_detail_query"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|num errors=0"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|provisioning/alerting"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|provisioning/notifiers"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|provisoning/plugins"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|active-responses.log"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|scanentropy"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integration policy"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|blob unknown"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|token required"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|zeekcaptureloss"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unable to create detection"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error installing new prebuilt rules"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|parent.error"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|req.LocalMeta.host.ip" # known issue in GH
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sendmail" # zeek
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|stats.log"
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context deadline exceeded"
|
||||
fi
|
||||
|
||||
RESULT=0
|
||||
|
||||
# Check Security Onion container stdout/stderr logs
|
||||
CONTAINER_IDS=$(docker ps -q)
|
||||
exclude_container so-kibana
|
||||
exclude_container so-idstools
|
||||
|
||||
for container_id in $CONTAINER_IDS; do
|
||||
status "Checking container $container_id"
|
||||
docker logs -n $RECENT_LOG_LINES $container_id > /tmp/log_check 2>&1
|
||||
check_for_errors
|
||||
done
|
||||
|
||||
# Check Security Onion related log files
|
||||
find /opt/so/log/ /nsm -name \*.log > /tmp/log_check_files
|
||||
if [[ -f /var/log/cron ]]; then
|
||||
echo "/var/log/cron" >> /tmp/log_check_files
|
||||
fi
|
||||
exclude_log "kibana.log"
|
||||
exclude_log "spool"
|
||||
exclude_log "import"
|
||||
|
||||
for log_file in $(cat /tmp/log_check_files); do
|
||||
status "Checking log file $log_file"
|
||||
tail -n $RECENT_LOG_LINES $log_file > /tmp/log_check
|
||||
check_for_errors
|
||||
done
|
||||
|
||||
# Cleanup temp files
|
||||
rm -f /tmp/log_check_files
|
||||
rm -f /tmp/log_check
|
||||
|
||||
if [[ $RESULT -eq 0 ]]; then
|
||||
echo -e "\nResult: No errors found"
|
||||
else
|
||||
echo -e "\nResult: One or more errors found"
|
||||
fi
|
||||
|
||||
exit $RESULT
|
||||
@@ -8,6 +8,7 @@
|
||||
worker_processes auto;
|
||||
error_log /var/log/nginx/error.log;
|
||||
pid /run/nginx.pid;
|
||||
user nobody;
|
||||
|
||||
include /usr/share/nginx/modules/*.conf;
|
||||
|
||||
|
||||
@@ -108,6 +108,11 @@ filecheck_stdout.log:
|
||||
|
||||
{% if GLOBALS.md_engine == 'ZEEK' %}
|
||||
|
||||
remove_filecheck_run:
|
||||
cron.absent:
|
||||
- identifier: filecheck_run
|
||||
- user: socore
|
||||
|
||||
filecheck_run_socore:
|
||||
cron.present:
|
||||
- name: 'ps -ef | grep filecheck | grep -v grep > /dev/null 2>&1 || python3 /opt/so/conf/strelka/filecheck >> /opt/so/log/strelka/filecheck_stdout.log 2>&1 &'
|
||||
@@ -121,6 +126,11 @@ remove_filecheck_run_suricata:
|
||||
|
||||
{% elif GLOBALS.md_engine == 'SURICATA'%}
|
||||
|
||||
remove_filecheck_run:
|
||||
cron.absent:
|
||||
- identifier: filecheck_run
|
||||
- user: suricata
|
||||
|
||||
filecheck_run_suricata:
|
||||
cron.present:
|
||||
- name: 'ps -ef | grep filecheck | grep -v grep > /dev/null 2>&1 || python3 /opt/so/conf/strelka/filecheck >> /opt/so/log/strelka/filecheck_stdout.log 2>&1 &'
|
||||
|
||||
@@ -11,7 +11,6 @@ telegraf:
|
||||
quiet: 'false'
|
||||
scripts:
|
||||
eval:
|
||||
- beatseps.sh
|
||||
- checkfiles.sh
|
||||
- influxdbsize.sh
|
||||
- oldpcap.sh
|
||||
@@ -23,7 +22,6 @@ telegraf:
|
||||
- zeekcaptureloss.sh
|
||||
- zeekloss.sh
|
||||
standalone:
|
||||
- beatseps.sh
|
||||
- checkfiles.sh
|
||||
- eps.sh
|
||||
- influxdbsize.sh
|
||||
@@ -36,13 +34,11 @@ telegraf:
|
||||
- zeekcaptureloss.sh
|
||||
- zeekloss.sh
|
||||
manager:
|
||||
- beatseps.sh
|
||||
- influxdbsize.sh
|
||||
- raid.sh
|
||||
- redis.sh
|
||||
- sostatus.sh
|
||||
managersearch:
|
||||
- beatseps.sh
|
||||
- eps.sh
|
||||
- influxdbsize.sh
|
||||
- raid.sh
|
||||
@@ -51,7 +47,6 @@ telegraf:
|
||||
import:
|
||||
- sostatus.sh
|
||||
sensor:
|
||||
- beatseps.sh
|
||||
- checkfiles.sh
|
||||
- oldpcap.sh
|
||||
- raid.sh
|
||||
@@ -61,7 +56,6 @@ telegraf:
|
||||
- zeekcaptureloss.sh
|
||||
- zeekloss.sh
|
||||
heavynode:
|
||||
- beatseps.sh
|
||||
- checkfiles.sh
|
||||
- eps.sh
|
||||
- oldpcap.sh
|
||||
@@ -75,12 +69,10 @@ telegraf:
|
||||
idh:
|
||||
- sostatus.sh
|
||||
searchnode:
|
||||
- beatseps.sh
|
||||
- eps.sh
|
||||
- raid.sh
|
||||
- sostatus.sh
|
||||
receiver:
|
||||
- beatseps.sh
|
||||
- eps.sh
|
||||
- raid.sh
|
||||
- redis.sh
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
|
||||
|
||||
# if this script isn't already running
|
||||
if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
|
||||
|
||||
PREVCOUNTFILE='/tmp/beatseps.txt'
|
||||
EVENTCOUNTCURRENT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.acked')"
|
||||
FAILEDEVENTCOUNT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.failed')"
|
||||
|
||||
if [ ! -z "$EVENTCOUNTCURRENT" ]; then
|
||||
|
||||
if [ -f "$PREVCOUNTFILE" ]; then
|
||||
EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE`
|
||||
else
|
||||
echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE
|
||||
# the division by 30 is because the agent interval is 30 seconds
|
||||
EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30))
|
||||
if [ "$EVENTS" -lt 0 ]; then
|
||||
EVENTS=0
|
||||
fi
|
||||
|
||||
echo "fbstats eps=${EVENTS%%.*},failed=$FAILEDEVENTCOUNT"
|
||||
fi
|
||||
|
||||
fi
|
||||
|
||||
exit 0
|
||||
@@ -8,9 +8,9 @@ zeek:
|
||||
buffer: 128*1024*1024
|
||||
zeekctl:
|
||||
MailTo: root@localhost
|
||||
MailConnectionSummary: 1
|
||||
MailConnectionSummary: 0
|
||||
MinDiskSpace: 5
|
||||
MailHostUpDown: 1
|
||||
MailHostUpDown: 0
|
||||
LogRotationInterval: 3600
|
||||
LogExpireInterval: 0
|
||||
StatsLogEnable: 1
|
||||
@@ -28,7 +28,6 @@ zeek:
|
||||
- misc/loaded-scripts
|
||||
- tuning/defaults
|
||||
- misc/capture-loss
|
||||
- misc/stats
|
||||
- frameworks/software/vulnerable
|
||||
- frameworks/software/version-changes
|
||||
- protocols/ftp/software
|
||||
|
||||
Reference in New Issue
Block a user