From 48801da44e9df1c589a165fa42f7778bfed26b93 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Tue, 26 Sep 2023 18:12:20 -0400
Subject: [PATCH 01/13] log check tool initial

---
 salt/common/tools/sbin/so-log-check | 174 ++++++++++++++++++++++++++++
 1 file changed, 174 insertions(+)
 create mode 100755 salt/common/tools/sbin/so-log-check

diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check
new file mode 100755
index 000000000..6a3ca9876
--- /dev/null
+++ b/salt/common/tools/sbin/so-log-check
@@ -0,0 +1,174 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+RECENT_LOG_LINES=200
+EXCLUDE_STARTUP_ERRORS=N
+EXCLUDE_FALSE_POSITIVE_ERRORS=N
+EXCLUDE_KNOWN_ERRORS=N
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --exclude-connection-errors)
+            EXCLUDE_STARTUP_ERRORS=Y
+            ;;
+        --exclude-false-positives)
+            EXCLUDE_FALSE_POSITIVE_ERRORS=Y
+            ;;
+        --exclude-known-errors)
+            EXCLUDE_KNOWN_ERRORS=Y
+            ;;
+        --unknown)
+            EXCLUDE_STARTUP_ERRORS=Y
+            EXCLUDE_FALSE_POSITIVE_ERRORS=Y
+            EXCLUDE_KNOWN_ERRORS=Y
+            ;;
+        --recent-log-lines)
+            shift
+            RECENT_LOG_LINES=$1
+            ;;
+        *)
+            echo "Usage: $0 [options]"
+            echo ""
+            echo "where options are:"
+            echo "  --recent-log-lines N            looks at the most recent N log lines per file or container; defaults to 200"
+            echo "  --exclude-connection-errors     exclude errors caused by a recent server or container restart"
+            echo "  --exclude-false-positives       exclude logs that are not actual errors but contain the error string"
+            echo "  --exclude-known-errors          exclude errors that are known and non-critical issues"
+            echo "  --unknown                       exclude everthing mentioned above; only show unknown errors"
+            echo ""
+            echo "A non-zero return value indicates errors were found"
+            exit 1
+            ;;
+    esac
+    shift
+done
+
+echo "Security Onion Log Check - $(date)"
+echo "-------------------------------------------"
+echo ""
+echo "- RECENT_LOG_LINES:                $RECENT_LOG_LINES"
+echo "- EXCLUDE_STARTUP_ERRORS:          $EXCLUDE_STARTUP_ERRORS"
+echo "- EXCLUDE_FALSE_POSITIVE_ERRORS:   $EXCLUDE_FALSE_POSITIVE_ERRORS"
+echo "- EXCLUDE_KNOWN_ERRORS:            $EXCLUDE_KNOWN_ERRORS"
+echo ""
+
+function status() {
+    header "$1"
+}
+
+function exclude_container() {
+    name=$1
+
+    exclude_id=$(docker ps | grep "$name" | awk '{print $1}')
+    if [[ -n "$exclude_id" ]]; then
+        CONTAINER_IDS=$(echo $CONTAINER_IDS | sed -e "s/$exclude_id//g")
+        return $?
+    fi
+    return $?
+}
+
+function exclude_log() {
+    name=$1
+
+    LOG_FILES=$(echo "$LOG_FILES" | sed -e "s/$name//g")
+}
+
+function check_for_errors() {
+    if cat /tmp/log_check | grep -i error | grep -vEi "$EXCLUDED_ERRORS"; then
+        RESULT=1
+    fi
+}
+
+EXCLUDED_ERRORS="__LOG_CHECK_PLACEHOLDER_EXCLUSION__"
+
+if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|database is locked"           # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|econnreset"                   # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unreachable"                  # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no route to host"             # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|not running"                  # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unavailable"                  # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|request.py"                   # server not yet ready (python stack output)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|httperror"                    # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|servfail"                     # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|connection refused"           # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing shards"               # server not yet ready
+fi
+
+if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|elastalert_status_error"      # false positive
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|elastalert_error.json"        # false positive
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error: '0'"                   # false positive
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|errors_index"                 # false positive
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|noerror"                      # false positive
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|emerging-all.rules"           # false positive (error in rulename)
+fi
+
+if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|eof"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|_ml"                          # Elastic ML errors 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|bookkeeper"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|noindices"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to start transient scope"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200"                   # request successful, contained error string in content
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|so-user.lock exists"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|systemd-run"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|retcode: 1"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|telemetry-task"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|redisqueue"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|fleet_detail_query"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|num errors=0"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|provisioning/alerting"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|provisioning/notifiers"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|provisoning/plugins"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|active-responses.log"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|scanentropy"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integration policy"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|blob unknown"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|token required"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|zeekcaptureloss"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unable to create detection"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error installing new prebuilt rules"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|parent.error"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|req.LocalMeta.host.ip"        # known issue 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sendmail"                     # zeek
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|example"                      # example test data
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|so_long_term"                 # setup in progress, influxdb not yet setup
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|stats.log"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context deadline exceeded"
+fi
+
+RESULT=0
+
+# Check Security Onion container stdout/stderr logs
+CONTAINER_IDS=$(docker ps -q)
+exclude_container so-kibana
+exclude_container so-idstools
+
+for container_id in $CONTAINER_IDS; do
+    status "Checking container $container_id"
+    docker logs -n $RECENT_LOG_LINES $container_id > /tmp/log_check 2>&1
+    check_for_errors
+done
+
+# Check Security Onion related log files
+LOG_FILES=$(find /opt/so/log/ /nsm -name \*.log)
+exclude_log "\s?.*kibana.log"
+LOG_FILES="$LOG_FILES /var/log/cron"
+
+for log_file in $LOG_FILES; do
+    status "Checking log file $log_file"
+    tail -n $RECENT_LOG_LINES $log_file > /tmp/log_check
+    check_for_errors
+done
+
+exit $RESULT
\ No newline at end of file

From 2c8d413f168fe2dedcf9e7eb91dfc806377ee3b5 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Tue, 26 Sep 2023 18:14:37 -0400
Subject: [PATCH 02/13] log check tool initial

---
 salt/common/tools/sbin/so-log-check | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check
index 6a3ca9876..752a6d51e 100755
--- a/salt/common/tools/sbin/so-log-check
+++ b/salt/common/tools/sbin/so-log-check
@@ -40,7 +40,7 @@ while [[ $# -gt 0 ]]; do
             echo "  --exclude-connection-errors     exclude errors caused by a recent server or container restart"
             echo "  --exclude-false-positives       exclude logs that are not actual errors but contain the error string"
             echo "  --exclude-known-errors          exclude errors that are known and non-critical issues"
-            echo "  --unknown                       exclude everthing mentioned above; only show unknown errors"
+            echo "  --unknown                       exclude everything mentioned above; only show unknown errors"
             echo ""
             echo "A non-zero return value indicates errors were found"
             exit 1

From 9c854a13ccf44b56163ed90d9ae8e26d163a0ff2 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Tue, 26 Sep 2023 21:41:44 -0400
Subject: [PATCH 03/13] skip zeek spool logs due to test data false positives

---
 salt/common/tools/sbin/so-log-check | 32 ++++++++++++++++++++++-------
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check
index 752a6d51e..6169e9720 100755
--- a/salt/common/tools/sbin/so-log-check
+++ b/salt/common/tools/sbin/so-log-check
@@ -38,7 +38,7 @@ while [[ $# -gt 0 ]]; do
             echo "where options are:"
             echo "  --recent-log-lines N            looks at the most recent N log lines per file or container; defaults to 200"
             echo "  --exclude-connection-errors     exclude errors caused by a recent server or container restart"
-            echo "  --exclude-false-positives       exclude logs that are not actual errors but contain the error string"
+            echo "  --exclude-false-positives       exclude logs that are known false positives"
             echo "  --exclude-known-errors          exclude errors that are known and non-critical issues"
             echo "  --unknown                       exclude everything mentioned above; only show unknown errors"
             echo ""
@@ -76,7 +76,8 @@ function exclude_container() {
 function exclude_log() {
     name=$1
 
-    LOG_FILES=$(echo "$LOG_FILES" | sed -e "s/$name//g")
+    cat /tmp/log_check_files | grep -v $name > /tmp/log_check_files.new
+    mv /tmp/log_check_files.new /tmp/log_check_files
 }
 
 function check_for_errors() {
@@ -97,8 +98,10 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|request.py"                   # server not yet ready (python stack output)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|httperror"                    # server not yet ready
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|servfail"                     # server not yet ready
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|connection refused"           # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|connect"                      # server not yet ready
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing shards"               # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to send metrics"       # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|influxsize kbytes"            # server not yet ready (telegraf)
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
@@ -107,11 +110,15 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error: '0'"                   # false positive
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|errors_index"                 # false positive
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|noerror"                      # false positive
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|deprecated"                   # false positive (playbook)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|could cause errors"           # false positive (playbook)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|id.orig_h"                    # false positive (zeek test data)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|emerging-all.rules"           # false positive (error in rulename)
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|eof"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|forbidden"                    # playbook
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|_ml"                          # Elastic ML errors 
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration"
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets"
@@ -161,14 +168,25 @@ for container_id in $CONTAINER_IDS; do
 done
 
 # Check Security Onion related log files
-LOG_FILES=$(find /opt/so/log/ /nsm -name \*.log)
-exclude_log "\s?.*kibana.log"
-LOG_FILES="$LOG_FILES /var/log/cron"
+find /opt/so/log/ /nsm -name \*.log > /tmp/log_check_files
+echo "/var/log/cron" >> /tmp/log_check_files
+exclude_log "kibana.log"
+exclude_log "spool"
 
-for log_file in $LOG_FILES; do
+for log_file in $(cat /tmp/log_check_files); do
     status "Checking log file $log_file"
     tail -n $RECENT_LOG_LINES $log_file > /tmp/log_check
     check_for_errors
 done
 
+# Cleanup temp files
+rm -f /tmp/log_check_files
+rm -f /tmp/log_check
+
+if [[ $RESULT -eq 0 ]]; then
+    echo -e "\nResult: No errors found"
+else
+    echo -e "\nResult: One or more errors found"
+fi
+
 exit $RESULT
\ No newline at end of file

From b47d915cb6318bfa8af3a29763fe38015764ec2f Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Wed, 27 Sep 2023 09:30:19 -0400
Subject: [PATCH 04/13] don't inspect imported zeek output

---
 salt/common/tools/sbin/so-log-check | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check
index 6169e9720..621f0027a 100755
--- a/salt/common/tools/sbin/so-log-check
+++ b/salt/common/tools/sbin/so-log-check
@@ -101,7 +101,12 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|connect"                      # server not yet ready
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing shards"               # server not yet ready
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to send metrics"       # server not yet ready
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|influxsize kbytes"            # server not yet ready (telegraf)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|influxsize kbytes"            # server not yet ready (telegraf waiting on influx)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|expected field at"            # server not yet ready (telegraf waiting on health data)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|cached the public key"        # server not yet ready (salt minion waiting on key acceptance)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no ingest nodes"              # server not yet ready (logstash waiting on elastic)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to poll"               # server not yet ready (sensoroni waiting on soc)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|minions returned with non"    # server not yet ready (salt waiting on minions)
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
@@ -110,14 +115,18 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error: '0'"                   # false positive
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|errors_index"                 # false positive
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|noerror"                      # false positive
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error-template"               # false positive (elastic templates)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|deprecated"                   # false positive (playbook)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|could cause errors"           # false positive (playbook)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|id.orig_h"                    # false positive (zeek test data)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|emerging-all.rules"           # false positive (error in rulename)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|invalid query input"          # false positive (Invalid user input in hunt query)
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|eof"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|urlerror"                     # idstools connection timeout
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeouterror"                 # idstools connection timeout
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|forbidden"                    # playbook
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|_ml"                          # Elastic ML errors 
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration"
@@ -146,7 +155,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unable to create detection"
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error installing new prebuilt rules"
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|parent.error"
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|req.LocalMeta.host.ip"        # known issue 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|req.LocalMeta.host.ip"        # known issue in GH
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sendmail"                     # zeek
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|example"                      # example test data
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|so_long_term"                 # setup in progress, influxdb not yet setup
@@ -172,6 +181,7 @@ find /opt/so/log/ /nsm -name \*.log > /tmp/log_check_files
 echo "/var/log/cron" >> /tmp/log_check_files
 exclude_log "kibana.log"
 exclude_log "spool"
+exclude_log "import"
 
 for log_file in $(cat /tmp/log_check_files); do
     status "Checking log file $log_file"

From 05e7c32cf9f6d9246e458d33d0b13aa43b337d06 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Wed, 27 Sep 2023 10:08:08 -0400
Subject: [PATCH 05/13] remove duplicate filecheck_run cron

---
 salt/strelka/filestream/config.sls | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/salt/strelka/filestream/config.sls b/salt/strelka/filestream/config.sls
index 833a08505..0f9f38914 100644
--- a/salt/strelka/filestream/config.sls
+++ b/salt/strelka/filestream/config.sls
@@ -108,6 +108,11 @@ filecheck_stdout.log:
 
 {% if GLOBALS.md_engine == 'ZEEK' %}
 
+remove_filecheck_run:
+  cron.absent:
+    - identifier: filecheck_run
+    - user: socore
+
 filecheck_run_socore:
   cron.present:
     - name: 'ps -ef | grep filecheck | grep -v grep > /dev/null 2>&1 || python3 /opt/so/conf/strelka/filecheck >> /opt/so/log/strelka/filecheck_stdout.log 2>&1 &'
@@ -121,6 +126,11 @@ remove_filecheck_run_suricata:
 
 {% elif GLOBALS.md_engine == 'SURICATA'%}
 
+remove_filecheck_run:
+  cron.absent:
+    - identifier: filecheck_run
+    - user: suricata
+
 filecheck_run_suricata:
   cron.present:
     - name: 'ps -ef | grep filecheck | grep -v grep > /dev/null 2>&1 || python3 /opt/so/conf/strelka/filecheck >> /opt/so/log/strelka/filecheck_stdout.log 2>&1 &'

From c4fea9cb9da5a5044ad1c7be9aeca417b4e6ab96 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 27 Sep 2023 11:03:58 -0400
Subject: [PATCH 06/13] Update nginx.conf

---
 salt/nginx/etc/nginx.conf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf
index 795663384..3ef0c5c1f 100644
--- a/salt/nginx/etc/nginx.conf
+++ b/salt/nginx/etc/nginx.conf
@@ -8,6 +8,7 @@
 worker_processes auto;
 error_log /var/log/nginx/error.log;
 pid /run/nginx.pid;
+user nobody;
 
 include /usr/share/nginx/modules/*.conf;
 

From 87cc389088eb0f1886c73610f70e3701eb625d20 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Wed, 27 Sep 2023 15:36:13 -0400
Subject: [PATCH 07/13] deb OS doesn't use /var/log/cron, skip

---
 salt/common/tools/sbin/so-log-check | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check
index 621f0027a..d377d0236 100755
--- a/salt/common/tools/sbin/so-log-check
+++ b/salt/common/tools/sbin/so-log-check
@@ -178,7 +178,9 @@ done
 
 # Check Security Onion related log files
 find /opt/so/log/ /nsm -name \*.log > /tmp/log_check_files
-echo "/var/log/cron" >> /tmp/log_check_files
+if [[ -f /var/log/cron ]]; then
+    echo "/var/log/cron" >> /tmp/log_check_files
+fi
 exclude_log "kibana.log"
 exclude_log "spool"
 exclude_log "import"

From f094b1162d7c23cd5bb2d98d7e34fb2f7b6afecf Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 27 Sep 2023 15:48:05 -0400
Subject: [PATCH 08/13] Update defaults.yaml

---
 salt/zeek/defaults.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/zeek/defaults.yaml b/salt/zeek/defaults.yaml
index 8e6814b2e..783c38820 100644
--- a/salt/zeek/defaults.yaml
+++ b/salt/zeek/defaults.yaml
@@ -8,9 +8,9 @@ zeek:
       buffer: 128*1024*1024
     zeekctl:
       MailTo: root@localhost
-      MailConnectionSummary: 1
+      MailConnectionSummary: 0
       MinDiskSpace: 5
-      MailHostUpDown: 1
+      MailHostUpDown: 0
       LogRotationInterval: 3600
       LogExpireInterval: 0
       StatsLogEnable: 1

From 4666916077db773dfb10d94e0decb85620d0c453 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Wed, 27 Sep 2023 15:48:52 -0400
Subject: [PATCH 09/13] ignore generic python stack trace log lines of code,
 rely on actual error messages

---
 salt/common/tools/sbin/so-log-check | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check
index d377d0236..9deeba1cd 100755
--- a/salt/common/tools/sbin/so-log-check
+++ b/salt/common/tools/sbin/so-log-check
@@ -125,6 +125,8 @@ fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|eof"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|raise error"                 # redis/python generic stack line, rely on other lines for actual error
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|fail(error)"                 # redis/python generic stack line, rely on other lines for actual error
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|urlerror"                     # idstools connection timeout
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeouterror"                 # idstools connection timeout
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|forbidden"                    # playbook

From 2427344dca2fc963b9b9d608f7f665141610d9aa Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 27 Sep 2023 15:58:58 -0400
Subject: [PATCH 10/13] Update defaults.yaml

---
 salt/zeek/defaults.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/salt/zeek/defaults.yaml b/salt/zeek/defaults.yaml
index 783c38820..4435670a2 100644
--- a/salt/zeek/defaults.yaml
+++ b/salt/zeek/defaults.yaml
@@ -28,7 +28,6 @@ zeek:
         - misc/loaded-scripts
         - tuning/defaults
         - misc/capture-loss
-        - misc/stats
         - frameworks/software/vulnerable
         - frameworks/software/version-changes
         - protocols/ftp/software

From 2fb73cd51621a8dea6a6f8d31597273e05ff12fb Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 27 Sep 2023 16:07:38 -0400
Subject: [PATCH 11/13] Update defaults.yaml

---
 salt/telegraf/defaults.yaml | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/salt/telegraf/defaults.yaml b/salt/telegraf/defaults.yaml
index a87fa952b..ab8679e57 100644
--- a/salt/telegraf/defaults.yaml
+++ b/salt/telegraf/defaults.yaml
@@ -11,7 +11,6 @@ telegraf:
     quiet: 'false'
   scripts:
     eval:
-      - beatseps.sh
       - checkfiles.sh
       - influxdbsize.sh
       - oldpcap.sh
@@ -23,7 +22,6 @@ telegraf:
       - zeekcaptureloss.sh
       - zeekloss.sh
     standalone:
-      - beatseps.sh
       - checkfiles.sh
       - eps.sh
       - influxdbsize.sh
@@ -36,13 +34,11 @@ telegraf:
       - zeekcaptureloss.sh
       - zeekloss.sh
     manager:
-      - beatseps.sh
       - influxdbsize.sh
       - raid.sh
       - redis.sh
       - sostatus.sh
     managersearch:
-      - beatseps.sh
       - eps.sh
       - influxdbsize.sh
       - raid.sh
@@ -51,7 +47,6 @@ telegraf:
     import:
       - sostatus.sh
     sensor:
-      - beatseps.sh
       - checkfiles.sh
       - oldpcap.sh
       - raid.sh
@@ -61,7 +56,6 @@ telegraf:
       - zeekcaptureloss.sh
       - zeekloss.sh
     heavynode:
-      - beatseps.sh
       - checkfiles.sh
       - eps.sh
       - oldpcap.sh
@@ -75,12 +69,10 @@ telegraf:
     idh:
       - sostatus.sh
     searchnode:
-      - beatseps.sh
       - eps.sh
       - raid.sh
       - sostatus.sh
     receiver:
-      - beatseps.sh
       - eps.sh
       - raid.sh
       - redis.sh

From 039d5ae9aa4a65e8e8bce9d5e45bfd51682c0e84 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 27 Sep 2023 16:09:27 -0400
Subject: [PATCH 12/13] Delete salt/telegraf/scripts/beatseps.sh

---
 salt/telegraf/scripts/beatseps.sh | 38 -------------------------------
 1 file changed, 38 deletions(-)
 delete mode 100644 salt/telegraf/scripts/beatseps.sh

diff --git a/salt/telegraf/scripts/beatseps.sh b/salt/telegraf/scripts/beatseps.sh
deleted file mode 100644
index 5f3db53f8..000000000
--- a/salt/telegraf/scripts/beatseps.sh
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/bin/bash
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-# if this script isn't already running
-if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
-
-  PREVCOUNTFILE='/tmp/beatseps.txt'
-  EVENTCOUNTCURRENT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.acked')"
-  FAILEDEVENTCOUNT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.failed')"
-
-  if [ ! -z "$EVENTCOUNTCURRENT" ]; then
-
-    if [ -f "$PREVCOUNTFILE" ]; then
-      EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE`
-    else
-      echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE
-      exit 0
-    fi
-
-    echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE
-    # the division by 30 is because the agent interval is 30 seconds
-    EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30))
-    if [ "$EVENTS" -lt 0 ]; then
-      EVENTS=0
-    fi
-
-    echo "fbstats eps=${EVENTS%%.*},failed=$FAILEDEVENTCOUNT"
-  fi
-
-fi
-
-exit 0

From 24def3a196efc12506f11de8c32e5d52cc6bd24b Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Wed, 27 Sep 2023 16:50:01 -0400
Subject: [PATCH 13/13] ignore generic python stack trace log lines of code,
 rely on actual error messages

---
 salt/common/tools/sbin/so-log-check | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check
index 9deeba1cd..f89995065 100755
--- a/salt/common/tools/sbin/so-log-check
+++ b/salt/common/tools/sbin/so-log-check
@@ -107,6 +107,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no ingest nodes"              # server not yet ready (logstash waiting on elastic)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to poll"               # server not yet ready (sensoroni waiting on soc)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|minions returned with non"    # server not yet ready (salt waiting on minions)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|so_long_term"                 # server not yet ready (influxdb not yet setup)
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
@@ -121,23 +122,25 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|id.orig_h"                    # false positive (zeek test data)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|emerging-all.rules"           # false positive (error in rulename)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|invalid query input"          # false positive (Invalid user input in hunt query)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|example"                      # false positive (example test data)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200"                   # false positive (request successful, contained error string in content)
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|eof"
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|raise error"                 # redis/python generic stack line, rely on other lines for actual error
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|fail(error)"                 # redis/python generic stack line, rely on other lines for actual error
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|raise error"                  # redis/python generic stack line, rely on other lines for actual error
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|fail\\(error\\)"              # redis/python generic stack line, rely on other lines for actual error
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|urlerror"                     # idstools connection timeout
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeouterror"                 # idstools connection timeout
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|forbidden"                    # playbook
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|_ml"                          # Elastic ML errors 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context canceled"             # elastic agent during shutdown
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration"
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets"
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed"
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|bookkeeper"
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|noindices"
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to start transient scope"
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200"                   # request successful, contained error string in content
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|so-user.lock exists"
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|systemd-run"
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|retcode: 1"
@@ -159,8 +162,6 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|parent.error"
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|req.LocalMeta.host.ip"        # known issue in GH
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sendmail"                     # zeek
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|example"                      # example test data
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|so_long_term"                 # setup in progress, influxdb not yet setup
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|stats.log"
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context deadline exceeded"
 fi