From 1de768c18238f4af25438378d1e8443b69fd9c94 Mon Sep 17 00:00:00 2001
From: Mike Reeves <luke@geekempire.com>
Date: Thu, 6 May 2021 12:02:05 -0400
Subject: [PATCH 01/48] Update HOTFIX

---
 HOTFIX | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/HOTFIX b/HOTFIX
index b0bdf03a0..364d0a6a8 100644
--- a/HOTFIX
+++ b/HOTFIX
@@ -1 +1 @@
-GRIDFIX
\ No newline at end of file
+GRIDFIX ZEEKFIX

From 90b3462eadfc67fe08b62f92d71f86011e1a0e95 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Thu, 6 May 2021 13:29:15 -0400
Subject: [PATCH 02/48] No recurse for you

---
 salt/zeek/init.sls | 1 +
 1 file changed, 1 insertion(+)

diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls
index fe6478464..02c1cc1ba 100644
--- a/salt/zeek/init.sls
+++ b/salt/zeek/init.sls
@@ -78,6 +78,7 @@ zeekspoolownership:
   file.directory:
     - name: /nsm/zeek/spool
     - user: 937
+    - max_depth: 0
     - recurse:
       - user
 

From a49f2e2d98ae87ea0bcaa1c1fa5f5f33393836fc Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Thu, 6 May 2021 13:38:16 -0400
Subject: [PATCH 03/48] change log_level_logfile to error for
 /opt/so/log/salt/minion

---
 salt/salt/minion.sls | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls
index 1a172d851..20a6aaac7 100644
--- a/salt/salt/minion.sls
+++ b/salt/salt/minion.sls
@@ -43,12 +43,18 @@ hold_salt_packages:
 {% endfor %}
 {% endif %}
 
+remove_info_log_level_logfile:
+  file.line:
+    - name: /etc/salt/minion
+    - match: "log_level_logfile: info"
+    - mode: delete
+
 set_log_levels:
   file.append:
     - name: /etc/salt/minion
     - text:
       - "log_level: info"
-      - "log_level_logfile: info"
+      - "log_level_logfile: error"
     - listen_in:
       - service: salt_minion_service
 

From fb986b5cff14fe0a7937373397a829e2eab85358 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Thu, 6 May 2021 14:55:14 -0400
Subject: [PATCH 04/48] set both log levels to error

---
 salt/salt/minion.sls | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls
index 20a6aaac7..f5770621b 100644
--- a/salt/salt/minion.sls
+++ b/salt/salt/minion.sls
@@ -49,11 +49,17 @@ remove_info_log_level_logfile:
     - match: "log_level_logfile: info"
     - mode: delete
 
+remove_info_log_level:
+  file.line:
+    - name: /etc/salt/minion
+    - match: "log_level: info"
+    - mode: delete
+
 set_log_levels:
   file.append:
     - name: /etc/salt/minion
     - text:
-      - "log_level: info"
+      - "log_level: error"
       - "log_level_logfile: error"
     - listen_in:
       - service: salt_minion_service

From 7b29c6427b5f5efc1ef53052c65e588d992b4a90 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Fri, 7 May 2021 10:55:17 -0400
Subject: [PATCH 05/48] Add preliminary error handling in soup

---
 salt/common/tools/sbin/so-common       |  15 ++++
 salt/common/tools/sbin/so-image-common |  13 ++-
 salt/common/tools/sbin/soup            | 107 +++++++++++++++++++++----
 3 files changed, 117 insertions(+), 18 deletions(-)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index 22c7543ea..f78919226 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -324,6 +324,21 @@ retry() {
 	return 1
 }
 
+run_check_net_err() {
+	local cmd=$1
+	local err_msg=${2:-"Unknown error occured, please check /root/$WHATWOULDYOUSAYYAHDOHERE.log for details."} # Really need to rename that variable
+
+	local exit_code
+	retry 50 10 "$cmd"
+	exit_code=$?
+	
+	if [[ $exit_code -ne 0 ]]; then
+		echo "Command failed with error $exit_code"
+		echo "$err_msg"
+		exit $exit_code
+	fi
+}
+
 set_os() {
 	if [ -f /etc/redhat-release ]; then
 		OS=centos
diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common
index be5a327f0..c855abd15 100755
--- a/salt/common/tools/sbin/so-image-common
+++ b/salt/common/tools/sbin/so-image-common
@@ -18,6 +18,7 @@
 # NOTE: This script depends on so-common
 IMAGEREPO=security-onion-solutions
 
+# shellcheck disable=SC2120
 container_list() {
   MANAGERCHECK=$1
   
@@ -128,7 +129,9 @@ update_docker_containers() {
   mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1 
 
   # Let's make sure we have the public key
-  retry 50 10 "curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" >> "$LOG_FILE" 2>&1
+  run_check_net_err \
+  "curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \
+  "Could not pull signature key file, please ensure connectivity to https://raw.gihubusercontent.com" >> "$LOG_FILE" 2>&1
   result=$?
   if [[ $result -eq 0 ]]; then
     cat $SIGNPATH/KEYS | gpg --import - >> "$LOG_FILE" 2>&1
@@ -148,10 +151,14 @@ update_docker_containers() {
 
     # Pull down the trusted docker image
     local image=$i:$VERSION$IMAGE_TAG_SUFFIX
-    retry 50 10 "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" >> "$LOG_FILE" 2>&1 
+    run_check_net_err \
+    "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" \
+    "Could not pull pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 
     
     # Get signature
-    retry 50 10 "curl -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" >> "$LOG_FILE" 2>&1
+    run_check_net_err \
+    "curl -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \
+    "Could not pull signature file, please ensure connectivity to https://sigs.securityonion.net " >> "$LOG_FILE" 2>&1
     if [[ $? -ne 0 ]]; then
       echo "Unable to pull signature file for $image" >> "$LOG_FILE" 2>&1 
       exit 1
diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index 8e7f70517..f43a31740 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -20,11 +20,83 @@
 UPDATE_DIR=/tmp/sogh/securityonion
 INSTALLEDVERSION=$(cat /etc/soversion)
 POSTVERSION=$INSTALLEDVERSION
-INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'})
+INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
 BATCHSIZE=5
 SOUP_LOG=/root/soup.log
 WHATWOULDYOUSAYYAHDOHERE=soup
 
+set -e
+
+trap 'check_err $? $BASH_LINENO $BASH_COMMAND' EXIT
+
+check_err() {
+  local exit_code=$1
+  local lineno=$2
+  local cmd=$3
+  local err_msg="Unknown error occured, please check $SOUP_LOG for details."
+
+  if [[ $exit_code -ne 0 ]]; then
+    printf '%s' "Soup failed on line $lineno with error $exit_code: "
+    case $exit_code in
+      2)
+        echo 'No such file or directory'
+      ;;
+      5)
+        echo 'Interrupted system call'
+      ;;
+      12)
+        echo 'Out of memory'
+      ;;
+      28)
+        echo 'No space left on device'
+        echo 'Likely ran out of space on disk, please review hardware requirements for Security Onion: https://docs.securityonion.net/en/2.3/hardware.html'
+      ;;
+      30)
+        echo 'Read-only file system'
+      ;;
+      35)
+        echo 'Resource temporarily unavailable'
+      ;;
+      64)
+        echo 'Machine is not on the network'
+      ;;
+      67)
+        echo 'Link has been severed'
+      ;;
+      100)
+        echo 'Netowrk is down'
+      ;;
+      101)
+        echo 'Network is unreachable'
+      ;;
+      102)
+        echo 'Network reset'
+      ;;
+      110)
+        echo 'Connection timed out'
+      ;;
+      111)
+        echo 'Connection refused'
+      ;;
+      112)
+        echo 'Host is down'
+      ;;
+      113)
+        echo 'No route to host'
+      ;;
+      *)
+        echo ''
+        echo "$err_msg"
+      ;;
+    esac
+    if [[ $exit_code -ge 64 && $exit_code -le 113 ]]; then
+      echo "$err_msg"
+    fi
+    exit $exit_code
+  fi
+
+}
+
 add_common() {
   cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
   cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
@@ -87,9 +159,9 @@ airgap_update_dockers() {
       docker stop so-dockerregistry
       docker rm so-dockerregistry
       echo "Copying the new dockers over"
-      tar xvf $AGDOCKER/registry.tar -C /nsm/docker-registry/docker
+      tar xvf "$AGDOCKER/registry.tar" -C /nsm/docker-registry/docker
       echo "Add Registry back"
-      docker load -i $AGDOCKER/registry_image.tar
+      docker load -i "$AGDOCKER/registry_image.tar"
     fi
   fi
 }
@@ -101,9 +173,9 @@ update_registry() {
 }
 
 check_sudoers() {
-	if grep -q "so-setup" /etc/sudoers; then
-		echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"."
-	fi
+  if grep -q "so-setup" /etc/sudoers; then
+    echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"."
+  fi
 }
 
 check_log_size_limit() {
@@ -177,7 +249,7 @@ check_os_updates() {
           echo "Continuing without updating packages"
       elif [[ "$confirm" == [uU] ]]; then
           echo "Applying Grid Updates"
-          salt \* -b 5 state.apply patch.os queue=True
+          run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.'
       else
           echo "Exiting soup"
           exit 0
@@ -205,7 +277,7 @@ clone_to_tmp() {
   if [ -n "$BRANCH" ]; then
     SOUP_BRANCH="-b $BRANCH"
   fi
-  git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git
+  run_check_net_err "git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git" "Could not clone repo, please ensure network access to https://github.com"
   cd /tmp
   if [ ! -f $UPDATE_DIR/VERSION ]; then
     echo "Update was unable to pull from github. Please check your internet."
@@ -586,13 +658,14 @@ upgrade_check() {
 }
 
 upgrade_check_salt() {
-  NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk {'print $2'})
+  NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk '{print $2}')
   if [ "$INSTALLEDSALTVERSION" == "$NEWSALTVERSION" ]; then
     echo "You are already running the correct version of Salt for Security Onion."
   else
     UPGRADESALT=1
   fi
 }   
+
 upgrade_salt() {
   SALTUPGRADED=True
   echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION."
@@ -604,7 +677,9 @@ upgrade_salt() {
     yum versionlock delete "salt-*"
     echo "Updating Salt packages and restarting services."
     echo ""
-    sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION"
+    run_check_net_err \
+    "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \
+    "Could not update soup, please check $SOUP_LOG for details."
     echo "Applying yum versionlock for Salt."
     echo ""
     yum versionlock add "salt-*"
@@ -617,7 +692,9 @@ upgrade_salt() {
     apt-mark unhold "salt-minion"
     echo "Updating Salt packages and restarting services."
     echo ""
-    sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION"
+    run_check_net_err \
+    "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \
+    "Could not update soup, please check $SOUP_LOG for details."
     echo "Applying apt hold for Salt."
     echo ""
     apt-mark hold "salt-common"
@@ -650,7 +727,7 @@ verify_latest_update_script() {
 }
 
 main () {
-echo "### Preparing soup at `date` ###"
+echo "### Preparing soup at $(date) ###"
 while getopts ":b" opt; do
   case "$opt" in
     b ) # process option b
@@ -756,7 +833,7 @@ else
   echo "Checking if Salt was upgraded."
   echo ""
   # Check that Salt was upgraded
-  SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk {'print $2'})
+  SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk '{print $2}')
   if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then
     echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG."
     echo "Once the issue is resolved, run soup again."
@@ -874,7 +951,7 @@ EOF
   fi
 fi
 
-echo "### soup has been served at `date` ###"
+echo "### soup has been served at $(date) ###"
 }
 
 cat << EOF
@@ -889,6 +966,6 @@ Press Enter to continue or Ctrl-C to cancel.
 
 EOF
 
-read input
+read -r input
 
 main "$@" | tee -a $SOUP_LOG

From f8dd6890b27f6109d8f31a7dbbf3db709d312ea9 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Fri, 7 May 2021 16:50:59 -0400
Subject: [PATCH 06/48] Unset/set exit on command fail for retries

---
 salt/common/tools/sbin/soup | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index f43a31740..b02a63fc6 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -249,7 +249,9 @@ check_os_updates() {
           echo "Continuing without updating packages"
       elif [[ "$confirm" == [uU] ]]; then
           echo "Applying Grid Updates"
+          set +e
           run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.'
+          set -e
       else
           echo "Exiting soup"
           exit 0
@@ -277,7 +279,9 @@ clone_to_tmp() {
   if [ -n "$BRANCH" ]; then
     SOUP_BRANCH="-b $BRANCH"
   fi
+  set +e
   run_check_net_err "git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git" "Could not clone repo, please ensure network access to https://github.com"
+  set -e
   cd /tmp
   if [ ! -f $UPDATE_DIR/VERSION ]; then
     echo "Update was unable to pull from github. Please check your internet."
@@ -677,9 +681,11 @@ upgrade_salt() {
     yum versionlock delete "salt-*"
     echo "Updating Salt packages and restarting services."
     echo ""
+    set +e
     run_check_net_err \
     "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \
     "Could not update soup, please check $SOUP_LOG for details."
+    set -e
     echo "Applying yum versionlock for Salt."
     echo ""
     yum versionlock add "salt-*"
@@ -692,9 +698,11 @@ upgrade_salt() {
     apt-mark unhold "salt-minion"
     echo "Updating Salt packages and restarting services."
     echo ""
+    set +e
     run_check_net_err \
     "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \
     "Could not update soup, please check $SOUP_LOG for details."
+    set -e
     echo "Applying apt hold for Salt."
     echo ""
     apt-mark hold "salt-common"
@@ -808,7 +816,9 @@ else
     check_os_updates
   else
     update_registry
+    set +e
     update_docker_containers "soup"
+    set -e
   fi
 
   echo ""

From 8b3730748b7a26d58489a9b803337b54e3f5827c Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Mon, 10 May 2021 10:58:29 -0400
Subject: [PATCH 07/48] Add debug line and remove exit command on retry failure

---
 salt/common/tools/sbin/soup | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index 398aed29c..fdc983d30 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -662,6 +662,7 @@ upgrade_check() {
       is_hotfix=true
     fi
   else
+    echo "DEBUG: here"
     is_hotfix=false
   fi 
 
@@ -889,7 +890,7 @@ else
   systemctl start salt-master
 
   # Testing that that salt-master is up by checking that is it connected to itself
-  retry 50 10 "salt-call state.show_top -l error" || exit 1
+  retry 50 10 "salt-call state.show_top -l error"
 
   echo ""
   echo "Ensuring python modules for Salt are installed and patched."
@@ -923,7 +924,7 @@ else
   systemctl start salt-master
 
   # Testing that that salt-master is up by checking that is it connected to itself
-  retry 50 10 "salt-call state.show_top -l error" || exit 1
+  retry 50 10 "salt-call state.show_top -l error"
 
   echo "Running a highstate. This could take several minutes."
   salt-call state.highstate -l info queue=True

From a75d4841d0a29eb06ec492fbf0518f3c6bb86b45 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Mon, 10 May 2021 11:05:24 -0400
Subject: [PATCH 08/48] Add debug lines

---
 salt/common/tools/sbin/soup | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index fdc983d30..d1d3f5e6e 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -734,7 +734,7 @@ verify_latest_update_script() {
     cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/
     cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
     cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
-    salt-call state.apply common queue=True
+    salt-call state.apply -l info common queue=True
     echo ""
     echo "soup has been updated. Please run soup again."
     exit 0
@@ -796,7 +796,9 @@ add_common
 fi
 
 echo "Let's see if we need to update Security Onion."
+echo "DEBUG: before upgrade_check"
 upgrade_check
+echo "DEBUG: after upgrade_check"
 upgrade_space
 
 echo "Checking for Salt Master and Minion updates."

From d3b08beb534a2b3cd336d7d2dcdb9dc8ab8dc7b0 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Mon, 10 May 2021 11:11:54 -0400
Subject: [PATCH 09/48] Only cat file if it exists

---
 salt/common/tools/sbin/soup | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index d1d3f5e6e..6c4af361b 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -651,7 +651,7 @@ upgrade_check() {
   # Let's make sure we actually need to update.
   NEWVERSION=$(cat $UPDATE_DIR/VERSION)
   HOTFIXVERSION=$(cat $UPDATE_DIR/HOTFIX)
-  CURRENTHOTFIX=$(cat /etc/sohotfix 2>/dev/null)
+  [[ -f /etc/sohotfix ]] && CURRENTHOTFIX=$(cat /etc/sohotfix)
   if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then
     echo "Checking to see if there are hotfixes needed"
     if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then

From fec868432f3d240a216b42647cc633a6cd9c168c Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Mon, 10 May 2021 11:59:22 -0400
Subject: [PATCH 10/48] Try to fix bash trap

---
 salt/common/tools/sbin/soup | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index 6c4af361b..364a769c0 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -28,13 +28,12 @@ WHATWOULDYOUSAYYAHDOHERE=soup
 
 set -e
 
-trap 'check_err $? $BASH_LINENO $BASH_COMMAND' EXIT
+trap 'check_err $? $BASH_LINENO' EXIT
 
 check_err() {
   local exit_code=$1
   local lineno=$2
-  local cmd=$3
-  local err_msg="Unknown error occured, please check $SOUP_LOG for details."
+  local err_msg="Unhandled error occured, please check $SOUP_LOG for details."
 
   if [[ $exit_code -ne 0 ]]; then
     printf '%s' "Soup failed on line $lineno with error $exit_code: "
@@ -86,7 +85,7 @@ check_err() {
         echo 'No route to host'
       ;;
       *)
-        echo ''
+        echo 'Unhandled error'
         echo "$err_msg"
       ;;
     esac

From f793450d972cb69743fe7cac96644e934ac275b6 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Mon, 10 May 2021 13:22:13 -0400
Subject: [PATCH 11/48] Return actual exit code from retry

---
 salt/common/tools/sbin/so-common | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index f78919226..2b2bbf944 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -302,6 +302,7 @@ retry() {
 	cmd=$3
 	expectedOutput=$4
 	attempt=0
+	local exitcode=0
 	while [[ $attempt -lt $maxAttempts ]]; do			
 		attempt=$((attempt+1))
 		echo "Executing command with retry support: $cmd"
@@ -321,7 +322,7 @@ retry() {
 		sleep $sleepDelay
 	done
 	echo "Command continues to fail; giving up."
-	return 1
+	return $exitcode
 }
 
 run_check_net_err() {

From c379822bf0df0fb52083392498feebc8e22ddc1e Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Tue, 11 May 2021 12:59:49 -0400
Subject: [PATCH 12/48] Set variable to skip trap if error already handled

---
 salt/common/tools/sbin/so-common | 1 +
 salt/common/tools/sbin/soup      | 1 +
 2 files changed, 2 insertions(+)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index 2b2bbf944..2b22ebbeb 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -334,6 +334,7 @@ run_check_net_err() {
 	exit_code=$?
 	
 	if [[ $exit_code -ne 0 ]]; then
+		ERR_HANDLED=true
 		echo "Command failed with error $exit_code"
 		echo "$err_msg"
 		exit $exit_code
diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index 364a769c0..c8601fe1a 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -35,6 +35,7 @@ check_err() {
   local lineno=$2
   local err_msg="Unhandled error occured, please check $SOUP_LOG for details."
 
+  [[ $ERR_HANDLED == true ]] && exit $exit_code
   if [[ $exit_code -ne 0 ]]; then
     printf '%s' "Soup failed on line $lineno with error $exit_code: "
     case $exit_code in

From 7bd04deae78f0199afb706afefb8ca4b20588b5e Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Wed, 12 May 2021 10:45:03 -0400
Subject: [PATCH 13/48] Unset exit on failure for pkill command

---
 salt/common/tools/sbin/soup | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index c8601fe1a..7c97a540a 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -834,7 +834,9 @@ else
   echo "Stopping Salt Minion service."
   systemctl stop salt-minion
   echo "Killing any remaining Salt Minion processes."
+  set +e
   pkill -9 -ef /usr/bin/salt-minion
+  set -e
   echo ""
   echo "Stopping Salt Master service."
   systemctl stop salt-master

From 807b525c797bd280036bf06d0a63a17693894f44 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Wed, 12 May 2021 11:19:33 -0400
Subject: [PATCH 14/48] Temp remove exit on failure + bash trap

---
 salt/common/tools/sbin/soup | 83 -------------------------------------
 1 file changed, 83 deletions(-)

diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index 7c97a540a..53dd7f176 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -26,77 +26,6 @@ SOUP_LOG=/root/soup.log
 INFLUXDB_MIGRATION_LOG=/opt/so/log/influxdb/soup_migration.log
 WHATWOULDYOUSAYYAHDOHERE=soup
 
-set -e
-
-trap 'check_err $? $BASH_LINENO' EXIT
-
-check_err() {
-  local exit_code=$1
-  local lineno=$2
-  local err_msg="Unhandled error occured, please check $SOUP_LOG for details."
-
-  [[ $ERR_HANDLED == true ]] && exit $exit_code
-  if [[ $exit_code -ne 0 ]]; then
-    printf '%s' "Soup failed on line $lineno with error $exit_code: "
-    case $exit_code in
-      2)
-        echo 'No such file or directory'
-      ;;
-      5)
-        echo 'Interrupted system call'
-      ;;
-      12)
-        echo 'Out of memory'
-      ;;
-      28)
-        echo 'No space left on device'
-        echo 'Likely ran out of space on disk, please review hardware requirements for Security Onion: https://docs.securityonion.net/en/2.3/hardware.html'
-      ;;
-      30)
-        echo 'Read-only file system'
-      ;;
-      35)
-        echo 'Resource temporarily unavailable'
-      ;;
-      64)
-        echo 'Machine is not on the network'
-      ;;
-      67)
-        echo 'Link has been severed'
-      ;;
-      100)
-        echo 'Netowrk is down'
-      ;;
-      101)
-        echo 'Network is unreachable'
-      ;;
-      102)
-        echo 'Network reset'
-      ;;
-      110)
-        echo 'Connection timed out'
-      ;;
-      111)
-        echo 'Connection refused'
-      ;;
-      112)
-        echo 'Host is down'
-      ;;
-      113)
-        echo 'No route to host'
-      ;;
-      *)
-        echo 'Unhandled error'
-        echo "$err_msg"
-      ;;
-    esac
-    if [[ $exit_code -ge 64 && $exit_code -le 113 ]]; then
-      echo "$err_msg"
-    fi
-    exit $exit_code
-  fi
-
-}
 
 add_common() {
   cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
@@ -250,9 +179,7 @@ check_os_updates() {
           echo "Continuing without updating packages"
       elif [[ "$confirm" == [uU] ]]; then
           echo "Applying Grid Updates"
-          set +e
           run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.'
-          set -e
       else
           echo "Exiting soup"
           exit 0
@@ -280,9 +207,7 @@ clone_to_tmp() {
   if [ -n "$BRANCH" ]; then
     SOUP_BRANCH="-b $BRANCH"
   fi
-  set +e
   run_check_net_err "git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git" "Could not clone repo, please ensure network access to https://github.com"
-  set -e
   cd /tmp
   if [ ! -f $UPDATE_DIR/VERSION ]; then
     echo "Update was unable to pull from github. Please check your internet."
@@ -688,11 +613,9 @@ upgrade_salt() {
     yum versionlock delete "salt-*"
     echo "Updating Salt packages and restarting services."
     echo ""
-    set +e
     run_check_net_err \
     "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \
     "Could not update soup, please check $SOUP_LOG for details."
-    set -e
     echo "Applying yum versionlock for Salt."
     echo ""
     yum versionlock add "salt-*"
@@ -705,11 +628,9 @@ upgrade_salt() {
     apt-mark unhold "salt-minion"
     echo "Updating Salt packages and restarting services."
     echo ""
-    set +e
     run_check_net_err \
     "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \
     "Could not update soup, please check $SOUP_LOG for details."
-    set -e
     echo "Applying apt hold for Salt."
     echo ""
     apt-mark hold "salt-common"
@@ -825,18 +746,14 @@ else
     check_os_updates
   else
     update_registry
-    set +e
     update_docker_containers "soup"
-    set -e
   fi
 
   echo ""
   echo "Stopping Salt Minion service."
   systemctl stop salt-minion
   echo "Killing any remaining Salt Minion processes."
-  set +e
   pkill -9 -ef /usr/bin/salt-minion
-  set -e
   echo ""
   echo "Stopping Salt Master service."
   systemctl stop salt-master

From 9ced391c11f255a7fe22b4e740b436398de0983d Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Wed, 12 May 2021 13:20:59 -0400
Subject: [PATCH 15/48] Fix indent in main(), re-add trap, remove ERR_HANDLED
 variable

---
 salt/common/tools/sbin/so-common |   1 -
 salt/common/tools/sbin/soup      | 534 ++++++++++++++++++-------------
 2 files changed, 309 insertions(+), 226 deletions(-)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index 2b22ebbeb..2b2bbf944 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -334,7 +334,6 @@ run_check_net_err() {
 	exit_code=$?
 	
 	if [[ $exit_code -ne 0 ]]; then
-		ERR_HANDLED=true
 		echo "Command failed with error $exit_code"
 		echo "$err_msg"
 		exit $exit_code
diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index 53dd7f176..814f9fd1b 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -26,6 +26,72 @@ SOUP_LOG=/root/soup.log
 INFLUXDB_MIGRATION_LOG=/opt/so/log/influxdb/soup_migration.log
 WHATWOULDYOUSAYYAHDOHERE=soup
 
+check_err() {
+  local exit_code=$1
+  local lineno=$2
+  local err_msg="Unhandled error occured, please check $SOUP_LOG for details."
+
+  if [[ $exit_code -ne 0 ]]; then
+    printf '%s' "Soup failed on line $lineno with error $exit_code: "
+    case $exit_code in
+      2)
+        echo 'No such file or directory'
+      ;;
+      5)
+        echo 'Interrupted system call'
+      ;;
+      12)
+        echo 'Out of memory'
+      ;;
+      28)
+        echo 'No space left on device'
+        echo 'Likely ran out of space on disk, please review hardware requirements for Security Onion: https://docs.securityonion.net/en/2.3/hardware.html'
+      ;;
+      30)
+        echo 'Read-only file system'
+      ;;
+      35)
+        echo 'Resource temporarily unavailable'
+      ;;
+      64)
+        echo 'Machine is not on the network'
+      ;;
+      67)
+        echo 'Link has been severed'
+      ;;
+      100)
+        echo 'Netowrk is down'
+      ;;
+      101)
+        echo 'Network is unreachable'
+      ;;
+      102)
+        echo 'Network reset'
+      ;;
+      110)
+        echo 'Connection timed out'
+      ;;
+      111)
+        echo 'Connection refused'
+      ;;
+      112)
+        echo 'Host is down'
+      ;;
+      113)
+        echo 'No route to host'
+      ;;
+      *)
+        echo 'Unhandled error'
+        echo "$err_msg"
+      ;;
+    esac
+    if [[ $exit_code -ge 64 && $exit_code -le 113 ]]; then
+      echo "$err_msg"
+    fi
+    exit $exit_code
+  fi
+
+}
 
 add_common() {
   cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
@@ -48,8 +114,8 @@ airgap_mounted() {
     echo "Example: /home/user/securityonion-2.X.0.iso"
     echo "Example: /dev/sdx1"
     echo ""
-    read -p 'Enter the location of the iso: ' ISOLOC
-    if [ -f $ISOLOC ]; then
+    read -rp 'Enter the location of the iso: ' ISOLOC
+    if [[ -f $ISOLOC ]]; then
       # Mounting the ISO image
       mkdir -p /tmp/soagupdate
       mount -t iso9660 -o loop $ISOLOC /tmp/soagupdate
@@ -61,7 +127,7 @@ airgap_mounted() {
       else
         echo "ISO has been mounted!"
       fi  
-    elif [ -f $ISOLOC/SecurityOnion/VERSION ]; then
+    elif [[ -f $ISOLOC/SecurityOnion/VERSION ]]; then
       ln -s $ISOLOC /tmp/soagupdate
       echo "Found the update content"
     else 
@@ -79,9 +145,9 @@ airgap_mounted() {
 }
 
 airgap_update_dockers() {
-  if [ $is_airgap -eq 0 ]; then
+  if [[ $is_airgap -eq 0 ]]; then
     # Let's copy the tarball
-    if [ ! -f $AGDOCKER/registry.tar ]; then
+    if [[ ! -f $AGDOCKER/registry.tar ]]; then
       echo "Unable to locate registry. Exiting"
       exit 1
     else
@@ -179,7 +245,9 @@ check_os_updates() {
           echo "Continuing without updating packages"
       elif [[ "$confirm" == [uU] ]]; then
           echo "Applying Grid Updates"
+          set +e
           run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.'
+          set -e
       else
           echo "Exiting soup"
           exit 0
@@ -207,7 +275,9 @@ clone_to_tmp() {
   if [ -n "$BRANCH" ]; then
     SOUP_BRANCH="-b $BRANCH"
   fi
+  set +e
   run_check_net_err "git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git" "Could not clone repo, please ensure network access to https://github.com"
+  set -e
   cd /tmp
   if [ ! -f $UPDATE_DIR/VERSION ]; then
     echo "Update was unable to pull from github. Please check your internet."
@@ -253,7 +323,11 @@ preupgrade_changes_2.3.50_repo() {
     # We made repo changes in 2.3.50 and this prepares for that on upgrade
     echo "Checking to see if 2.3.50 repo changes are needed."
 
-    [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50_repo
+    if [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]]; then
+      up_2.3.3X_to_2.3.50_repo
+    else
+      echo "No changes needed."
+    fi
 }
 
 preupgrade_changes() {
@@ -433,7 +507,7 @@ up_2.3.2X_to_2.3.30() {
   sed -i "/  imagerepo: securityonion/c\  imagerepo: 'security-onion-solutions'" /opt/so/saltstack/local/pillar/global.sls 
 
   # Strelka rule repo pillar addition
-  if [ $is_airgap -eq 0 ]; then
+  if [[ $is_airgap -eq 0 ]]; then
       # Add manager as default Strelka YARA rule repo
       sed -i "/^strelka:/a \\  repos: \n    - https://$HOSTNAME/repo/rules/strelka" /opt/so/saltstack/local/pillar/global.sls;
   else
@@ -460,7 +534,7 @@ up_2.3.3X_to_2.3.50_repo() {
         rm -f "/etc/yum.repos.d/$DELREPO.repo"
       fi
     done
-    if [ $is_airgap -eq 1 ]; then
+    if [[ $is_airgap -eq 1 ]]; then
       # Copy the new repo file if not airgap
       cp $UPDATE_DIR/salt/repo/client/files/centos/securityonion.repo /etc/yum.repos.d/
       yum clean all
@@ -613,9 +687,11 @@ upgrade_salt() {
     yum versionlock delete "salt-*"
     echo "Updating Salt packages and restarting services."
     echo ""
+    set +e
     run_check_net_err \
     "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \
     "Could not update soup, please check $SOUP_LOG for details."
+    set -e
     echo "Applying yum versionlock for Salt."
     echo ""
     yum versionlock add "salt-*"
@@ -628,9 +704,11 @@ upgrade_salt() {
     apt-mark unhold "salt-minion"
     echo "Updating Salt packages and restarting services."
     echo ""
+    set +e
     run_check_net_err \
     "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \
     "Could not update soup, please check $SOUP_LOG for details."
+    set -e
     echo "Applying apt hold for Salt."
     echo ""
     apt-mark hold "salt-common"
@@ -662,229 +740,235 @@ verify_latest_update_script() {
   fi
 }
 
-main () {
-echo "### Preparing soup at $(date) ###"
-while getopts ":b" opt; do
-  case "$opt" in
-    b ) # process option b
-       shift
-       BATCHSIZE=$1
-       if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then
-         echo "Batch size must be a number greater than 0."
-         exit 1
-       fi
-    ;;
-    \? ) 
-      echo "Usage: cmd [-b]"
-    ;;
-  esac
-done
-
-echo "Checking to see if this is a manager."
-echo ""
-require_manager
-set_minionid
-echo "Checking to see if this is an airgap install"
-echo ""
-check_airgap
-echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
-echo ""
-set_os
-set_palette
-check_elastic_license
-echo ""
-if [ $is_airgap -eq 0 ]; then
-  # Let's mount the ISO since this is airgap
-  airgap_mounted
-else
-  echo "Cloning Security Onion github repo into $UPDATE_DIR."
-  echo "Removing previous upgrade sources."
-  rm -rf $UPDATE_DIR
-  clone_to_tmp
-fi
-check_os_updates
-echo ""
-echo "Verifying we have the latest soup script."
-verify_latest_update_script
-echo ""
-
-echo "Generating new repo archive"
-generate_and_clean_tarballs
-if [ -f /usr/sbin/so-image-common ]; then
-  . /usr/sbin/so-image-common
-else 
-add_common
-fi
-
-echo "Let's see if we need to update Security Onion."
-echo "DEBUG: before upgrade_check"
-upgrade_check
-echo "DEBUG: after upgrade_check"
-upgrade_space
-
-echo "Checking for Salt Master and Minion updates."
-upgrade_check_salt
-
-
-if [ "$is_hotfix" == "true" ]; then
-  echo "Applying $HOTFIXVERSION" 
-  copy_new_files
-  echo ""
-  update_version
-  salt-call state.highstate -l info queue=True
+main() {
+  set -e
+  trap 'check_err $? $BASH_LINENO' EXIT
   
-else
-  echo ""
-  echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
-  echo ""
-
-  echo "Updating dockers to $NEWVERSION."
-  if [ $is_airgap -eq 0 ]; then
-    airgap_update_dockers
-    update_centos_repo
-    yum clean all
-    check_os_updates
-  else
-    update_registry
-    update_docker_containers "soup"
-  fi
-
-  echo ""
-  echo "Stopping Salt Minion service."
-  systemctl stop salt-minion
-  echo "Killing any remaining Salt Minion processes."
-  pkill -9 -ef /usr/bin/salt-minion
-  echo ""
-  echo "Stopping Salt Master service."
-  systemctl stop salt-master
-  echo ""
-
-  preupgrade_changes_2.3.50_repo
-
-  # Does salt need upgraded. If so update it.
-  if [ "$UPGRADESALT" == "1" ]; then
-    echo "Upgrading Salt"
-    # Update the repo files so it can actually upgrade
-    upgrade_salt
-  fi
-
-  echo "Checking if Salt was upgraded."
-  echo ""
-  # Check that Salt was upgraded
-  SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk '{print $2}')
-  if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then
-    echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG."
-    echo "Once the issue is resolved, run soup again."
-    echo "Exiting."
-    echo ""
-    exit 1
-  else
-    echo "Salt upgrade success."
-    echo ""
-  fi
-
-  preupgrade_changes
-  echo ""
-
-  if [ $is_airgap -eq 0 ]; then
-    echo "Updating Rule Files to the Latest."
-    update_airgap_rules
-  fi
-
-  # Only update the repo if its airgap
-  if [[ $is_airgap -eq 0 ]] && [[ "$UPGRADESALT" != "1" ]]; then
-  update_centos_repo
-  fi
-
-  echo ""
-  echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR."
-  copy_new_files
-  echo ""
-  update_version
-
-  echo ""
-  echo "Locking down Salt Master for upgrade"
-  masterlock
-
-  echo ""
-  echo "Starting Salt Master service."
-  systemctl start salt-master
-
-  # Testing that that salt-master is up by checking that is it connected to itself
-  retry 50 10 "salt-call state.show_top -l error"
-
-  echo ""
-  echo "Ensuring python modules for Salt are installed and patched."
-  salt-call state.apply salt.python3-influxdb -l info queue=True
-  echo ""
-
-  # Only regenerate osquery packages if Fleet is enabled
-  FLEET_MANAGER=$(lookup_pillar fleet_manager)
-  FLEET_NODE=$(lookup_pillar fleet_node)
-  if [[ "$FLEET_MANAGER" == "True" || "$FLEET_NODE" == "True" ]]; then
-    echo ""
-    echo "Regenerating Osquery Packages.... This will take several minutes."
-    salt-call state.apply fleet.event_gen-packages -l info queue=True
-    echo ""
-  fi
-
-  echo ""
-  echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
-  salt-call state.highstate -l info queue=True
-  echo ""
-  echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete."
-
-  echo ""
-  echo "Stopping Salt Master to remove ACL"
-  systemctl stop salt-master
-
-  masterunlock
-
-  echo ""
-  echo "Starting Salt Master service."
-  systemctl start salt-master
-
-  # Testing that that salt-master is up by checking that is it connected to itself
-  retry 50 10 "salt-call state.show_top -l error"
-
-  echo "Running a highstate. This could take several minutes."
-  salt-call state.highstate -l info queue=True
-  postupgrade_changes
-  unmount_update
-  thehive_maint
-
-  if [ "$UPGRADESALT" == "1" ]; then
-    if [ $is_airgap -eq 0 ]; then
-      echo ""
-      echo "Cleaning repos on remote Security Onion nodes."
-      salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone and G@os:CentOS' cmd.run "yum clean all"
-      echo ""
-    fi
-  fi
-
-  check_sudoers
-
-  if [[ -n $lsl_msg ]]; then
-    case $lsl_msg in
-      'distributed')
-        echo "[INFO] The value of log_size_limit in any heavy node minion pillars may be incorrect."
-        echo " -> We recommend checking and adjusting the values as necessary."
-        echo " -> Minion pillar directory: /opt/so/saltstack/local/pillar/minions/"
+  echo "### Preparing soup at $(date) ###"
+  while getopts ":b" opt; do
+    case "$opt" in
+      b ) # process option b
+        shift
+        BATCHSIZE=$1
+        if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then
+          echo "Batch size must be a number greater than 0."
+          exit 1
+        fi
       ;;
-      'single-node')
-        # We can assume the lsl_details array has been set if lsl_msg has this value
-        echo "[WARNING] The value of log_size_limit (${lsl_details[0]}) does not match the recommended value of ${lsl_details[1]}."
-        echo " -> We recommend checking and adjusting the value as necessary."
-        echo " -> File: /opt/so/saltstack/local/pillar/minions/${lsl_details[2]}.sls"
+      \? ) 
+        echo "Usage: cmd [-b]"
       ;;
     esac
+  done
+
+  echo "Checking to see if this is a manager."
+  echo ""
+  require_manager
+  set_minionid
+  echo "Checking to see if this is an airgap install"
+  echo ""
+  check_airgap
+  echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
+  echo ""
+  set_os
+  set_palette
+  check_elastic_license
+  echo ""
+  if [[ $is_airgap -eq 0 ]]; then
+    # Let's mount the ISO since this is airgap
+    airgap_mounted
+  else
+    echo "Cloning Security Onion github repo into $UPDATE_DIR."
+    echo "Removing previous upgrade sources."
+    rm -rf $UPDATE_DIR
+    clone_to_tmp
+  fi
+  check_os_updates
+  echo ""
+  echo "Verifying we have the latest soup script."
+  verify_latest_update_script
+  echo ""
+
+  echo "Generating new repo archive"
+  generate_and_clean_tarballs
+  if [ -f /usr/sbin/so-image-common ]; then
+    . /usr/sbin/so-image-common
+  else 
+  add_common
   fi
 
-  NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l)
+  echo "Let's see if we need to update Security Onion."
+  echo "DEBUG: before upgrade_check"
+  upgrade_check
+  echo "DEBUG: after upgrade_check"
+  upgrade_space
 
-  if [ $NUM_MINIONS -gt 1 ]; then
+  echo "Checking for Salt Master and Minion updates."
+  upgrade_check_salt
 
-    cat << EOF
+
+  if [ "$is_hotfix" == "true" ]; then
+    echo "Applying $HOTFIXVERSION" 
+    copy_new_files
+    echo ""
+    update_version
+    salt-call state.highstate -l info queue=True
+  else
+    echo ""
+    echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
+    echo ""
+
+    echo "Updating dockers to $NEWVERSION."
+    if [[ $is_airgap -eq 0 ]]; then
+      airgap_update_dockers
+      update_centos_repo
+      yum clean all
+      check_os_updates
+    else
+      update_registry
+      set +e
+      update_docker_containers "soup"
+      set -e
+    fi
+
+    echo ""
+    echo "Stopping Salt Minion service."
+    systemctl stop salt-minion
+    echo "Killing any remaining Salt Minion processes."
+    set +e
+    pkill -9 -ef /usr/bin/salt-minion
+    set -e
+    echo ""
+    echo "Stopping Salt Master service."
+    systemctl stop salt-master
+    echo ""
+
+    preupgrade_changes_2.3.50_repo
+
+    # Does salt need upgraded. If so update it.
+    if [[ $UPGRADESALT -eq 1 ]]; then
+      echo "Upgrading Salt"
+      # Update the repo files so it can actually upgrade
+      upgrade_salt
+    fi
+
+    echo "Checking if Salt was upgraded."
+    echo ""
+    # Check that Salt was upgraded
+    SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk '{print $2}')
+    if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then
+      echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG."
+      echo "Once the issue is resolved, run soup again."
+      echo "Exiting."
+      echo ""
+      exit 1
+    else
+      echo "Salt upgrade success."
+      echo ""
+    fi
+
+    preupgrade_changes
+    echo ""
+
+    if [[ $is_airgap -eq 0 ]]; then
+      echo "Updating Rule Files to the Latest."
+      update_airgap_rules
+    fi
+
+    # Only update the repo if its airgap
+    if [[ $is_airgap -eq 0 ]] && [[ $UPGRADESALT -ne 1 ]]; then
+      update_centos_repo
+    fi
+
+    echo ""
+    echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR."
+    copy_new_files
+    echo ""
+    update_version
+
+    echo ""
+    echo "Locking down Salt Master for upgrade"
+    masterlock
+
+    echo ""
+    echo "Starting Salt Master service."
+    systemctl start salt-master
+
+    # Testing that that salt-master is up by checking that is it connected to itself
+    retry 50 10 "salt-call state.show_top -l error"
+
+    echo ""
+    echo "Ensuring python modules for Salt are installed and patched."
+    salt-call state.apply salt.python3-influxdb -l info queue=True
+    echo ""
+
+    # Only regenerate osquery packages if Fleet is enabled
+    FLEET_MANAGER=$(lookup_pillar fleet_manager)
+    FLEET_NODE=$(lookup_pillar fleet_node)
+    if [[ "$FLEET_MANAGER" == "True" || "$FLEET_NODE" == "True" ]]; then
+      echo ""
+      echo "Regenerating Osquery Packages.... This will take several minutes."
+      salt-call state.apply fleet.event_gen-packages -l info queue=True
+      echo ""
+    fi
+
+    echo ""
+    echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
+    salt-call state.highstate -l info queue=True
+    echo ""
+    echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete."
+
+    echo ""
+    echo "Stopping Salt Master to remove ACL"
+    systemctl stop salt-master
+
+    masterunlock
+
+    echo ""
+    echo "Starting Salt Master service."
+    systemctl start salt-master
+
+    # Testing that that salt-master is up by checking that is it connected to itself
+    retry 50 10 "salt-call state.show_top -l error"
+
+    echo "Running a highstate. This could take several minutes."
+    salt-call state.highstate -l info queue=True
+    postupgrade_changes
+    unmount_update
+    thehive_maint
+
+    if [[ $UPGRADESALT -eq 1 ]]; then
+      if [[ $is_airgap -eq 0 ]]; then
+        echo ""
+        echo "Cleaning repos on remote Security Onion nodes."
+        salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone and G@os:CentOS' cmd.run "yum clean all"
+        echo ""
+      fi
+    fi
+
+    check_sudoers
+
+    if [[ -n $lsl_msg ]]; then
+      case $lsl_msg in
+        'distributed')
+          echo "[INFO] The value of log_size_limit in any heavy node minion pillars may be incorrect."
+          echo " -> We recommend checking and adjusting the values as necessary."
+          echo " -> Minion pillar directory: /opt/so/saltstack/local/pillar/minions/"
+        ;;
+        'single-node')
+          # We can assume the lsl_details array has been set if lsl_msg has this value
+          echo "[WARNING] The value of log_size_limit (${lsl_details[0]}) does not match the recommended value of ${lsl_details[1]}."
+          echo " -> We recommend checking and adjusting the value as necessary."
+          echo " -> File: /opt/so/saltstack/local/pillar/minions/${lsl_details[2]}.sls"
+        ;;
+      esac
+    fi
+
+    NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l)
+
+    if [[ $NUM_MINIONS -gt 1 ]]; then
+
+      cat << EOF
     
   
   
@@ -898,10 +982,10 @@ For more information, please see https://docs.securityonion.net/en/2.3/soup.html
 
 EOF
 
+    fi
   fi
-fi
 
-echo "### soup has been served at $(date) ###"
+  echo "### soup has been served at $(date) ###"
 }
 
 cat << EOF

From 192b5db25a5c3d2746da34f8cf8d79c8a388ba28 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Wed, 12 May 2021 15:26:39 -0400
Subject: [PATCH 16/48] Add true to end of functions ending with shorthand
 comparison

Functions ending with test using [[ <false> ]] && <cmd> will trip set -e, so adding true to the last line of the function will prevent the function from returning a nonzero code
---
 salt/common/tools/sbin/soup | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index 814f9fd1b..e5c09a487 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -28,11 +28,10 @@ WHATWOULDYOUSAYYAHDOHERE=soup
 
 check_err() {
   local exit_code=$1
-  local lineno=$2
   local err_msg="Unhandled error occured, please check $SOUP_LOG for details."
 
   if [[ $exit_code -ne 0 ]]; then
-    printf '%s' "Soup failed on line $lineno with error $exit_code: "
+    printf '%s' "Soup failed with error $exit_code: "
     case $exit_code in
       2)
         echo 'No such file or directory'
@@ -323,11 +322,8 @@ preupgrade_changes_2.3.50_repo() {
     # We made repo changes in 2.3.50 and this prepares for that on upgrade
     echo "Checking to see if 2.3.50 repo changes are needed."
 
-    if [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]]; then
-      up_2.3.3X_to_2.3.50_repo
-    else
-      echo "No changes needed."
-    fi
+    [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50_repo
+    true
 }
 
 preupgrade_changes() {
@@ -340,6 +336,7 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20
     [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30
     [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50
+    true
 }
 
 postupgrade_changes() {
@@ -350,6 +347,7 @@ postupgrade_changes() {
     [[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30
     [[ "$POSTVERSION" == 2.3.30 ]] && post_2.3.30_to_2.3.40
     [[ "$POSTVERSION" == 2.3.50 ]] && post_2.3.5X_to_2.3.60
+    true
 }
 
 post_rc1_to_2.3.21() {
@@ -661,7 +659,6 @@ upgrade_check() {
       is_hotfix=true
     fi
   else
-    echo "DEBUG: here"
     is_hotfix=false
   fi 
 
@@ -742,7 +739,7 @@ verify_latest_update_script() {
 
 main() {
   set -e
-  trap 'check_err $? $BASH_LINENO' EXIT
+  trap 'check_err $?' EXIT
   
   echo "### Preparing soup at $(date) ###"
   while getopts ":b" opt; do
@@ -876,7 +873,7 @@ main() {
     fi
 
     # Only update the repo if its airgap
-    if [[ $is_airgap -eq 0 ]] && [[ $UPGRADESALT -ne 1 ]]; then
+    if [[ $is_airgap -eq 0 && $UPGRADESALT -ne 1 ]]; then
       update_centos_repo
     fi
 

From 3625453668953d6d6a7c18cea26e4c16909a6344 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Mon, 17 May 2021 11:00:28 -0400
Subject: [PATCH 17/48] Don't unmount airgap directory if not airgap

---
 salt/common/tools/sbin/soup | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index e5c09a487..676c5b0bc 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -931,7 +931,7 @@ main() {
     echo "Running a highstate. This could take several minutes."
     salt-call state.highstate -l info queue=True
     postupgrade_changes
-    unmount_update
+    [[ $is_airgap -eq 0 ]] && unmount_update
     thehive_maint
 
     if [[ $UPGRADESALT -eq 1 ]]; then

From 2bcd51b21c3771e3ee488a7a63b6342e8481bebe Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Mon, 17 May 2021 15:10:57 -0400
Subject: [PATCH 18/48] Fix error message

---
 salt/common/tools/sbin/soup | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index 676c5b0bc..323e755b2 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -687,7 +687,7 @@ upgrade_salt() {
     set +e
     run_check_net_err \
     "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \
-    "Could not update soup, please check $SOUP_LOG for details."
+    "Could not update salt, please check $SOUP_LOG or scrollback for details."
     set -e
     echo "Applying yum versionlock for Salt."
     echo ""
@@ -704,7 +704,7 @@ upgrade_salt() {
     set +e
     run_check_net_err \
     "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \
-    "Could not update soup, please check $SOUP_LOG for details."
+    "Could not update salt, please check $SOUP_LOG or scrollback for details."
     set -e
     echo "Applying apt hold for Salt."
     echo ""

From 992b76a0f09c40b6e7631767397e94039c5a79f0 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Mon, 17 May 2021 15:38:10 -0400
Subject: [PATCH 19/48] Remove debug lines

---
 salt/common/tools/sbin/soup | 2 --
 1 file changed, 2 deletions(-)

diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index 323e755b2..2f27939e6 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -795,9 +795,7 @@ main() {
   fi
 
   echo "Let's see if we need to update Security Onion."
-  echo "DEBUG: before upgrade_check"
   upgrade_check
-  echo "DEBUG: after upgrade_check"
   upgrade_space
 
   echo "Checking for Salt Master and Minion updates."

From 4c7cee4ebcb9bcf3395ee4bb62eb36b5c858890e Mon Sep 17 00:00:00 2001
From: Mike Reeves <luke@geekempire.com>
Date: Mon, 17 May 2021 15:55:49 -0400
Subject: [PATCH 20/48] Update VERSION

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 58e1f4fa9..a986af08b 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.3.50
+2.3.51

From d001597e52c6b03d5138b1360e7cc465ed77b78f Mon Sep 17 00:00:00 2001
From: Mike Reeves <luke@geekempire.com>
Date: Mon, 17 May 2021 15:56:46 -0400
Subject: [PATCH 21/48] Update README.md

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 353e51866..d71e83575 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
-## Security Onion 2.3.50
+## Security Onion 2.3.51
 
-Security Onion 2.3.50 is here!
+Security Onion 2.3.51 is here!
 
 ## Screenshots
 

From b89091cc7d441361e571f31d1dd723606088ebb7 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Mon, 17 May 2021 15:58:25 -0400
Subject: [PATCH 22/48] Try retrying in curl instead of shell function

---
 salt/common/tools/sbin/so-common       | 10 ++++++++--
 salt/common/tools/sbin/so-image-common | 10 ++++++----
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index 2b2bbf944..d5450f096 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -328,10 +328,16 @@ retry() {
 run_check_net_err() {
 	local cmd=$1
 	local err_msg=${2:-"Unknown error occured, please check /root/$WHATWOULDYOUSAYYAHDOHERE.log for details."} # Really need to rename that variable
+	local no_retry=$3
 
 	local exit_code
-	retry 50 10 "$cmd"
-	exit_code=$?
+	if [[ -z $no_retry ]]; then
+		retry 50 10 "$cmd"
+		exit_code=$?
+	else
+		eval "$cmd"
+		exit_code=$?
+	fi
 	
 	if [[ $exit_code -ne 0 ]]; then
 		echo "Command failed with error $exit_code"
diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common
index c855abd15..fa1d6c6dd 100755
--- a/salt/common/tools/sbin/so-image-common
+++ b/salt/common/tools/sbin/so-image-common
@@ -130,8 +130,9 @@ update_docker_containers() {
 
   # Let's make sure we have the public key
   run_check_net_err \
-  "curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \
-  "Could not pull signature key file, please ensure connectivity to https://raw.gihubusercontent.com" >> "$LOG_FILE" 2>&1
+  "curl --retry 20 --retry-delay 30 -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \
+  "Could not pull signature key file, please ensure connectivity to https://raw.gihubusercontent.com" \
+  noretry >> "$LOG_FILE" 2>&1
   result=$?
   if [[ $result -eq 0 ]]; then
     cat $SIGNPATH/KEYS | gpg --import - >> "$LOG_FILE" 2>&1
@@ -157,8 +158,9 @@ update_docker_containers() {
     
     # Get signature
     run_check_net_err \
-    "curl -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \
-    "Could not pull signature file, please ensure connectivity to https://sigs.securityonion.net " >> "$LOG_FILE" 2>&1
+    "curl --retry 20 --retry-delay 30 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \
+    "Could not pull signature file, please ensure connectivity to https://sigs.securityonion.net " \
+    noretry >> "$LOG_FILE" 2>&1
     if [[ $? -ne 0 ]]; then
       echo "Unable to pull signature file for $image" >> "$LOG_FILE" 2>&1 
       exit 1

From 3f238f7a4ad911e605c5c859d9edaf564a012aa4 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Mon, 17 May 2021 16:02:52 -0400
Subject: [PATCH 23/48] Set flag so trap doesn't repeat info

---
 salt/common/tools/sbin/so-common | 1 +
 salt/common/tools/sbin/soup      | 1 +
 2 files changed, 2 insertions(+)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index d5450f096..0390a2bf2 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -340,6 +340,7 @@ run_check_net_err() {
 	fi
 	
 	if [[ $exit_code -ne 0 ]]; then
+		ERR_HANDLED=true
 		echo "Command failed with error $exit_code"
 		echo "$err_msg"
 		exit $exit_code
diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index 2f27939e6..9a1afb2bb 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -30,6 +30,7 @@ check_err() {
   local exit_code=$1
   local err_msg="Unhandled error occured, please check $SOUP_LOG for details."
 
+  [[ $ERR_HANDLED == true ]] && exit $exit_code
   if [[ $exit_code -ne 0 ]]; then
     printf '%s' "Soup failed with error $exit_code: "
     case $exit_code in

From 4e1fba5b38183cad4c479fab7149946c06c8be83 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Mon, 17 May 2021 16:04:13 -0400
Subject: [PATCH 24/48] Only echo error code if not using retry

---
 salt/common/tools/sbin/so-common | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index 0390a2bf2..0b8cd97aa 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -341,7 +341,7 @@ run_check_net_err() {
 	
 	if [[ $exit_code -ne 0 ]]; then
 		ERR_HANDLED=true
-		echo "Command failed with error $exit_code"
+		[[ -z $no_retry ]] && echo "Command failed with error $exit_code"
 		echo "$err_msg"
 		exit $exit_code
 	fi

From ce9554281ee2d83567a9cd1f2b5a9a7a3f1eab13 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Mon, 17 May 2021 16:08:34 -0400
Subject: [PATCH 25/48] Fix backwards logic

---
 salt/common/tools/sbin/so-common | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index 0b8cd97aa..c6b0c0bd8 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -341,7 +341,7 @@ run_check_net_err() {
 	
 	if [[ $exit_code -ne 0 ]]; then
 		ERR_HANDLED=true
-		[[ -z $no_retry ]] && echo "Command failed with error $exit_code"
+		[[ -z $no_retry ]] || echo "Command failed with error $exit_code"
 		echo "$err_msg"
 		exit $exit_code
 	fi

From dbd5ef70c995eb6a8065487d9e3908307e41bf26 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Mon, 17 May 2021 16:19:31 -0400
Subject: [PATCH 26/48] Change retry delay + count

---
 salt/common/tools/sbin/so-common | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index c6b0c0bd8..3ab260eb8 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -332,7 +332,7 @@ run_check_net_err() {
 
 	local exit_code
 	if [[ -z $no_retry ]]; then
-		retry 50 10 "$cmd"
+		retry 20 30 "$cmd"
 		exit_code=$?
 	else
 		eval "$cmd"

From e50002e0ca5f5764212b28c3ee9bc3919984b6bc Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Mon, 17 May 2021 16:26:12 -0400
Subject: [PATCH 27/48] influx and grafana default for manager nodes -
 https://github.com/Security-Onion-Solutions/securityonion/issues/4207

---
 salt/grafana/init.sls       |  2 +-
 salt/influxdb/init.sls      |  2 +-
 salt/soc/files/soc/soc.json |  2 +-
 setup/so-whiptail           | 12 ++++++++++++
 4 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/salt/grafana/init.sls b/salt/grafana/init.sls
index 4a0cc7b08..b6e20bb9d 100644
--- a/salt/grafana/init.sls
+++ b/salt/grafana/init.sls
@@ -11,7 +11,7 @@
 {% set GRAFANA_SETTINGS = salt['grains.filter_by'](default_settings, default='grafana', merge=salt['pillar.get']('grafana', {})) %}
 
 
-{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %}
+{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] or (grains.role == 'so-eval' and GRAFANA == 1) %}
 
 # Grafana all the things
 grafanadir:
diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls
index aace4e827..485cc951b 100644
--- a/salt/influxdb/init.sls
+++ b/salt/influxdb/init.sls
@@ -6,7 +6,7 @@
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 
-{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %}
+{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] or (grains.role == 'so-eval' and GRAFANA == 1) %}
 
 # Influx DB
 influxconfdir:
diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json
index 6f1c3a6da..266b602aa 100644
--- a/salt/soc/files/soc/soc.json
+++ b/salt/soc/files/soc/soc.json
@@ -54,7 +54,7 @@
         "verifyCert": false
       },
       "influxdb": {
-{%- if grains['role'] in ['so-import'] %}
+{%- if grains['role'] in ['so-import'] or GRAFANA == 0 %}
         "hostUrl": "",
 {%- else %}
         "hostUrl": "https://{{ MANAGERIP }}:8086",
diff --git a/setup/so-whiptail b/setup/so-whiptail
index 09346828f..5eca2d39a 100755
--- a/setup/so-whiptail
+++ b/setup/so-whiptail
@@ -408,6 +408,7 @@ whiptail_enable_components() {
 	PLAYBOOK=0
 	STRELKA=0
 
+if [[ $is_eval ]]; then
 	COMPONENTS=$(whiptail --title "Security Onion Setup" --checklist \
 	"Select Components to install:" 20 75 8 \
 	GRAFANA "Enable Grafana for system monitoring" ON \
@@ -416,6 +417,17 @@ whiptail_enable_components() {
 	THEHIVE "Enable TheHive" ON \
 	PLAYBOOK "Enable Playbook" ON \
 	STRELKA "Enable Strelka" ON 3>&1 1>&2 2>&3)
+else
+	COMPONENTS=$(whiptail --title "Security Onion Setup" --checklist \
+	"Select Components to install:" 20 75 7 \
+	OSQUERY "Enable Fleet with osquery" ON \
+	WAZUH "Enable Wazuh" ON \
+	THEHIVE "Enable TheHive" ON \
+	PLAYBOOK "Enable Playbook" ON \
+	STRELKA "Enable Strelka" ON 3>&1 1>&2 2>&3)
+	export "GRAFANA=1"
+fi
+
 	local exitstatus=$?
 	whiptail_check_exitstatus $exitstatus
 

From ef32bff302a8781e68bbf55ceb9fcaa079125fba Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Mon, 17 May 2021 18:29:27 -0400
Subject: [PATCH 28/48] fix up soc.json

---
 salt/soc/files/soc/soc.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json
index 266b602aa..5dfd364d2 100644
--- a/salt/soc/files/soc/soc.json
+++ b/salt/soc/files/soc/soc.json
@@ -54,7 +54,7 @@
         "verifyCert": false
       },
       "influxdb": {
-{%- if grains['role'] in ['so-import'] or GRAFANA == 0 %}
+{%- if grains['role'] in ['so-import'] or (grains['role'] == 'so-eval' and GRAFANA == 0) %}
         "hostUrl": "",
 {%- else %}
         "hostUrl": "https://{{ MANAGERIP }}:8086",

From 0de1c9a6696144f8e46eca2a6a420649690ba55b Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Tue, 18 May 2021 07:57:00 -0400
Subject: [PATCH 29/48] removing unreference pillar file docker/config.sls

---
 pillar/docker/config.sls | 208 ---------------------------------------
 1 file changed, 208 deletions(-)
 delete mode 100644 pillar/docker/config.sls

diff --git a/pillar/docker/config.sls b/pillar/docker/config.sls
deleted file mode 100644
index da6fac83a..000000000
--- a/pillar/docker/config.sls
+++ /dev/null
@@ -1,208 +0,0 @@
-{%- set FLEETMANAGER = salt['pillar.get']('global:fleet_manager', False) -%}
-{%- set FLEETNODE = salt['pillar.get']('global:fleet_node', False) -%}
-{% set WAZUH = salt['pillar.get']('manager:wazuh', '0') %}
-{% set THEHIVE = salt['pillar.get']('manager:thehive', '0') %}
-{% set PLAYBOOK = salt['pillar.get']('manager:playbook', '0') %}
-{% set FREQSERVER = salt['pillar.get']('manager:freq', '0') %}
-{% set DOMAINSTATS = salt['pillar.get']('manager:domainstats', '0') %}
-{% set ZEEKVER = salt['pillar.get']('global:mdengine', 'COMMUNITY') %}
-{% set GRAFANA = salt['pillar.get']('manager:grafana', '0') %}
-
-eval:
-  containers:
-    - so-nginx
-    - so-telegraf
-    {% if  GRAFANA == '1' %}
-    - so-influxdb
-    - so-grafana
-    {% endif %}
-    - so-dockerregistry
-    - so-soc
-    - so-kratos
-    - so-idstools
-    {% if FLEETMANAGER %}
-    - so-mysql
-    - so-fleet
-    - so-redis
-    {% endif %}
-    - so-elasticsearch
-    - so-logstash
-    - so-kibana
-    - so-steno
-    - so-suricata
-    - so-zeek
-    - so-curator
-    - so-elastalert
-    {% if WAZUH != '0' %}
-    - so-wazuh
-    {% endif %}
-    - so-soctopus
-    {% if THEHIVE != '0' %}
-    - so-thehive
-    - so-thehive-es
-    - so-cortex
-    {% endif %}
-    {% if PLAYBOOK != '0' %}
-    - so-playbook
-    {% endif %}
-    {% if FREQSERVER != '0' %}
-    - so-freqserver
-    {% endif %}
-    {% if DOMAINSTATS != '0' %}
-    - so-domainstats
-    {% endif %}
-heavy_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-redis
-    - so-logstash
-    - so-elasticsearch
-    - so-curator
-    - so-steno
-    - so-suricata
-    - so-wazuh
-    - so-filebeat
-    {% if ZEEKVER != 'SURICATA' %}
-    - so-zeek
-    {% endif %}
-helix:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-idstools
-    - so-steno
-    - so-zeek
-    - so-redis
-    - so-logstash
-    - so-filebeat
-hot_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-logstash
-    - so-elasticsearch
-    - so-curator
-manager_search:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-soc
-    - so-kratos
-    - so-acng
-    - so-idstools
-    - so-redis
-    - so-logstash
-    - so-elasticsearch
-    - so-curator
-    - so-kibana
-    - so-elastalert
-    - so-filebeat
-    - so-soctopus
-    {% if FLEETMANAGER %}
-    - so-mysql
-    - so-fleet
-    - so-redis
-    {% endif %}
-    {% if WAZUH != '0' %}
-    - so-wazuh
-    {% endif %}
-    - so-soctopus
-    {% if THEHIVE != '0' %}
-    - so-thehive
-    - so-thehive-es
-    - so-cortex
-    {% endif %}
-    {% if PLAYBOOK != '0' %}
-    - so-playbook
-    {% endif %}
-    {% if FREQSERVER != '0' %}
-    - so-freqserver
-    {% endif %}
-    {% if DOMAINSTATS != '0' %}
-    - so-domainstats
-    {% endif %}
-manager:
-  containers:
-    - so-dockerregistry
-    - so-nginx
-    - so-telegraf
-    {% if  GRAFANA == '1' %}
-    - so-influxdb
-    - so-grafana
-    {% endif %}
-    - so-soc
-    - so-kratos
-    - so-acng
-    - so-idstools
-    - so-redis
-    - so-elasticsearch
-    - so-logstash
-    - so-kibana
-    - so-elastalert
-    - so-filebeat
-    {% if FLEETMANAGER %}
-    - so-mysql
-    - so-fleet
-    - so-redis
-    {% endif %}
-    {% if WAZUH != '0' %}
-    - so-wazuh
-    {% endif %}
-    - so-soctopus
-    {% if THEHIVE != '0' %}
-    - so-thehive
-    - so-thehive-es
-    - so-cortex
-    {% endif %}
-    {% if PLAYBOOK != '0' %}
-    - so-playbook
-    {% endif %}
-    {% if FREQSERVER != '0' %}
-    - so-freqserver
-    {% endif %}
-    {% if DOMAINSTATS != '0' %}
-    - so-domainstats
-    {% endif %}
-parser_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-logstash
-search_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-logstash
-    - so-elasticsearch
-    - so-curator
-    - so-filebeat
-    {% if WAZUH != '0' %}
-    - so-wazuh
-    {% endif %}
-sensor:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-steno
-    - so-suricata
-    {% if ZEEKVER != 'SURICATA' %}
-    - so-zeek
-    {% endif %}
-    - so-wazuh
-    - so-filebeat
-warm_node:
-  containers:
-    - so-nginx
-    - so-telegraf
-    - so-elasticsearch
-fleet:
-  containers:
-    {% if FLEETNODE %}
-    - so-mysql
-    - so-fleet
-    - so-redis
-    - so-filebeat
-    - so-nginx
-    - so-telegraf
-    {% endif %}
\ No newline at end of file

From fe155222c24c3ca01769c659e20a25f01a178cfb Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Tue, 18 May 2021 09:51:54 -0400
Subject: [PATCH 30/48] Introduce mixed-case sensor into distributed test

---
 setup/automation/distributed-iso-sensor | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/automation/distributed-iso-sensor b/setup/automation/distributed-iso-sensor
index 955019bd8..90be8ccd5 100644
--- a/setup/automation/distributed-iso-sensor
+++ b/setup/automation/distributed-iso-sensor
@@ -34,7 +34,7 @@ ZEEKVERSION=ZEEK
 # HELIXAPIKEY=
 HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
 HNSENSOR=inherit
-HOSTNAME=distributed-sensor
+HOSTNAME=Distributed-Sensor
 install_type=SENSOR
 # LSINPUTBATCHCOUNT=
 # LSINPUTTHREADS=

From bdd53ed5e3a806ed26786a43df20b85ce9002294 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Tue, 18 May 2021 12:23:40 -0400
Subject: [PATCH 31/48] Change retry delay + count

---
 salt/common/tools/sbin/so-common       | 2 +-
 salt/common/tools/sbin/so-image-common | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index 3ab260eb8..55350d702 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -332,7 +332,7 @@ run_check_net_err() {
 
 	local exit_code
 	if [[ -z $no_retry ]]; then
-		retry 20 30 "$cmd"
+		retry 5 60 "$cmd"
 		exit_code=$?
 	else
 		eval "$cmd"
diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common
index fa1d6c6dd..2c61838b5 100755
--- a/salt/common/tools/sbin/so-image-common
+++ b/salt/common/tools/sbin/so-image-common
@@ -130,7 +130,7 @@ update_docker_containers() {
 
   # Let's make sure we have the public key
   run_check_net_err \
-  "curl --retry 20 --retry-delay 30 -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \
+  "curl --retry 5 --retry-delay 60 -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \
   "Could not pull signature key file, please ensure connectivity to https://raw.gihubusercontent.com" \
   noretry >> "$LOG_FILE" 2>&1
   result=$?
@@ -158,7 +158,7 @@ update_docker_containers() {
     
     # Get signature
     run_check_net_err \
-    "curl --retry 20 --retry-delay 30 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \
+    "curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \
     "Could not pull signature file, please ensure connectivity to https://sigs.securityonion.net " \
     noretry >> "$LOG_FILE" 2>&1
     if [[ $? -ne 0 ]]; then

From 25e2edc6d238fceccdf6e6d3e48bf883bd35f6ba Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Tue, 18 May 2021 12:31:33 -0400
Subject: [PATCH 32/48] Reset HOTFIX with new release

---
 HOTFIX | 1 -
 1 file changed, 1 deletion(-)

diff --git a/HOTFIX b/HOTFIX
index 364d0a6a8..e69de29bb 100644
--- a/HOTFIX
+++ b/HOTFIX
@@ -1 +0,0 @@
-GRIDFIX ZEEKFIX

From f445186f1ebb2b28fd3f6592e8b07c98fce06fd5 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Tue, 18 May 2021 13:38:55 -0400
Subject: [PATCH 33/48] Remove redundant error messages

---
 salt/common/tools/sbin/so-image-common | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common
index 2c61838b5..f840e07ab 100755
--- a/salt/common/tools/sbin/so-image-common
+++ b/salt/common/tools/sbin/so-image-common
@@ -136,9 +136,6 @@ update_docker_containers() {
   result=$?
   if [[ $result -eq 0 ]]; then
     cat $SIGNPATH/KEYS | gpg --import - >> "$LOG_FILE" 2>&1
-  else
-    echo "Failed to pull signature key file: $result"
-    exit 1
   fi
   
   # Download the containers from the interwebs
@@ -159,12 +156,8 @@ update_docker_containers() {
     # Get signature
     run_check_net_err \
     "curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \
-    "Could not pull signature file, please ensure connectivity to https://sigs.securityonion.net " \
+    "Could not pull signature file for $image, please ensure connectivity to https://sigs.securityonion.net " \
     noretry >> "$LOG_FILE" 2>&1
-    if [[ $? -ne 0 ]]; then
-      echo "Unable to pull signature file for $image" >> "$LOG_FILE" 2>&1 
-      exit 1
-    fi
     # Dump our hash values
     DOCKERINSPECT=$(docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$image)
        

From ac42cba50b6e4679d5dd976bfb1f11fa8428eec6 Mon Sep 17 00:00:00 2001
From: Masaya-A <68965261+Masaya-A@users.noreply.github.com>
Date: Wed, 19 May 2021 09:06:02 +0900
Subject: [PATCH 34/48] Adding MAC Address for NIC List

---
 setup/so-functions | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/setup/so-functions b/setup/so-functions
index dfe79f674..bc5d9d1ca 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -1348,15 +1348,16 @@ filter_unused_nics() {
 	
 	nic_list=()
 	for nic in "${filtered_nics[@]}"; do
+		local nic_mac=$(cat "/sys/class/net/${nic}/address")
 		case $(cat "/sys/class/net/${nic}/carrier" 2>/dev/null) in
 			1)
-				nic_list+=("$nic" "Link UP " "OFF")
+				nic_list+=("$nic" "$nic_mac" "Link UP " "OFF")
 				;;
 			0)
-				nic_list+=("$nic" "Link DOWN " "OFF")
+				nic_list+=("$nic" "$nic_mac" "Link DOWN " "OFF")
 				;;
 			*)
-				nic_list+=("$nic" "Link UNKNOWN " "OFF")
+				nic_list+=("$nic" "$nic_mac" "Link UNKNOWN " "OFF")
 				;;
 		esac
 	done

From 6af126b87200e616864e5788798790919928b84a Mon Sep 17 00:00:00 2001
From: Masaya-A <68965261+Masaya-A@users.noreply.github.com>
Date: Wed, 19 May 2021 10:02:19 +0900
Subject: [PATCH 35/48] Fix array

---
 setup/so-functions | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/setup/so-functions b/setup/so-functions
index bc5d9d1ca..8cdbc2596 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -1351,13 +1351,13 @@ filter_unused_nics() {
 		local nic_mac=$(cat "/sys/class/net/${nic}/address")
 		case $(cat "/sys/class/net/${nic}/carrier" 2>/dev/null) in
 			1)
-				nic_list+=("$nic" "$nic_mac" "Link UP " "OFF")
+				nic_list+=("$nic" "$nic_mac  Link UP " "OFF")
 				;;
 			0)
-				nic_list+=("$nic" "$nic_mac" "Link DOWN " "OFF")
+				nic_list+=("$nic" "$nic_mac  Link DOWN " "OFF")
 				;;
 			*)
-				nic_list+=("$nic" "$nic_mac" "Link UNKNOWN " "OFF")
+				nic_list+=("$nic" "$nic_mac  Link UNKNOWN " "OFF")
 				;;
 		esac
 	done

From b440f733366c6eb946f4a7852207ad59a70ae283 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Wed, 19 May 2021 18:37:08 -0400
Subject: [PATCH 36/48] Truncate wait_for_web_response.log before each wait
 invocation

---
 salt/common/tools/sbin/so-common | 1 +
 1 file changed, 1 insertion(+)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index 22c7543ea..56123f418 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -487,6 +487,7 @@ wait_for_web_response() {
 	expected=$2
 	maxAttempts=${3:-300}
 	logfile=/root/wait_for_web_response.log
+	truncate -s 0 "$logfile"
 	attempt=0
 	while [[ $attempt -lt $maxAttempts ]]; do
 		attempt=$((attempt+1))

From 3450219bc7073fb692259ed185fc88eca2dd516e Mon Sep 17 00:00:00 2001
From: Masaya-A <68965261+Masaya-A@users.noreply.github.com>
Date: Thu, 20 May 2021 09:33:14 +0900
Subject: [PATCH 37/48] Drop error to /dev/null

---
 setup/so-functions | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/so-functions b/setup/so-functions
index 8cdbc2596..5ce3d6dee 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -1348,7 +1348,7 @@ filter_unused_nics() {
 	
 	nic_list=()
 	for nic in "${filtered_nics[@]}"; do
-		local nic_mac=$(cat "/sys/class/net/${nic}/address")
+		local nic_mac=$(cat "/sys/class/net/${nic}/address" 2>/dev/null)
 		case $(cat "/sys/class/net/${nic}/carrier" 2>/dev/null) in
 			1)
 				nic_list+=("$nic" "$nic_mac  Link UP " "OFF")

From 7aed01658fe25e821e9542e9947665d2972a6fd4 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Thu, 20 May 2021 22:10:36 -0400
Subject: [PATCH 38/48] Sig file for 2.3.51

---
 VERIFY_ISO.md                     |  22 +++++++++++-----------
 sigs/securityonion-2.3.51.iso.sig | Bin 0 -> 543 bytes
 2 files changed, 11 insertions(+), 11 deletions(-)
 create mode 100644 sigs/securityonion-2.3.51.iso.sig

diff --git a/VERIFY_ISO.md b/VERIFY_ISO.md
index 2b97521dd..018d9e916 100644
--- a/VERIFY_ISO.md
+++ b/VERIFY_ISO.md
@@ -1,17 +1,17 @@
-### 2.3.50 ISO image built on 2021/04/27
+### 2.3.51 ISO image built on 2021/04/27
 
 
 ### Download and Verify
 
-2.3.50 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.3.50.iso
+2.3.51 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.3.51.iso
 
-MD5: C39CEA68B5A8AFC5CFFB2481797C0374  
-SHA1: 00AD9F29ABE3AB495136989E62EBB8FA00DA82C6  
-SHA256: D77AE370D7863837A989F6735413D1DD46B866D8D135A4C363B0633E3990387E 
+MD5: 7CFB525BEFC0A9F2ED148F5831E387FA  
+SHA1: 8CC34FCCC36822B309B8168AA706B3D1EC7F3BFD  
+SHA256: 9892C2546C9AE5A48015160F379B070F0BE30C89693B97F3F1E1592DDCE1DEE0 
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.50.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.51.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.50.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.51.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.3.50.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.3.51.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.3.50.iso.sig securityonion-2.3.50.iso
+gpg --verify securityonion-2.3.51.iso.sig securityonion-2.3.51.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Tue 27 Apr 2021 02:17:25 PM EDT using RSA key ID FE507013
+gpg: Signature made Thu 20 May 2021 07:49:57 AM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.
diff --git a/sigs/securityonion-2.3.51.iso.sig b/sigs/securityonion-2.3.51.iso.sig
new file mode 100644
index 0000000000000000000000000000000000000000..80137a352152bb142a0ae0d7926ac00aff4b0357
GIT binary patch
literal 543
zcmV+)0^t3L0vrSY0RjL91p;8EOyvLy2@re`V7LBIa1%&k5C2n+tWPThb0<hx<Q1zu
zy$tQDr%}pK0~?{X0Kjqo?p#+VXHNj6^GoePf7DYk-ozV0Cr<H(e*b{;(S@ZTuj>s2
zN5K+$mR8o|yx}I|O-OsnamJ#p&1!`3iX*IDpTyXuf8hx7^*Mx;#+8b5@IkzIu^S7t
zubXL$t0%IHQ!0^Pt$2s;cvlCAxW??=O3#4`i@DTuTc_tm*}+iNodb~MhRI|d-7s-_
zRtSk<NV0|1CeVyLreD{cjZ1e8inMh!!r}qTCuqPMw*2k1AT|bTN>={KP|UGUfHV%G
z8+S&nNnv4jm}<^1etZC{v}TLvI2!{w=wjc1E*=nu%|OFkP)q{t$W+qxldpv|BguU$
zwd5eMir&G8VS4vhh!r5wD4>c&Nhn6pVPNao_5G3?n}KLn)tSjpD0tg1;xnn8q_Vbp
zSpYprNxdL|n+Ol~FhWIupF2ELwoq&m3PBW0Sfra18PlSt0f%l2<M(1Fzmm#b8ZWG>
ztVa<g<@&EXC6m!^Hf1sX-KNBFKaeBoWXO))u*UX~Bx6^_abHY>FaGx?;kjP{C`Ln<
z4@d%!)Q~Z_NPCkH*8Y(aY%ajD8jB#hVTkFP<ngD@=n1Gu&zSD?<WP9o#F(!$Fdqp~
h^ZAGT_Ec8Hvi8*3Kb<IY{}858sYBN5E3NnrC>rpc2&4c2

literal 0
HcmV?d00001


From c1dd4dafe47001101e949d5a8fbefaca0d97f987 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Fri, 21 May 2021 12:41:10 -0400
Subject: [PATCH 39/48] Fix influx state

---
 salt/influxdb/init.sls | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls
index bafcbd53b..346d971fd 100644
--- a/salt/influxdb/init.sls
+++ b/salt/influxdb/init.sls
@@ -3,7 +3,7 @@
 
 {% set GRAFANA = salt['pillar.get']('manager:grafana', '0') %}
 
-{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %}
+{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] or (grains.role == 'so-eval' and GRAFANA == 1) %}
 
 {% set MANAGER = salt['grains.get']('master') %}
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}

From 99d41d1606245cc0d75520545e55bf9b27dd1e9b Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Mon, 24 May 2021 11:29:40 -0400
Subject: [PATCH 40/48] Add ending newline to soup

---
 salt/common/tools/sbin/soup | 1 +
 1 file changed, 1 insertion(+)

diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index 9a1afb2bb..eb15305aa 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -999,3 +999,4 @@ EOF
 read -r input
 
 main "$@" | tee -a $SOUP_LOG
+  
\ No newline at end of file

From 686c7c5a6c5cf5010851381900f1cd8f9ea3a2f5 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Mon, 24 May 2021 13:26:43 -0400
Subject: [PATCH 41/48] Add exception handling for docker API error to
 so-docker-prune

---
 salt/common/tools/sbin/so-docker-prune | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/salt/common/tools/sbin/so-docker-prune b/salt/common/tools/sbin/so-docker-prune
index f6c043ef3..754a0f171 100755
--- a/salt/common/tools/sbin/so-docker-prune
+++ b/salt/common/tools/sbin/so-docker-prune
@@ -38,7 +38,7 @@ def get_image_version(string) -> str:
       Version(ver)
     except InvalidVersion:
       # Strip the last substring following a hyphen for automated branches
-      ver = '-'.join(ver.split('-')[:-1])  
+      ver = '-'.join(ver.split('-')[:-1])
     return ver
 
 
@@ -73,8 +73,12 @@ def main(quiet):
           for tag in group:
             if not quiet: print(f'Removing image {tag}')
             client.images.remove(tag)
-    except InvalidVersion as e:
-      print(f'so-{get_so_image_basename(t_list[0])}: {e.args[0]}', file=sys.stderr)
+    except (docker.errors.APIError, InvalidVersion) as e:
+      print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
+      exit(1)
+    except Exception as e:
+      print('Unhandled exception occurred:')
+      print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
       exit(1)
 
   if no_prunable and not quiet:
@@ -86,4 +90,4 @@ if __name__ == "__main__":
   main_parser.add_argument('-q', '--quiet', action='store_const', const=True, required=False)
   args = main_parser.parse_args(sys.argv[1:])
  
-  main(args.quiet)
\ No newline at end of file
+  main(args.quiet)

From dcb89b704a373cf804b318b2c261e97e39dcb4e2 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Mon, 24 May 2021 13:45:12 -0400
Subject: [PATCH 42/48] Move sensoroni state out of the * block of top.sls

Resolves #3559
---
 salt/top.sls | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/salt/top.sls b/salt/top.sls
index 8a12aaa26..f3ccc9dda 100644
--- a/salt/top.sls
+++ b/salt/top.sls
@@ -38,7 +38,6 @@ base:
     - patch.os.schedule
     - motd
     - salt.minion-check
-    - sensoroni
     - salt.lasthighstate
   
   '*_helixsensor and G@saltversion:{{saltversion}}':
@@ -47,6 +46,7 @@ base:
     - ca
     - ssl
     - registry
+    - sensoroni
     - telegraf
     - firewall
     - idstools
@@ -66,6 +66,7 @@ base:
     - match: compound
     - ca
     - ssl
+    - sensoroni
     - telegraf
     - firewall
     - nginx
@@ -92,6 +93,7 @@ base:
     - ca
     - ssl
     - registry
+    - sensoroni
     - manager
     - nginx
     - telegraf
@@ -160,6 +162,7 @@ base:
     - ca
     - ssl
     - registry
+    - sensoroni
     - nginx
     - telegraf
     - influxdb
@@ -220,6 +223,7 @@ base:
     - ca
     - ssl
     - registry
+    - sensoroni
     - manager
     - nginx
     - telegraf
@@ -290,6 +294,7 @@ base:
     - match: compound
     - ca
     - ssl
+    - sensoroni
     - nginx
     - telegraf
     - firewall
@@ -320,6 +325,7 @@ base:
     - ca
     - ssl
     - registry
+    - sensoroni
     - nginx
     - telegraf
     - influxdb
@@ -382,6 +388,7 @@ base:
     - match: compound
     - ca
     - ssl
+    - sensoroni
     - nginx
     - telegraf
     - firewall
@@ -424,6 +431,7 @@ base:
     - match: compound
     - ca
     - ssl
+    - sensoroni
     - nginx
     - telegraf
     - firewall
@@ -441,6 +449,7 @@ base:
     - ca
     - ssl
     - registry
+    - sensoroni
     - manager
     - nginx
     - soc

From ffd5bfc48063aeb501def0030748b93091dd6678 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Mon, 24 May 2021 15:25:03 -0400
Subject: [PATCH 43/48] Force images from automated branches to a very high
 semver

---
 salt/common/tools/sbin/so-docker-prune | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/salt/common/tools/sbin/so-docker-prune b/salt/common/tools/sbin/so-docker-prune
index 754a0f171..2dd183c10 100755
--- a/salt/common/tools/sbin/so-docker-prune
+++ b/salt/common/tools/sbin/so-docker-prune
@@ -37,8 +37,10 @@ def get_image_version(string) -> str:
     try:
       Version(ver)
     except InvalidVersion:
-      # Strip the last substring following a hyphen for automated branches
-      ver = '-'.join(ver.split('-')[:-1])
+      # Also return a very high (but less than 'latest') semver for automated branches 
+      # since the image will most likely be the latest version
+      if any(substr in ver for substr in ['bravo', 'delta', 'foxtrot', 'kilo']):
+        return '99999.9.9'   
     return ver
 
 

From 3bc0def02af00ab5741f903722f8316e68d3bfe1 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Mon, 24 May 2021 16:45:05 -0400
Subject: [PATCH 44/48] Add failure message to salt-master check

---
 salt/common/tools/sbin/soup | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index eb15305aa..d2b96b194 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -890,8 +890,10 @@ main() {
     echo "Starting Salt Master service."
     systemctl start salt-master
 
-    # Testing that that salt-master is up by checking that is it connected to itself
-    retry 50 10 "salt-call state.show_top -l error"
+    # Testing that salt-master is up by checking that is it connected to itself
+    set +e
+    retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check /root/soup.log for details."
+    set -e
 
     echo ""
     echo "Ensuring python modules for Salt are installed and patched."
@@ -924,9 +926,11 @@ main() {
     echo "Starting Salt Master service."
     systemctl start salt-master
 
-    # Testing that that salt-master is up by checking that is it connected to itself
-    retry 50 10 "salt-call state.show_top -l error"
-
+    # Testing that salt-master is up by checking that is it connected to itself
+    set +e
+    retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check /root/soup.log for details."
+    set -e
+    
     echo "Running a highstate. This could take several minutes."
     salt-call state.highstate -l info queue=True
     postupgrade_changes

From dbc4ffd69ad757d59d9e3d8351a7f1c1510fa0e6 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Tue, 25 May 2021 09:20:45 -0400
Subject: [PATCH 45/48] Fix typo

---
 salt/common/tools/sbin/so-image-common | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common
index f840e07ab..9b6e2174a 100755
--- a/salt/common/tools/sbin/so-image-common
+++ b/salt/common/tools/sbin/so-image-common
@@ -151,7 +151,7 @@ update_docker_containers() {
     local image=$i:$VERSION$IMAGE_TAG_SUFFIX
     run_check_net_err \
     "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" \
-    "Could not pull pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 
+    "Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 
     
     # Get signature
     run_check_net_err \

From 7e008378bae365a16ae4d9741e3346eebcae04c0 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Tue, 25 May 2021 09:23:44 -0400
Subject: [PATCH 46/48] Replace string with variable, remove unnecessary text

---
 salt/common/tools/sbin/soup | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index d2b96b194..7fca8ceb9 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -688,7 +688,7 @@ upgrade_salt() {
     set +e
     run_check_net_err \
     "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \
-    "Could not update salt, please check $SOUP_LOG or scrollback for details."
+    "Could not update salt, please check $SOUP_LOG for details."
     set -e
     echo "Applying yum versionlock for Salt."
     echo ""
@@ -705,7 +705,7 @@ upgrade_salt() {
     set +e
     run_check_net_err \
     "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \
-    "Could not update salt, please check $SOUP_LOG or scrollback for details."
+    "Could not update salt, please check $SOUP_LOG for details."
     set -e
     echo "Applying apt hold for Salt."
     echo ""
@@ -892,7 +892,7 @@ main() {
 
     # Testing that salt-master is up by checking that is it connected to itself
     set +e
-    retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check /root/soup.log for details."
+    retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check $SOUP_LOG for details."
     set -e
 
     echo ""
@@ -928,7 +928,7 @@ main() {
 
     # Testing that salt-master is up by checking that is it connected to itself
     set +e
-    retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check /root/soup.log for details."
+    retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check $SOUP_LOG for details."
     set -e
     
     echo "Running a highstate. This could take several minutes."

From 192cec182550135ad711b4c58b61cbe86c5ae7bb Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Tue, 25 May 2021 11:25:12 -0400
Subject: [PATCH 47/48] Change how version with dashes are handled by
 so-docker-prune

---
 salt/common/tools/sbin/so-docker-prune | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/salt/common/tools/sbin/so-docker-prune b/salt/common/tools/sbin/so-docker-prune
index 2dd183c10..b1c359b58 100755
--- a/salt/common/tools/sbin/so-docker-prune
+++ b/salt/common/tools/sbin/so-docker-prune
@@ -32,15 +32,15 @@ def get_image_version(string) -> str:
   ver = string.split(':')[-1]
   if ver == 'latest':
     # Version doesn't like "latest", so use a high semver
-    return '999999.9.9'
+    return '99999.9.9'
   else:
     try:
       Version(ver)
     except InvalidVersion:
-      # Also return a very high (but less than 'latest') semver for automated branches 
-      # since the image will most likely be the latest version
-      if any(substr in ver for substr in ['bravo', 'delta', 'foxtrot', 'kilo']):
-        return '99999.9.9'   
+      # Also return a very high semver for any version 
+      # with a dash in it since it will likely be a dev version of some kind
+      if '-' in ver:
+        return '999999.9.9'
     return ver
 
 

From 94af55a95104cd4229a315343973c8167b6cadd6 Mon Sep 17 00:00:00 2001
From: William Wernert <william.wernert@securityonionsolutions.com>
Date: Tue, 25 May 2021 11:25:37 -0400
Subject: [PATCH 48/48] Fix typo

---
 salt/common/tools/sbin/soup | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index 7fca8ceb9..d1261fe30 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -60,7 +60,7 @@ check_err() {
         echo 'Link has been severed'
       ;;
       100)
-        echo 'Netowrk is down'
+        echo 'Network is down'
       ;;
       101)
         echo 'Network is unreachable'