Merge pull request #4289 from Security-Onion-Solutions/foxtrot

Soup error handling, reorder sensoroni state

salt/common/tools/sbin/so-common

@@ -302,6 +302,7 @@ retry() {
 	cmd=$3
 	expectedOutput=$4
 	attempt=0
+	local exitcode=0
 	while [[ $attempt -lt $maxAttempts ]]; do			
 		attempt=$((attempt+1))
 		echo "Executing command with retry support: $cmd"
@@ -321,7 +322,29 @@ retry() {
 		sleep $sleepDelay
 	done
 	echo "Command continues to fail; giving up."
-	return 1
+	return $exitcode
+}
+
+run_check_net_err() {
+	local cmd=$1
+	local err_msg=${2:-"Unknown error occured, please check /root/$WHATWOULDYOUSAYYAHDOHERE.log for details."} # Really need to rename that variable
+	local no_retry=$3
+
+	local exit_code
+	if [[ -z $no_retry ]]; then
+		retry 5 60 "$cmd"
+		exit_code=$?
+	else
+		eval "$cmd"
+		exit_code=$?
+	fi
+	
+	if [[ $exit_code -ne 0 ]]; then
+		ERR_HANDLED=true
+		[[ -z $no_retry ]] || echo "Command failed with error $exit_code"
+		echo "$err_msg"
+		exit $exit_code
+	fi
 }
 
 set_os() {

salt/common/tools/sbin/so-docker-prune

@@ -32,13 +32,15 @@ def get_image_version(string) -> str:
   ver = string.split(':')[-1]
   if ver == 'latest':
     # Version doesn't like "latest", so use a high semver
-    return '999999.9.9'
+    return '99999.9.9'
   else:
     try:
       Version(ver)
     except InvalidVersion:
-      # Strip the last substring following a hyphen for automated branches
+      # Also return a very high semver for any version 
-      ver = '-'.join(ver.split('-')[:-1])  
+      # with a dash in it since it will likely be a dev version of some kind
+      if '-' in ver:
+        return '999999.9.9'
     return ver
 
 
@@ -73,8 +75,12 @@ def main(quiet):
           for tag in group:
             if not quiet: print(f'Removing image {tag}')
             client.images.remove(tag)
-    except InvalidVersion as e:
+    except (docker.errors.APIError, InvalidVersion) as e:
-      print(f'so-{get_so_image_basename(t_list[0])}: {e.args[0]}', file=sys.stderr)
+      print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
+      exit(1)
+    except Exception as e:
+      print('Unhandled exception occurred:')
+      print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
       exit(1)
 
   if no_prunable and not quiet:
@@ -86,4 +92,4 @@ if __name__ == "__main__":
   main_parser.add_argument('-q', '--quiet', action='store_const', const=True, required=False)
   args = main_parser.parse_args(sys.argv[1:])
  
-  main(args.quiet)
+  main(args.quiet)

salt/common/tools/sbin/so-image-common

@@ -18,6 +18,7 @@
 # NOTE: This script depends on so-common
 IMAGEREPO=security-onion-solutions
 
+# shellcheck disable=SC2120
 container_list() {
   MANAGERCHECK=$1
   
@@ -128,13 +129,13 @@ update_docker_containers() {
   mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1 
 
   # Let's make sure we have the public key
-  retry 50 10 "curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" >> "$LOG_FILE" 2>&1
+  run_check_net_err \
+  "curl --retry 5 --retry-delay 60 -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \
+  "Could not pull signature key file, please ensure connectivity to https://raw.gihubusercontent.com" \
+  noretry >> "$LOG_FILE" 2>&1
   result=$?
   if [[ $result -eq 0 ]]; then
     cat $SIGNPATH/KEYS | gpg --import - >> "$LOG_FILE" 2>&1
-  else
-    echo "Failed to pull signature key file: $result"
-    exit 1
   fi
   
   # Download the containers from the interwebs
@@ -148,14 +149,15 @@ update_docker_containers() {
 
     # Pull down the trusted docker image
     local image=$i:$VERSION$IMAGE_TAG_SUFFIX
-    retry 50 10 "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" >> "$LOG_FILE" 2>&1 
+    run_check_net_err \
+    "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" \
+    "Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 
     
     # Get signature
-    retry 50 10 "curl -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" >> "$LOG_FILE" 2>&1
+    run_check_net_err \
-    if [[ $? -ne 0 ]]; then
+    "curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \
-      echo "Unable to pull signature file for $image" >> "$LOG_FILE" 2>&1 
+    "Could not pull signature file for $image, please ensure connectivity to https://sigs.securityonion.net " \
-      exit 1
+    noretry >> "$LOG_FILE" 2>&1
-    fi
     # Dump our hash values
     DOCKERINSPECT=$(docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$image)
        

salt/common/tools/sbin/soup

@@ -20,12 +20,79 @@
 UPDATE_DIR=/tmp/sogh/securityonion
 INSTALLEDVERSION=$(cat /etc/soversion)
 POSTVERSION=$INSTALLEDVERSION
-INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'})
+INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
 BATCHSIZE=5
 SOUP_LOG=/root/soup.log
 INFLUXDB_MIGRATION_LOG=/opt/so/log/influxdb/soup_migration.log
 WHATWOULDYOUSAYYAHDOHERE=soup
 
+check_err() {
+  local exit_code=$1
+  local err_msg="Unhandled error occured, please check $SOUP_LOG for details."
+
+  [[ $ERR_HANDLED == true ]] && exit $exit_code
+  if [[ $exit_code -ne 0 ]]; then
+    printf '%s' "Soup failed with error $exit_code: "
+    case $exit_code in
+      2)
+        echo 'No such file or directory'
+      ;;
+      5)
+        echo 'Interrupted system call'
+      ;;
+      12)
+        echo 'Out of memory'
+      ;;
+      28)
+        echo 'No space left on device'
+        echo 'Likely ran out of space on disk, please review hardware requirements for Security Onion: https://docs.securityonion.net/en/2.3/hardware.html'
+      ;;
+      30)
+        echo 'Read-only file system'
+      ;;
+      35)
+        echo 'Resource temporarily unavailable'
+      ;;
+      64)
+        echo 'Machine is not on the network'
+      ;;
+      67)
+        echo 'Link has been severed'
+      ;;
+      100)
+        echo 'Network is down'
+      ;;
+      101)
+        echo 'Network is unreachable'
+      ;;
+      102)
+        echo 'Network reset'
+      ;;
+      110)
+        echo 'Connection timed out'
+      ;;
+      111)
+        echo 'Connection refused'
+      ;;
+      112)
+        echo 'Host is down'
+      ;;
+      113)
+        echo 'No route to host'
+      ;;
+      *)
+        echo 'Unhandled error'
+        echo "$err_msg"
+      ;;
+    esac
+    if [[ $exit_code -ge 64 && $exit_code -le 113 ]]; then
+      echo "$err_msg"
+    fi
+    exit $exit_code
+  fi
+
+}
+
 add_common() {
   cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
   cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
@@ -47,8 +114,8 @@ airgap_mounted() {
     echo "Example: /home/user/securityonion-2.X.0.iso"
     echo "Example: /dev/sdx1"
     echo ""
-    read -p 'Enter the location of the iso: ' ISOLOC
+    read -rp 'Enter the location of the iso: ' ISOLOC
-    if [ -f $ISOLOC ]; then
+    if [[ -f $ISOLOC ]]; then
       # Mounting the ISO image
       mkdir -p /tmp/soagupdate
       mount -t iso9660 -o loop $ISOLOC /tmp/soagupdate
@@ -60,7 +127,7 @@ airgap_mounted() {
       else
         echo "ISO has been mounted!"
       fi  
-    elif [ -f $ISOLOC/SecurityOnion/VERSION ]; then
+    elif [[ -f $ISOLOC/SecurityOnion/VERSION ]]; then
       ln -s $ISOLOC /tmp/soagupdate
       echo "Found the update content"
     else 
@@ -78,9 +145,9 @@ airgap_mounted() {
 }
 
 airgap_update_dockers() {
-  if [ $is_airgap -eq 0 ]; then
+  if [[ $is_airgap -eq 0 ]]; then
     # Let's copy the tarball
-    if [ ! -f $AGDOCKER/registry.tar ]; then
+    if [[ ! -f $AGDOCKER/registry.tar ]]; then
       echo "Unable to locate registry. Exiting"
       exit 1
     else
@@ -88,9 +155,9 @@ airgap_update_dockers() {
       docker stop so-dockerregistry
       docker rm so-dockerregistry
       echo "Copying the new dockers over"
-      tar xvf $AGDOCKER/registry.tar -C /nsm/docker-registry/docker
+      tar xvf "$AGDOCKER/registry.tar" -C /nsm/docker-registry/docker
       echo "Add Registry back"
-      docker load -i $AGDOCKER/registry_image.tar
+      docker load -i "$AGDOCKER/registry_image.tar"
     fi
   fi
 }
@@ -102,9 +169,9 @@ update_registry() {
 }
 
 check_sudoers() {
-	if grep -q "so-setup" /etc/sudoers; then
+  if grep -q "so-setup" /etc/sudoers; then
-		echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"."
+    echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"."
-	fi
+  fi
 }
 
 check_log_size_limit() {
@@ -178,7 +245,9 @@ check_os_updates() {
           echo "Continuing without updating packages"
       elif [[ "$confirm" == [uU] ]]; then
           echo "Applying Grid Updates"
-          salt \* -b 5 state.apply patch.os queue=True
+          set +e
+          run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.'
+          set -e
       else
           echo "Exiting soup"
           exit 0
@@ -206,7 +275,9 @@ clone_to_tmp() {
   if [ -n "$BRANCH" ]; then
     SOUP_BRANCH="-b $BRANCH"
   fi
-  git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git
+  set +e
+  run_check_net_err "git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git" "Could not clone repo, please ensure network access to https://github.com"
+  set -e
   cd /tmp
   if [ ! -f $UPDATE_DIR/VERSION ]; then
     echo "Update was unable to pull from github. Please check your internet."
@@ -253,6 +324,7 @@ preupgrade_changes_2.3.50_repo() {
     echo "Checking to see if 2.3.50 repo changes are needed."
 
     [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50_repo
+    true
 }
 
 preupgrade_changes() {
@@ -265,6 +337,7 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20
     [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30
     [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50
+    true
 }
 
 postupgrade_changes() {
@@ -275,6 +348,7 @@ postupgrade_changes() {
     [[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30
     [[ "$POSTVERSION" == 2.3.30 ]] && post_2.3.30_to_2.3.40
     [[ "$POSTVERSION" == 2.3.50 ]] && post_2.3.5X_to_2.3.60
+    true
 }
 
 post_rc1_to_2.3.21() {
@@ -432,7 +506,7 @@ up_2.3.2X_to_2.3.30() {
   sed -i "/  imagerepo: securityonion/c\  imagerepo: 'security-onion-solutions'" /opt/so/saltstack/local/pillar/global.sls 
 
   # Strelka rule repo pillar addition
-  if [ $is_airgap -eq 0 ]; then
+  if [[ $is_airgap -eq 0 ]]; then
       # Add manager as default Strelka YARA rule repo
       sed -i "/^strelka:/a \\  repos: \n    - https://$HOSTNAME/repo/rules/strelka" /opt/so/saltstack/local/pillar/global.sls;
   else
@@ -459,7 +533,7 @@ up_2.3.3X_to_2.3.50_repo() {
         rm -f "/etc/yum.repos.d/$DELREPO.repo"
       fi
     done
-    if [ $is_airgap -eq 1 ]; then
+    if [[ $is_airgap -eq 1 ]]; then
       # Copy the new repo file if not airgap
       cp $UPDATE_DIR/salt/repo/client/files/centos/securityonion.repo /etc/yum.repos.d/
       yum clean all
@@ -575,7 +649,7 @@ upgrade_check() {
   # Let's make sure we actually need to update.
   NEWVERSION=$(cat $UPDATE_DIR/VERSION)
   HOTFIXVERSION=$(cat $UPDATE_DIR/HOTFIX)
-  CURRENTHOTFIX=$(cat /etc/sohotfix 2>/dev/null)
+  [[ -f /etc/sohotfix ]] && CURRENTHOTFIX=$(cat /etc/sohotfix)
   if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then
     echo "Checking to see if there are hotfixes needed"
     if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then
@@ -592,13 +666,14 @@ upgrade_check() {
 }
 
 upgrade_check_salt() {
-  NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk {'print $2'})
+  NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk '{print $2}')
   if [ "$INSTALLEDSALTVERSION" == "$NEWSALTVERSION" ]; then
     echo "You are already running the correct version of Salt for Security Onion."
   else
     UPGRADESALT=1
   fi
 }   
+
 upgrade_salt() {
   SALTUPGRADED=True
   echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION."
@@ -610,7 +685,11 @@ upgrade_salt() {
     yum versionlock delete "salt-*"
     echo "Updating Salt packages and restarting services."
     echo ""
-    sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION"
+    set +e
+    run_check_net_err \
+    "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \
+    "Could not update salt, please check $SOUP_LOG for details."
+    set -e
     echo "Applying yum versionlock for Salt."
     echo ""
     yum versionlock add "salt-*"
@@ -623,7 +702,11 @@ upgrade_salt() {
     apt-mark unhold "salt-minion"
     echo "Updating Salt packages and restarting services."
     echo ""
-    sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION"
+    set +e
+    run_check_net_err \
+    "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \
+    "Could not update salt, please check $SOUP_LOG for details."
+    set -e
     echo "Applying apt hold for Salt."
     echo ""
     apt-mark hold "salt-common"
@@ -648,234 +731,244 @@ verify_latest_update_script() {
     cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/
     cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
     cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
-    salt-call state.apply common queue=True
+    salt-call state.apply -l info common queue=True
     echo ""
     echo "soup has been updated. Please run soup again."
     exit 0
   fi
 }
 
-main () {
+main() {
-echo "### Preparing soup at `date` ###"
+  set -e
-while getopts ":b" opt; do
+  trap 'check_err $?' EXIT
-  case "$opt" in
-    b ) # process option b
-       shift
-       BATCHSIZE=$1
-       if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then
-         echo "Batch size must be a number greater than 0."
-         exit 1
-       fi
-    ;;
-    \? ) 
-      echo "Usage: cmd [-b]"
-    ;;
-  esac
-done
-
-echo "Checking to see if this is a manager."
-echo ""
-require_manager
-set_minionid
-echo "Checking to see if this is an airgap install"
-echo ""
-check_airgap
-echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
-echo ""
-set_os
-set_palette
-check_elastic_license
-echo ""
-if [ $is_airgap -eq 0 ]; then
-  # Let's mount the ISO since this is airgap
-  airgap_mounted
-else
-  echo "Cloning Security Onion github repo into $UPDATE_DIR."
-  echo "Removing previous upgrade sources."
-  rm -rf $UPDATE_DIR
-  clone_to_tmp
-fi
-check_os_updates
-echo ""
-echo "Verifying we have the latest soup script."
-verify_latest_update_script
-echo ""
-
-echo "Generating new repo archive"
-generate_and_clean_tarballs
-if [ -f /usr/sbin/so-image-common ]; then
-  . /usr/sbin/so-image-common
-else 
-add_common
-fi
-
-echo "Let's see if we need to update Security Onion."
-upgrade_check
-upgrade_space
-
-echo "Checking for Salt Master and Minion updates."
-upgrade_check_salt
-
-
-if [ "$is_hotfix" == "true" ]; then
-  echo "Applying $HOTFIXVERSION" 
-  copy_new_files
-  echo ""
-  update_version
-  salt-call state.highstate -l info queue=True
   
-else
+  echo "### Preparing soup at $(date) ###"
-  echo ""
+  while getopts ":b" opt; do
-  echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
+    case "$opt" in
-  echo ""
+      b ) # process option b
-
+        shift
-  echo "Updating dockers to $NEWVERSION."
+        BATCHSIZE=$1
-  if [ $is_airgap -eq 0 ]; then
+        if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then
-    airgap_update_dockers
+          echo "Batch size must be a number greater than 0."
-    update_centos_repo
+          exit 1
-    yum clean all
+        fi
-    check_os_updates
-  else
-    update_registry
-    update_docker_containers "soup"
-  fi
-
-  echo ""
-  echo "Stopping Salt Minion service."
-  systemctl stop salt-minion
-  echo "Killing any remaining Salt Minion processes."
-  pkill -9 -ef /usr/bin/salt-minion
-  echo ""
-  echo "Stopping Salt Master service."
-  systemctl stop salt-master
-  echo ""
-
-  preupgrade_changes_2.3.50_repo
-
-  # Does salt need upgraded. If so update it.
-  if [ "$UPGRADESALT" == "1" ]; then
-    echo "Upgrading Salt"
-    # Update the repo files so it can actually upgrade
-    upgrade_salt
-  fi
-
-  echo "Checking if Salt was upgraded."
-  echo ""
-  # Check that Salt was upgraded
-  SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk {'print $2'})
-  if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then
-    echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG."
-    echo "Once the issue is resolved, run soup again."
-    echo "Exiting."
-    echo ""
-    exit 1
-  else
-    echo "Salt upgrade success."
-    echo ""
-  fi
-
-  preupgrade_changes
-  echo ""
-
-  if [ $is_airgap -eq 0 ]; then
-    echo "Updating Rule Files to the Latest."
-    update_airgap_rules
-  fi
-
-  # Only update the repo if its airgap
-  if [[ $is_airgap -eq 0 ]] && [[ "$UPGRADESALT" != "1" ]]; then
-  update_centos_repo
-  fi
-
-  echo ""
-  echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR."
-  copy_new_files
-  echo ""
-  update_version
-
-  echo ""
-  echo "Locking down Salt Master for upgrade"
-  masterlock
-
-  echo ""
-  echo "Starting Salt Master service."
-  systemctl start salt-master
-
-  # Testing that that salt-master is up by checking that is it connected to itself
-  retry 50 10 "salt-call state.show_top -l error" || exit 1
-
-  echo ""
-  echo "Ensuring python modules for Salt are installed and patched."
-  salt-call state.apply salt.python3-influxdb -l info queue=True
-  echo ""
-
-  # Only regenerate osquery packages if Fleet is enabled
-  FLEET_MANAGER=$(lookup_pillar fleet_manager)
-  FLEET_NODE=$(lookup_pillar fleet_node)
-  if [[ "$FLEET_MANAGER" == "True" || "$FLEET_NODE" == "True" ]]; then
-    echo ""
-    echo "Regenerating Osquery Packages.... This will take several minutes."
-    salt-call state.apply fleet.event_gen-packages -l info queue=True
-    echo ""
-  fi
-
-  echo ""
-  echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
-  salt-call state.highstate -l info queue=True
-  echo ""
-  echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete."
-
-  echo ""
-  echo "Stopping Salt Master to remove ACL"
-  systemctl stop salt-master
-
-  masterunlock
-
-  echo ""
-  echo "Starting Salt Master service."
-  systemctl start salt-master
-
-  # Testing that that salt-master is up by checking that is it connected to itself
-  retry 50 10 "salt-call state.show_top -l error" || exit 1
-
-  echo "Running a highstate. This could take several minutes."
-  salt-call state.highstate -l info queue=True
-  postupgrade_changes
-  unmount_update
-  thehive_maint
-
-  if [ "$UPGRADESALT" == "1" ]; then
-    if [ $is_airgap -eq 0 ]; then
-      echo ""
-      echo "Cleaning repos on remote Security Onion nodes."
-      salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone and G@os:CentOS' cmd.run "yum clean all"
-      echo ""
-    fi
-  fi
-
-  check_sudoers
-
-  if [[ -n $lsl_msg ]]; then
-    case $lsl_msg in
-      'distributed')
-        echo "[INFO] The value of log_size_limit in any heavy node minion pillars may be incorrect."
-        echo " -> We recommend checking and adjusting the values as necessary."
-        echo " -> Minion pillar directory: /opt/so/saltstack/local/pillar/minions/"
       ;;
-      'single-node')
+      \? ) 
-        # We can assume the lsl_details array has been set if lsl_msg has this value
+        echo "Usage: cmd [-b]"
-        echo "[WARNING] The value of log_size_limit (${lsl_details[0]}) does not match the recommended value of ${lsl_details[1]}."
-        echo " -> We recommend checking and adjusting the value as necessary."
-        echo " -> File: /opt/so/saltstack/local/pillar/minions/${lsl_details[2]}.sls"
       ;;
     esac
+  done
+
+  echo "Checking to see if this is a manager."
+  echo ""
+  require_manager
+  set_minionid
+  echo "Checking to see if this is an airgap install"
+  echo ""
+  check_airgap
+  echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
+  echo ""
+  set_os
+  set_palette
+  check_elastic_license
+  echo ""
+  if [[ $is_airgap -eq 0 ]]; then
+    # Let's mount the ISO since this is airgap
+    airgap_mounted
+  else
+    echo "Cloning Security Onion github repo into $UPDATE_DIR."
+    echo "Removing previous upgrade sources."
+    rm -rf $UPDATE_DIR
+    clone_to_tmp
+  fi
+  check_os_updates
+  echo ""
+  echo "Verifying we have the latest soup script."
+  verify_latest_update_script
+  echo ""
+
+  echo "Generating new repo archive"
+  generate_and_clean_tarballs
+  if [ -f /usr/sbin/so-image-common ]; then
+    . /usr/sbin/so-image-common
+  else 
+  add_common
   fi
 
-  NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l)
+  echo "Let's see if we need to update Security Onion."
+  upgrade_check
+  upgrade_space
 
-  if [ $NUM_MINIONS -gt 1 ]; then
+  echo "Checking for Salt Master and Minion updates."
+  upgrade_check_salt
 
-    cat << EOF
+
+  if [ "$is_hotfix" == "true" ]; then
+    echo "Applying $HOTFIXVERSION" 
+    copy_new_files
+    echo ""
+    update_version
+    salt-call state.highstate -l info queue=True
+  else
+    echo ""
+    echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
+    echo ""
+
+    echo "Updating dockers to $NEWVERSION."
+    if [[ $is_airgap -eq 0 ]]; then
+      airgap_update_dockers
+      update_centos_repo
+      yum clean all
+      check_os_updates
+    else
+      update_registry
+      set +e
+      update_docker_containers "soup"
+      set -e
+    fi
+
+    echo ""
+    echo "Stopping Salt Minion service."
+    systemctl stop salt-minion
+    echo "Killing any remaining Salt Minion processes."
+    set +e
+    pkill -9 -ef /usr/bin/salt-minion
+    set -e
+    echo ""
+    echo "Stopping Salt Master service."
+    systemctl stop salt-master
+    echo ""
+
+    preupgrade_changes_2.3.50_repo
+
+    # Does salt need upgraded. If so update it.
+    if [[ $UPGRADESALT -eq 1 ]]; then
+      echo "Upgrading Salt"
+      # Update the repo files so it can actually upgrade
+      upgrade_salt
+    fi
+
+    echo "Checking if Salt was upgraded."
+    echo ""
+    # Check that Salt was upgraded
+    SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk '{print $2}')
+    if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then
+      echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG."
+      echo "Once the issue is resolved, run soup again."
+      echo "Exiting."
+      echo ""
+      exit 1
+    else
+      echo "Salt upgrade success."
+      echo ""
+    fi
+
+    preupgrade_changes
+    echo ""
+
+    if [[ $is_airgap -eq 0 ]]; then
+      echo "Updating Rule Files to the Latest."
+      update_airgap_rules
+    fi
+
+    # Only update the repo if its airgap
+    if [[ $is_airgap -eq 0 && $UPGRADESALT -ne 1 ]]; then
+      update_centos_repo
+    fi
+
+    echo ""
+    echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR."
+    copy_new_files
+    echo ""
+    update_version
+
+    echo ""
+    echo "Locking down Salt Master for upgrade"
+    masterlock
+
+    echo ""
+    echo "Starting Salt Master service."
+    systemctl start salt-master
+
+    # Testing that salt-master is up by checking that is it connected to itself
+    set +e
+    retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check $SOUP_LOG for details."
+    set -e
+
+    echo ""
+    echo "Ensuring python modules for Salt are installed and patched."
+    salt-call state.apply salt.python3-influxdb -l info queue=True
+    echo ""
+
+    # Only regenerate osquery packages if Fleet is enabled
+    FLEET_MANAGER=$(lookup_pillar fleet_manager)
+    FLEET_NODE=$(lookup_pillar fleet_node)
+    if [[ "$FLEET_MANAGER" == "True" || "$FLEET_NODE" == "True" ]]; then
+      echo ""
+      echo "Regenerating Osquery Packages.... This will take several minutes."
+      salt-call state.apply fleet.event_gen-packages -l info queue=True
+      echo ""
+    fi
+
+    echo ""
+    echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
+    salt-call state.highstate -l info queue=True
+    echo ""
+    echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete."
+
+    echo ""
+    echo "Stopping Salt Master to remove ACL"
+    systemctl stop salt-master
+
+    masterunlock
+
+    echo ""
+    echo "Starting Salt Master service."
+    systemctl start salt-master
+
+    # Testing that salt-master is up by checking that is it connected to itself
+    set +e
+    retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check $SOUP_LOG for details."
+    set -e
+    
+    echo "Running a highstate. This could take several minutes."
+    salt-call state.highstate -l info queue=True
+    postupgrade_changes
+    [[ $is_airgap -eq 0 ]] && unmount_update
+    thehive_maint
+
+    if [[ $UPGRADESALT -eq 1 ]]; then
+      if [[ $is_airgap -eq 0 ]]; then
+        echo ""
+        echo "Cleaning repos on remote Security Onion nodes."
+        salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone and G@os:CentOS' cmd.run "yum clean all"
+        echo ""
+      fi
+    fi
+
+    check_sudoers
+
+    if [[ -n $lsl_msg ]]; then
+      case $lsl_msg in
+        'distributed')
+          echo "[INFO] The value of log_size_limit in any heavy node minion pillars may be incorrect."
+          echo " -> We recommend checking and adjusting the values as necessary."
+          echo " -> Minion pillar directory: /opt/so/saltstack/local/pillar/minions/"
+        ;;
+        'single-node')
+          # We can assume the lsl_details array has been set if lsl_msg has this value
+          echo "[WARNING] The value of log_size_limit (${lsl_details[0]}) does not match the recommended value of ${lsl_details[1]}."
+          echo " -> We recommend checking and adjusting the value as necessary."
+          echo " -> File: /opt/so/saltstack/local/pillar/minions/${lsl_details[2]}.sls"
+        ;;
+      esac
+    fi
+
+    NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l)
+
+    if [[ $NUM_MINIONS -gt 1 ]]; then
+
+      cat << EOF
     
   
   
@@ -889,10 +982,10 @@ For more information, please see https://docs.securityonion.net/en/2.3/soup.html
 
 EOF
 
+    fi
   fi
-fi
 
-echo "### soup has been served at `date` ###"
+  echo "### soup has been served at $(date) ###"
 }
 
 cat << EOF
@@ -907,6 +1000,7 @@ Press Enter to continue or Ctrl-C to cancel.
 
 EOF
 
-read input
+read -r input
 
 main "$@" | tee -a $SOUP_LOG
+  

salt/top.sls

@@ -38,7 +38,6 @@ base:
     - patch.os.schedule
     - motd
     - salt.minion-check
-    - sensoroni
     - salt.lasthighstate
   
   '*_helixsensor and G@saltversion:{{saltversion}}':
@@ -47,6 +46,7 @@ base:
     - ca
     - ssl
     - registry
+    - sensoroni
     - telegraf
     - firewall
     - idstools
@@ -66,6 +66,7 @@ base:
     - match: compound
     - ca
     - ssl
+    - sensoroni
     - telegraf
     - firewall
     - nginx
@@ -92,6 +93,7 @@ base:
     - ca
     - ssl
     - registry
+    - sensoroni
     - manager
     - nginx
     - telegraf
@@ -160,6 +162,7 @@ base:
     - ca
     - ssl
     - registry
+    - sensoroni
     - nginx
     - telegraf
     - influxdb
@@ -220,6 +223,7 @@ base:
     - ca
     - ssl
     - registry
+    - sensoroni
     - manager
     - nginx
     - telegraf
@@ -290,6 +294,7 @@ base:
     - match: compound
     - ca
     - ssl
+    - sensoroni
     - nginx
     - telegraf
     - firewall
@@ -320,6 +325,7 @@ base:
     - ca
     - ssl
     - registry
+    - sensoroni
     - nginx
     - telegraf
     - influxdb
@@ -382,6 +388,7 @@ base:
     - match: compound
     - ca
     - ssl
+    - sensoroni
     - nginx
     - telegraf
     - firewall
@@ -424,6 +431,7 @@ base:
     - match: compound
     - ca
     - ssl
+    - sensoroni
     - nginx
     - telegraf
     - firewall
@@ -441,6 +449,7 @@ base:
     - ca
     - ssl
     - registry
+    - sensoroni
     - manager
     - nginx
     - soc