rework autosoup + prompt for next ISO on airgap soup if required

salt/manager/tools/sbin/soup

@@ -93,6 +93,10 @@ check_err() {
       161)
         echo 'Required intermediate Elasticsearch upgrade not complete'
       ;;
+      170)
+        echo "Intermediate upgrade completed successfully to $next_step_so_version, but next soup to Security Onion $originally_requested_so_version could not be started automatically."
+        echo "Start soup again manually to continue the upgrade to Security Onion $originally_requested_so_version."
+      ;;
       *)
         echo 'Unhandled error'
         echo "$err_msg"
@@ -206,14 +210,18 @@ check_airgap() {
   else
     nonairgap_useiso=1
   fi
+
+  if [[ "$AIRGAP" == "true" ]]; then
+      is_airgap=0
+    else
+      is_airgap=1
+  fi
+
   # use ISO if its airgap install OR ISOLOC was set with -f <path>
   if [[ "$AIRGAP" == "true" ]] || [[ $nonairgap_useiso -eq 0 ]]; then
-      is_airgap=0
       UPDATE_DIR=/tmp/soagupdate/SecurityOnion
       AGDOCKER=/tmp/soagupdate/docker
       AGREPO=/tmp/soagupdate/minimal/Packages
-  else
-      is_airgap=1
   fi
 }
 
@@ -1689,7 +1697,7 @@ verify_latest_update_script() {
 
 verify_es_version_compatibility() {
 
-  local es_required_version_statefile="/opt/so/state/so_es_required_upgrade_version.txt"
+    es_required_version_statefile="/opt/so/state/so_es_required_upgrade_version.txt"
     local es_verification_script="/tmp/so_intermediate_upgrade_verification.sh"
     # supported upgrade paths for SO-ES versions
     declare -A es_upgrade_map=(
@@ -1710,6 +1718,7 @@ verify_es_version_compatibility() {
         es_version=$(echo "$es_version_raw" | jq -r '.version.number' )
     else
         echo "Could not determine current Elasticsearch version to validate compatibility with post soup Elasticsearch version."
+
         exit 160
     fi
 
@@ -1719,6 +1728,7 @@ verify_es_version_compatibility() {
         # if so-yaml.py failed to get the ES version AND the version we are upgrading to is newer than 2.4.110 then we should bail
         if [[ $(cat $UPDATE_DIR/VERSION | cut -d'.' -f3) > 110 ]]; then
             echo "Couldn't determine the target Elasticsearch version (post soup version) to ensure compatibility with current Elasticsearch version. Exiting"
+
             exit 160
         fi
 
@@ -1745,6 +1755,7 @@ verify_es_version_compatibility() {
             echo "A previous required intermediate Elasticsearch upgrade to $es_required_version_statefile_value has yet to successfully complete across the grid. Please allow time for all Searchnodes/Heavynodes to have upgraded Elasticsearch to $es_required_version_statefile_value before running soup again to avoid potential data loss!"
 
             echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
+
             exit 161
         fi
         echo -e "\n##############################################################################################################################\n"
@@ -1773,31 +1784,105 @@ verify_es_version_compatibility() {
         # We expect to upgrade to the latest compatiable minor version of ES
         create_intermediate_upgrade_verification_script $es_verification_script
 
-    if [[ $is_airgap -eq 0 ]]; then
+        if [[ $is_airgap -eq 0 ]] || [[ $nonairgap_useiso -eq 0 ]] ; then
-      echo "You can download the $next_step_so_version ISO image from https://download.securityonion.net/file/securityonion/securityonion-$next_step_so_version.iso"
+            run_airgap_intermediate_upgrade
-      echo "*** Once you have updated to $next_step_so_version, you can then run soup again to update to $(cat $UPDATE_DIR/VERSION). ***"
-      echo -e "\n##############################################################################################################################\n"
-      exit 160
         else
+            # Make sure ISOLOC is not set. Network installs that used soup -f would have ISOLOC set.
+            unset ISOLOC
+
+            run_network_intermediate_upgrade
+	    fi
+    fi
+
+}
+
+run_airgap_intermediate_upgrade() {
+    local originally_requested_so_version=$(cat $UPDATE_DIR/VERSION)
+    # preserve ISOLOC value, so we can try to use it post intermediate upgrade
+    local originally_requested_iso_location="$ISOLOC"
+
+    if [[ -d /tmp/soagupdate ]]; then
+        echo -e "\nUnmounting current ISO before running intermediate upgrade\n"
+        unmount_update
+    fi
+
+    echo "You can download the $next_step_so_version ISO image from https://download.securityonion.net/file/securityonion/securityonion-$next_step_so_version.iso"
+    echo -e "\nIf you have the next ISO / USB ready, enter the path now eg. /dev/sdd, /home/onion/securityonion-$next_step_so_version.iso:"
+
+    while [[ -z "$next_iso_location" ]] || [[ ! -f "$next_iso_location" && ! -b "$next_iso_location" ]]; do
+        # List removable devices if any are present
+        local removable_devices=$(lsblk -no PATH,SIZE,TYPE,MOUNTPOINTS,RM | awk '$NF==1')
+        if [[ -n "$removable_devices" ]]; then
+            echo "PATH        SIZE    TYPE    MOUNTPOINTS    RM"
+            echo "$removable_devices"
+        fi
+
+        read -rp "Device/ISO Path (or 'exit' to quit): " next_iso_location
+        if [[ "${next_iso_location,,}" == "exit" ]]; then
+            echo "Exiting soup. Before reattempting to upgrade to $originally_requested_so_version, please first upgrade to $next_step_so_version to ensure Elasticsearch can properly update through the required versions."
+
+            exit 160
+        fi
+
+        if [[ ! -f "$next_iso_location" && ! -b "$next_iso_location" ]]; then
+            echo "$next_iso_location is not a valid file or block device."
+            next_iso_location=""
+        fi
+    done
+
+    echo "Using $next_iso_location for required intermediary upgrade."
+    exec bash <<EOF
+        ISOLOC=$next_iso_location soup -y && \
+        ISOLOC=$next_iso_location soup -y && \
+
+        echo -e "\n##############################################################################################################################\n" && \
+        echo -e "Verifying Elasticsearch was successfully upgraded to $required_es_upgrade_version across the grid. This part can take a while as Searchnodes/Heavynodes sync up with the Manager! \n\nOnce verification completes the next soup will begin automatically. If verification takes longer than 1 hour it will stop waiting and your grid will remain at $next_step_so_version. Allowing for all Searchnodes/Heavynodes to upgrade Elasticsearch to the required version on their own time.\n" && \
+
+        timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh $required_es_upgrade_version $es_required_version_statefile && \
+
+        echo -e "\n##############################################################################################################################\n" && \
+
+        # automatically start the next soup if the original ISO isn't using the same block device we just used
+        if [[ -n "$originally_requested_iso_location" ]] && [[ "$originally_requested_iso_location" != "$next_iso_location" ]]; then
+            ISOLOC=$originally_requested_iso_location soup -y && \
+            ISOLOC=$originally_requested_iso_location soup -y
+        else
+            echo "Could not automatically start next soup to $originally_requested_so_version. Soup will now exit here at $(cat /etc/soversion)" && \
+
+            exit 170
+        fi
+
+        echo -e "\n##############################################################################################################################\n"
+EOF
+}
+
+run_network_intermediate_upgrade() {
     # preserve BRANCH value if set originally
     if [[ -n "$BRANCH" ]]; then
-        local originally_requested_so_version="$BRANCH"
+        local originally_requested_so_branch="$BRANCH"
     else
-        local originally_requested_so_version="2.4/main"
+        local originally_requested_so_branch="2.4/main"
     fi
 
     echo "Starting automated intermediate upgrade to $next_step_so_version."
     echo "After completion, the system will automatically attempt to upgrade to the latest version."
     echo -e "\n##############################################################################################################################\n"
-      exec bash -c "BRANCH=$next_step_so_version soup -y && BRANCH=$next_step_so_version soup -y && \
+    exec bash << EOF
-       echo -e \"\n##############################################################################################################################\n\" && \
+        BRANCH=$next_step_so_version soup -y && \
-       echo -e \"Verifying Elasticsearch was successfully upgraded to $required_es_upgrade_version across the grid. This part can take a while as Searchnodes/Heavynodes sync up with the Manager! \n\nOnce verification completes the next soup will begin automatically. If verification takes longer than 1 hour it will stop waiting and your grid will remain at $next_step_so_version. Allowing for all Searchnodes/Heavynodes to upgrade Elasticsearch to the required version on their own time.\n\" \
+        BRANCH=$next_step_so_version soup -y && \
-       && timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh $required_es_upgrade_version $es_required_version_statefile && \
-       echo -e \"\n##############################################################################################################################\n\" \
-       && BRANCH=$originally_requested_so_version soup -y && BRANCH=$originally_requested_so_version soup -y"
-    fi
-  fi
 
+        echo -e "\n##############################################################################################################################\n" && \
+        echo -e "Verifying Elasticsearch was successfully upgraded to $required_es_upgrade_version across the grid. This part can take a while as Searchnodes/Heavynodes sync up with the Manager! \n\nOnce verification completes the next soup will begin automatically. If verification takes longer than 1 hour it will stop waiting and your grid will remain at $next_step_so_version. Allowing for all Searchnodes/Heavynodes to upgrade Elasticsearch to the required version on their own time.\n" && \
+
+        timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh $required_es_upgrade_version $es_required_version_statefile && \
+
+        echo -e "\n##############################################################################################################################\n" && \
+
+        BRANCH=$originally_requested_so_branch soup -y && \
+        BRANCH=$originally_requested_so_branch soup -y
+
+        echo -e "\n##############################################################################################################################\n"
+EOF
 }
 
 create_intermediate_upgrade_verification_script() {
@@ -2023,6 +2108,7 @@ main() {
   echo "Verifying we have the latest soup script."
   verify_latest_update_script
 
+  echo "Verifying Elasticsearch version compatibility before upgrading."
   verify_es_version_compatibility
 
   echo "Let's see if we need to update Security Onion."