soup debug wip

salt/manager/tools/sbin/soup

@@ -16,6 +16,7 @@ POSTVERSION=$INSTALLEDVERSION
 INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
 BATCHSIZE=5
 SOUP_LOG=/root/soup.log
+SOUP_DEBUG_LOG=/root/soup-debug.log
 WHATWOULDYOUSAYYAHDOHERE=soup
 whiptail_title='Security Onion UPdater'
 NOTIFYCUSTOMELASTICCONFIG=false
@@ -108,6 +109,7 @@ check_err() {
       *)
         echo 'Unhandled error'
         echo "$err_msg"
+        echo "Debug log (if enabled) at $SOUP_DEBUG_LOG. Re-run soup with SOUP_DEBUG=1 to create $SOUP_DEBUG_LOG"
       ;;
     esac
     if [[ $exit_code -ge 64 && $exit_code -le 113 ]]; then
@@ -119,6 +121,39 @@ check_err() {
 
 }
 
+# on_err captures context when a command fails under `set -e`. It is always
+# installed as an ERR trap. It writes a concise one-line summary to stdout so it
+# flows through `tee` into soup.log + terminal. When SOUP_DEBUG=1, it also writes
+# a multi-line stack trace to the xtrace FD ($SOUP_XTRACE_FD -> soup-debug.log).
+# It deliberately does NOT exit so the existing EXIT trap (check_err $?) keeps
+# ownership of termination behavior. Writes are guarded with `|| true` to
+# avoid re-entrancy in pathological conditions (e.g. disk full).
+on_err() {
+  local exit_code=$?
+
+  local cmd=$BASH_COMMAND
+  local line=${BASH_LINENO[0]}
+  local func=${FUNCNAME[1]:-main}
+  local src=${BASH_SOURCE[1]##*/}
+
+  echo "ERROR at ${src}:${line} in ${func}(): '${cmd}' exited with code ${exit_code}" || true
+
+  # Dump a stack trace to the xtrace FD if it is open.
+  if [[ -n $SOUP_XTRACE_FD ]]; then
+    {
+      echo "=== ERR trap $(date '+%F %T.%6N') ==="
+      echo "Exit: $exit_code  Command: $cmd"
+      local i
+      for ((i=0; i<${#FUNCNAME[@]}; i++)); do
+        echo "  at ${BASH_SOURCE[$i]##*/}:${BASH_LINENO[$i]} in ${FUNCNAME[$i]}()"
+      done
+      echo "=== end ERR trap ==="
+    } >&"$SOUP_XTRACE_FD" || true
+  fi
+
+  return $exit_code
+}
+
 airgap_mounted() {
   # Let's see if the ISO is already mounted.
   if [[ -f /tmp/soagupdate/SecurityOnion/VERSION ]]; then
@@ -378,6 +413,15 @@ postupgrade_changes() {
     [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0
     true
 }
+make_an_error() {
+    so-elasticsearch-query _index_templates/.kibana-streams --fail
+}
+
+postupgrade_changes_fail() {
+
+    make_an_error
+
+}
 
 check_minimum_version() {
   if [[ ! "$INSTALLEDVERSION" =~ ^(2\.4\.21[0-9]+|3\.) ]]; then
@@ -1697,6 +1741,9 @@ main() {
   upgrade_check_salt
   set -e
 
+    #force quick soup
+    postupgrade_changes_fail
+
   if [[ $is_airgap -eq 0 ]]; then
     update_airgap_repo
     dnf clean all
@@ -1982,4 +2029,29 @@ EOF
   read -r input
 fi
 
+
+# Error context is enabled for all soup runs. Verbose developer-only debug
+# tracing remains opt-in via SOUP_DEBUG=1. When debug is enabled:
+#   - Rotates any existing soup-debug.log from a prior SOUP_DEBUG=1 run.
+#   - Opens an auto-assigned FD to soup-debug.log and points BASH_XTRACEFD at it
+#     so `set -x` output goes ONLY to soup-debug.log (not to terminal or soup.log).
+#   - Exports SOUP_DEBUG so child invocations via `exec bash <<EOF` in the
+#     intermediate-upgrade paths inherit debug mode (each child rotates its
+#     own soup-debug.log).
+set -o errtrace
+trap on_err ERR
+
+if [[ $SOUP_DEBUG == 1 ]]; then
+  if [ -f $SOUP_DEBUG_LOG ]; then
+    SOUP_DEBUG_ROTATE_TIME=$(date +%Y%m%d.%H%M%S)
+    mv $SOUP_DEBUG_LOG $SOUP_DEBUG_LOG.$INSTALLEDVERSION.$SOUP_DEBUG_ROTATE_TIME
+  fi
+  exec {SOUP_XTRACE_FD}>>"$SOUP_DEBUG_LOG"
+  export SOUP_XTRACE_FD
+  BASH_XTRACEFD=$SOUP_XTRACE_FD
+  PS4='+ ${BASH_SOURCE##*/}:${LINENO} ${FUNCNAME[0]:-main}() | '
+  set -x
+  export SOUP_DEBUG
+fi
+
 main "$@" | tee -a $SOUP_LOG