From 0388912ba7a75f47b69e6319f5aa856f6091a8c2 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Wed, 12 Jan 2022 11:05:47 -0500
Subject: [PATCH] kill all salt jobs across grid before stopping salt-master.
 kill all salt jobs on manager before stopping salt-minion.

---
 salt/common/tools/sbin/soup | 81 ++++++++++++++++++++-----------------
 1 file changed, 45 insertions(+), 36 deletions(-)

diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index e6e878163..607396195 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -468,6 +468,48 @@ post_to_2.3.100() {
   echo "Post Processing for .100"
 }
 
+stop_salt_master() {
+    # kill all salt jobs across the grid because the hang indefinitely if they are queued and salt-master restarts
+    set +e
+    echo ""
+    echo "Killing all Salt jobs across the grid."
+    salt \* saltutil.kill_all_jobs
+    set -e
+
+    echo ""
+    echo "Storing salt-master pid."
+    MASTERPID=$(pgrep salt-master | head -1)
+    echo "Found salt-master PID $MASTERPID"
+    echo ""
+    echo "Stopping Salt Master service at $(date +"%T.%6N")"
+    systemctl stop salt-master
+    echo ""
+    timeout 30 tail --pid=$MASTERPID -f /dev/null || echo "salt-master still running at $(date +"%T.%6N") after waiting 30s. We cannot kill due to systemd restart option."
+}
+
+stop_salt_minion() {
+    echo "Disabling highstate to prevent from running if salt-minion restarts."
+    salt-call state.disable highstate -l info --local
+    echo ""
+
+    # kill all salt jobs before stopping salt-minion
+    set +e
+    echo ""
+    echo "Killing Salt jobs on this node."
+    salt-call saltutil.kill_all_jobs --local
+    set -e
+
+    echo "Storing salt-minion pid."
+    MINIONPID=$(pgrep salt-minion | head -1)
+    echo "Found salt-minion PID $MINIONPID"
+    echo "Stopping Salt Minion service at $(date +"%T.%6N")."
+    systemctl stop salt-minion
+
+    set +e
+    timeout 30 tail --pid=$MINIONPID -f /dev/null || echo "Killing salt-minion at $(date +"%T.%6N") after waiting 30s" && pkill -9 -ef /usr/bin/salt-minion
+    set -e
+}
+
 up_to_2.3.20(){
   DOCKERSTUFFBIP=$(echo $DOCKERSTUFF | awk -F'.' '{print $1,$2,$3,1}' OFS='.')/24
   # Remove PCAP from global
@@ -992,34 +1034,9 @@ main() {
       set -e
     fi
 
-    echo "Disabling highstate to prevent from running if salt-minion restarts."
-    salt-call state.disable highstate -l info --local
-    echo ""
+    stop_salt_minion
 
-    echo "Storing salt-minion pid."
-    MINIONPID=$(pgrep salt-minion | head -1)
-    echo "Found salt-minion PID $MINIONPID"
-    echo "Stopping Salt Minion service at $(date +"%T.%6N")."
-    systemctl stop salt-minion
-
-    set +e
-    timeout 30 tail --pid=$MINIONPID -f /dev/null || echo "Killing salt-minion at $(date +"%T.%6N") after waiting 30s" && pkill -9 -ef /usr/bin/salt-minion
-    set -e
-
-    #echo "Killing any remaining Salt Minion processes at $(date +"%T.%6N")"
-    #set +e
-    #pkill -9 -ef /usr/bin/salt-minion
-    #set -e
-
-    echo ""
-    echo "Storing salt-master pid."
-    MASTERPID=$(pgrep salt-master | head -1)
-    echo "Found salt-master PID $MASTERPID"
-    echo ""
-    echo "Stopping Salt Master service at $(date +"%T.%6N")"
-    systemctl stop salt-master
-    echo ""
-    timeout 30 tail --pid=$MASTERPID -f /dev/null || echo "salt-master still running at $(date +"%T.%6N") after waiting 30s. We cannot kill due to systemd restart option."
+    stop_salt_master
 
     upgrade_to_2.3.50_repo
 
@@ -1110,15 +1127,7 @@ main() {
     salt-call state.highstate -l info queue=True
     set -e
 
-    echo ""
-    echo "Storing salt-master pid."
-    MASTERPID=$(pgrep salt-master | head -1)
-    echo "Found salt-master PID $MASTERPID"
-    echo ""
-    echo "Stopping Salt Master service to remove ACL(masterunlock) at $(date +"%T.%6N")"
-    systemctl stop salt-master
-    echo ""
-    timeout 30 tail --pid=$MASTERPID -f /dev/null || echo "salt-master still running at $(date +"%T.%6N") after waiting 30s. We cannot kill due to systemd restart option."
+    stop_salt_master
 
     masterunlock