kill all salt jobs across grid before stopping salt-master. kill all salt jobs on manager before stopping salt-minion.

This commit is contained in:
m0duspwnens
2022-01-12 11:05:47 -05:00
parent 494737549d
commit 0388912ba7

View File

@@ -468,6 +468,48 @@ post_to_2.3.100() {
echo "Post Processing for .100" echo "Post Processing for .100"
} }
stop_salt_master() {
# kill all salt jobs across the grid because the hang indefinitely if they are queued and salt-master restarts
set +e
echo ""
echo "Killing all Salt jobs across the grid."
salt \* saltutil.kill_all_jobs
set -e
echo ""
echo "Storing salt-master pid."
MASTERPID=$(pgrep salt-master | head -1)
echo "Found salt-master PID $MASTERPID"
echo ""
echo "Stopping Salt Master service at $(date +"%T.%6N")"
systemctl stop salt-master
echo ""
timeout 30 tail --pid=$MASTERPID -f /dev/null || echo "salt-master still running at $(date +"%T.%6N") after waiting 30s. We cannot kill due to systemd restart option."
}
stop_salt_minion() {
echo "Disabling highstate to prevent from running if salt-minion restarts."
salt-call state.disable highstate -l info --local
echo ""
# kill all salt jobs before stopping salt-minion
set +e
echo ""
echo "Killing Salt jobs on this node."
salt-call saltutil.kill_all_jobs --local
set -e
echo "Storing salt-minion pid."
MINIONPID=$(pgrep salt-minion | head -1)
echo "Found salt-minion PID $MINIONPID"
echo "Stopping Salt Minion service at $(date +"%T.%6N")."
systemctl stop salt-minion
set +e
timeout 30 tail --pid=$MINIONPID -f /dev/null || echo "Killing salt-minion at $(date +"%T.%6N") after waiting 30s" && pkill -9 -ef /usr/bin/salt-minion
set -e
}
up_to_2.3.20(){ up_to_2.3.20(){
DOCKERSTUFFBIP=$(echo $DOCKERSTUFF | awk -F'.' '{print $1,$2,$3,1}' OFS='.')/24 DOCKERSTUFFBIP=$(echo $DOCKERSTUFF | awk -F'.' '{print $1,$2,$3,1}' OFS='.')/24
# Remove PCAP from global # Remove PCAP from global
@@ -992,34 +1034,9 @@ main() {
set -e set -e
fi fi
echo "Disabling highstate to prevent from running if salt-minion restarts." stop_salt_minion
salt-call state.disable highstate -l info --local
echo ""
echo "Storing salt-minion pid." stop_salt_master
MINIONPID=$(pgrep salt-minion | head -1)
echo "Found salt-minion PID $MINIONPID"
echo "Stopping Salt Minion service at $(date +"%T.%6N")."
systemctl stop salt-minion
set +e
timeout 30 tail --pid=$MINIONPID -f /dev/null || echo "Killing salt-minion at $(date +"%T.%6N") after waiting 30s" && pkill -9 -ef /usr/bin/salt-minion
set -e
#echo "Killing any remaining Salt Minion processes at $(date +"%T.%6N")"
#set +e
#pkill -9 -ef /usr/bin/salt-minion
#set -e
echo ""
echo "Storing salt-master pid."
MASTERPID=$(pgrep salt-master | head -1)
echo "Found salt-master PID $MASTERPID"
echo ""
echo "Stopping Salt Master service at $(date +"%T.%6N")"
systemctl stop salt-master
echo ""
timeout 30 tail --pid=$MASTERPID -f /dev/null || echo "salt-master still running at $(date +"%T.%6N") after waiting 30s. We cannot kill due to systemd restart option."
upgrade_to_2.3.50_repo upgrade_to_2.3.50_repo
@@ -1110,15 +1127,7 @@ main() {
salt-call state.highstate -l info queue=True salt-call state.highstate -l info queue=True
set -e set -e
echo "" stop_salt_master
echo "Storing salt-master pid."
MASTERPID=$(pgrep salt-master | head -1)
echo "Found salt-master PID $MASTERPID"
echo ""
echo "Stopping Salt Master service to remove ACL(masterunlock) at $(date +"%T.%6N")"
systemctl stop salt-master
echo ""
timeout 30 tail --pid=$MASTERPID -f /dev/null || echo "salt-master still running at $(date +"%T.%6N") after waiting 30s. We cannot kill due to systemd restart option."
masterunlock masterunlock