add timeouts to check_salt_minion_status and check_salt_master_status - https://github.com/Security-Onion-Solutions/securityonion/issues/5818

This commit is contained in:
m0duspwnens
2021-10-13 09:45:15 -04:00
parent 2561480371
commit 6e7a5fa326

View File

@@ -229,9 +229,10 @@ check_service_status() {
} }
check_salt_master_status() { check_salt_master_status() {
local timeout=$1
echo "Checking if we can talk to the salt master" >> "$setup_log" 2>&1 echo "Checking if we can talk to the salt master" >> "$setup_log" 2>&1
salt-call saltutil.kill_all_jobs > /dev/null 2>&1 salt-call saltutil.kill_all_jobs > /dev/null 2>&1
salt-call state.show_top > /dev/null 2>&1 salt-call state.show_top -t $timeout > /dev/null 2>&1
local status=$? local status=$?
if [ $status -gt 0 ]; then if [ $status -gt 0 ]; then
echo " Could not talk to salt master" >> "$setup_log" 2>&1 echo " Could not talk to salt master" >> "$setup_log" 2>&1
@@ -244,8 +245,9 @@ check_salt_master_status() {
} }
check_salt_minion_status() { check_salt_minion_status() {
local timeout=$1
echo "Checking if the salt minion will respond to jobs" >> "$setup_log" 2>&1 echo "Checking if the salt minion will respond to jobs" >> "$setup_log" 2>&1
salt "$MINION_ID" test.ping > /dev/null 2>&1 salt "$MINION_ID" test.ping -t $timeout > /dev/null 2>&1
local status=$? local status=$?
if [ $status -gt 0 ]; then if [ $status -gt 0 ]; then
echo " Minion did not respond" >> "$setup_log" 2>&1 echo " Minion did not respond" >> "$setup_log" 2>&1
@@ -2293,27 +2295,31 @@ salt_checkin() {
done done
count=0 count=0
while ! (check_salt_master_status); do timeout=60
echo "salt minion cannot talk to salt master" >> "$setup_log" 2>&1 while ! (check_salt_master_status $timeout); do
if [ $count -gt 30 ]; then echo "salt minion cannot talk to salt master after $timeout seconds" >> "$setup_log" 2>&1
echo "salt minion could not talk to salt master after 30 attempts, exiting" >> "$setup_log" 2>&1 if [ $count -gt 3 ]; then
echo "salt minion could not talk to salt master after 3 attempts, exiting" >> "$setup_log" 2>&1
exit 1 exit 1
fi fi
sleep 1; sleep 1;
((count++)) ((count++))
timeout=$(( $timeout + (20 * $count) )) # add 20s to the timeout each attempt
done done
count=0 count=0
while ! (check_salt_minion_status); do timeout=60
echo "salt master did not get a job response from salt minion" >> "$setup_log" 2>&1 while ! (check_salt_minion_status $timeout ; do
if [ $count -gt 30 ]; then echo "salt master did not get a job response from salt minion after $timeout seconds" >> "$setup_log" 2>&1
echo "salt master did not get a job response from salt minion after 30 attempts, exiting" >> "$setup_log" 2>&1 if [ $count -gt 3 ]; then
echo "salt master did not get a job response from salt minion after 3 attempts, exiting" >> "$setup_log" 2>&1
exit 1 exit 1
fi fi
systemctl kill salt-minion systemctl kill salt-minion
systemctl start salt-minion systemctl start salt-minion
sleep 1; sleep 1;
((count++)) ((count++))
timeout=$(( $timeout + (20 * $count) )) # add 20s to the timeout each attempt
done done
echo " Confirming existence of the CA certificate" echo " Confirming existence of the CA certificate"