From 6e7a5fa3263db973ff1f33bc4a4c68d2722da0c4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 13 Oct 2021 09:45:15 -0400 Subject: [PATCH 1/6] add timeouts to check_salt_minion_status and check_salt_master_status - https://github.com/Security-Onion-Solutions/securityonion/issues/5818 --- setup/so-functions | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index f7d489f42..039d8a602 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -229,9 +229,10 @@ check_service_status() { } check_salt_master_status() { + local timeout=$1 echo "Checking if we can talk to the salt master" >> "$setup_log" 2>&1 salt-call saltutil.kill_all_jobs > /dev/null 2>&1 - salt-call state.show_top > /dev/null 2>&1 + salt-call state.show_top -t $timeout > /dev/null 2>&1 local status=$? if [ $status -gt 0 ]; then echo " Could not talk to salt master" >> "$setup_log" 2>&1 @@ -244,8 +245,9 @@ check_salt_master_status() { } check_salt_minion_status() { + local timeout=$1 echo "Checking if the salt minion will respond to jobs" >> "$setup_log" 2>&1 - salt "$MINION_ID" test.ping > /dev/null 2>&1 + salt "$MINION_ID" test.ping -t $timeout > /dev/null 2>&1 local status=$? if [ $status -gt 0 ]; then echo " Minion did not respond" >> "$setup_log" 2>&1 @@ -2293,27 +2295,31 @@ salt_checkin() { done count=0 - while ! (check_salt_master_status); do - echo "salt minion cannot talk to salt master" >> "$setup_log" 2>&1 - if [ $count -gt 30 ]; then - echo "salt minion could not talk to salt master after 30 attempts, exiting" >> "$setup_log" 2>&1 + timeout=60 + while ! (check_salt_master_status $timeout); do + echo "salt minion cannot talk to salt master after $timeout seconds" >> "$setup_log" 2>&1 + if [ $count -gt 3 ]; then + echo "salt minion could not talk to salt master after 3 attempts, exiting" >> "$setup_log" 2>&1 exit 1 fi sleep 1; ((count++)) + timeout=$(( $timeout + (20 * $count) )) # add 20s to the timeout each attempt done count=0 - while ! (check_salt_minion_status); do - echo "salt master did not get a job response from salt minion" >> "$setup_log" 2>&1 - if [ $count -gt 30 ]; then - echo "salt master did not get a job response from salt minion after 30 attempts, exiting" >> "$setup_log" 2>&1 + timeout=60 + while ! (check_salt_minion_status $timeout ; do + echo "salt master did not get a job response from salt minion after $timeout seconds" >> "$setup_log" 2>&1 + if [ $count -gt 3 ]; then + echo "salt master did not get a job response from salt minion after 3 attempts, exiting" >> "$setup_log" 2>&1 exit 1 fi systemctl kill salt-minion systemctl start salt-minion sleep 1; ((count++)) + timeout=$(( $timeout + (20 * $count) )) # add 20s to the timeout each attempt done echo " Confirming existence of the CA certificate" From adb8292814b4769416345291b613f89ec7f840aa Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 13 Oct 2021 10:37:18 -0400 Subject: [PATCH 2/6] add missing ) --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 039d8a602..0402f4ec8 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2309,7 +2309,7 @@ salt_checkin() { count=0 timeout=60 - while ! (check_salt_minion_status $timeout ; do + while ! (check_salt_minion_status $timeout) ; do echo "salt master did not get a job response from salt minion after $timeout seconds" >> "$setup_log" 2>&1 if [ $count -gt 3 ]; then echo "salt master did not get a job response from salt minion after 3 attempts, exiting" >> "$setup_log" 2>&1 From 7a4fa8879c611c608f5d3710dbc3e4c500502f89 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 13 Oct 2021 12:13:24 -0400 Subject: [PATCH 3/6] change count, attempts and timeout --- setup/so-functions | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 0402f4ec8..84403eac2 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2294,32 +2294,32 @@ salt_checkin() { done done - count=0 + count=1 timeout=60 while ! (check_salt_master_status $timeout); do echo "salt minion cannot talk to salt master after $timeout seconds" >> "$setup_log" 2>&1 if [ $count -gt 3 ]; then - echo "salt minion could not talk to salt master after 3 attempts, exiting" >> "$setup_log" 2>&1 + echo "salt minion could not talk to salt master after $count attempts, exiting" >> "$setup_log" 2>&1 exit 1 fi sleep 1; ((count++)) - timeout=$(( $timeout + (20 * $count) )) # add 20s to the timeout each attempt + (($timeout += 20)) # add 20s to the timeout each attempt done - count=0 + count=1 timeout=60 while ! (check_salt_minion_status $timeout) ; do echo "salt master did not get a job response from salt minion after $timeout seconds" >> "$setup_log" 2>&1 if [ $count -gt 3 ]; then - echo "salt master did not get a job response from salt minion after 3 attempts, exiting" >> "$setup_log" 2>&1 + echo "salt master did not get a job response from salt minion after $count attempts, exiting" >> "$setup_log" 2>&1 exit 1 fi systemctl kill salt-minion systemctl start salt-minion sleep 1; ((count++)) - timeout=$(( $timeout + (20 * $count) )) # add 20s to the timeout each attempt + (($timeout += 20)) # add 20s to the timeout each attempt done echo " Confirming existence of the CA certificate" From 880c1b97b08577be5df751ad07001857d53ee59a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 13 Oct 2021 12:25:11 -0400 Subject: [PATCH 4/6] remove $ from var --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 84403eac2..dfb935ff8 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2304,7 +2304,7 @@ salt_checkin() { fi sleep 1; ((count++)) - (($timeout += 20)) # add 20s to the timeout each attempt + ((timeout+=20)) # add 20s to the timeout each attempt done count=1 @@ -2319,7 +2319,7 @@ salt_checkin() { systemctl start salt-minion sleep 1; ((count++)) - (($timeout += 20)) # add 20s to the timeout each attempt + ((timeout+=20)) # add 20s to the timeout each attempt done echo " Confirming existence of the CA certificate" From a4957795520dc9ca26c57e63c724a5102020f00d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 13 Oct 2021 12:34:56 -0400 Subject: [PATCH 5/6] only 3 attempts with 120s max attemps --- setup/so-functions | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index dfb935ff8..13ed6cda3 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2298,20 +2298,20 @@ salt_checkin() { timeout=60 while ! (check_salt_master_status $timeout); do echo "salt minion cannot talk to salt master after $timeout seconds" >> "$setup_log" 2>&1 - if [ $count -gt 3 ]; then + if [ $count -gt 2 ]; then echo "salt minion could not talk to salt master after $count attempts, exiting" >> "$setup_log" 2>&1 exit 1 fi sleep 1; ((count++)) - ((timeout+=20)) # add 20s to the timeout each attempt + ((timeout+=30)) # add 20s to the timeout each attempt done count=1 timeout=60 while ! (check_salt_minion_status $timeout) ; do echo "salt master did not get a job response from salt minion after $timeout seconds" >> "$setup_log" 2>&1 - if [ $count -gt 3 ]; then + if [ $count -gt 2 ]; then echo "salt master did not get a job response from salt minion after $count attempts, exiting" >> "$setup_log" 2>&1 exit 1 fi @@ -2319,7 +2319,7 @@ salt_checkin() { systemctl start salt-minion sleep 1; ((count++)) - ((timeout+=20)) # add 20s to the timeout each attempt + ((timeout+=30)) # add 20s to the timeout each attempt done echo " Confirming existence of the CA certificate" From 3044edb104a943257a74d4d86a8de46ebae373a8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 13 Oct 2021 12:38:58 -0400 Subject: [PATCH 6/6] update comment --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 13ed6cda3..4398cfbcc 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2304,7 +2304,7 @@ salt_checkin() { fi sleep 1; ((count++)) - ((timeout+=30)) # add 20s to the timeout each attempt + ((timeout+=30)) # add 30s to the timeout each attempt done count=1 @@ -2319,7 +2319,7 @@ salt_checkin() { systemctl start salt-minion sleep 1; ((count++)) - ((timeout+=30)) # add 20s to the timeout each attempt + ((timeout+=30)) # add 30s to the timeout each attempt done echo " Confirming existence of the CA certificate"