From 07ef464375de8ef3ba0ad1320ba51be9f8aac288 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 13 Aug 2020 16:01:53 -0400 Subject: [PATCH 01/21] https://github.com/Security-Onion-Solutions/securityonion/issues/1170 --- setup/so-functions | 79 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 75 insertions(+), 4 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 4f9d4938e..1ed19006f 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1370,6 +1370,34 @@ saltify() { } +check_service_status() { + + local service_name=$1 + systemctl status $service_name > /dev/null 2>&1 + local service_status=$? + if [ $service_status -gt 0 ]; then + service_status=1 + else + service_status=0 + fi + + return $service_status + +} + +check_salt_master_status() { + salt-call state.show_top >> "$setup_log" 2>&1 + local exit_code=$? + if [ $exit_code -gt 0 ]; then + exit_code=1 + else + exit_code=0 + fi + + return $exit_code + +} + salt_checkin() { case "$install_type" in @@ -1378,10 +1406,53 @@ salt_checkin() { echo "Building Certificate Authority"; salt-call state.apply ca; echo " *** Restarting Salt to fix any SSL errors. ***"; - systemctl restart salt-master; - sleep 5; - systemctl restart salt-minion; - sleep 15; + + local SALT_SERVICES=(\ + "salt-minion" \ + "salt-master" + ) + local LOOP_COUNT=0 + for service in "${SALT_SERVICES[@]}"; do + systemctl stop "$service"; + LOOP_COUNT=0 + while check_service_status "$service"; do + echo "$service still running" >> "$setup_log" 2>&1 + if [ LOOP_COUNT -gt 120 ]; then + echo "$service could not be stopped in 120 seconds" >> "$setup_log" 2>&1 + whiptail_setup_failed() + exit 1; + fi + sleep 1; + ((LOOP_COUNT+=1)) + done + + systemctl start "$service"; + LOOP_COUNT=0 + while ! check_service_status "$service"; do + echo "$service still not running" >> "$setup_log" 2>&1 + if [ LOOP_COUNT -gt 120 ]; then + echo "$service could not be started in 120 seconds" >> "$setup_log" 2>&1 + whiptail_setup_failed() + exit 1; + fi + sleep 1; + ((LOOP_COUNT+=1)) + done + + done + + LOOP_COUNT=0 + while check_salt_master_status; do + echo "salt-minion cannot talk to salt-master" >> "$setup_log" 2>&1 + if [ LOOP_COUNT -gt 20 ]; then + echo "salt-minion could not talk to salt-master after 20 attempts" >> "$setup_log" 2>&1 + whiptail_setup_failed() + exit 1; + fi + sleep 1; + ((LOOP_COUNT+=1)) + done + echo " Confirming existence of the CA certificate" cat /etc/pki/ca.crt echo " Applyng a mine hack"; From 1b4029f74b37280b504501a156caff7d13095562 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 13 Aug 2020 16:18:02 -0400 Subject: [PATCH 02/21] fix syntax errors --- setup/so-functions | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 1ed19006f..c7c296fd3 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1417,10 +1417,10 @@ salt_checkin() { LOOP_COUNT=0 while check_service_status "$service"; do echo "$service still running" >> "$setup_log" 2>&1 - if [ LOOP_COUNT -gt 120 ]; then + if [ $LOOP_COUNT -gt 120 ]; then echo "$service could not be stopped in 120 seconds" >> "$setup_log" 2>&1 - whiptail_setup_failed() - exit 1; + whiptail_setup_failed + exit 1 fi sleep 1; ((LOOP_COUNT+=1)) @@ -1430,10 +1430,10 @@ salt_checkin() { LOOP_COUNT=0 while ! check_service_status "$service"; do echo "$service still not running" >> "$setup_log" 2>&1 - if [ LOOP_COUNT -gt 120 ]; then + if [ $LOOP_COUNT -gt 120 ]; then echo "$service could not be started in 120 seconds" >> "$setup_log" 2>&1 - whiptail_setup_failed() - exit 1; + whiptail_setup_failed + exit 1 fi sleep 1; ((LOOP_COUNT+=1)) @@ -1444,10 +1444,10 @@ salt_checkin() { LOOP_COUNT=0 while check_salt_master_status; do echo "salt-minion cannot talk to salt-master" >> "$setup_log" 2>&1 - if [ LOOP_COUNT -gt 20 ]; then + if [ $LOOP_COUNT -gt 20 ]; then echo "salt-minion could not talk to salt-master after 20 attempts" >> "$setup_log" 2>&1 - whiptail_setup_failed() - exit 1; + whiptail_setup_failed + exit 1 fi sleep 1; ((LOOP_COUNT+=1)) From 3d20cc03412841498aa5db3729b09d290d78e8da Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 13 Aug 2020 16:34:18 -0400 Subject: [PATCH 03/21] some debugging --- setup/so-functions | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index c7c296fd3..b93e556f2 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1386,6 +1386,7 @@ check_service_status() { } check_salt_master_status() { + echo "Checking salt-master status" >> "$setup_log" 2>&1 salt-call state.show_top >> "$setup_log" 2>&1 local exit_code=$? if [ $exit_code -gt 0 ]; then @@ -1394,6 +1395,7 @@ check_salt_master_status() { exit_code=0 fi + echo "$exit_code" >> "$setup_log" 2>&1 return $exit_code } @@ -1413,26 +1415,26 @@ salt_checkin() { ) local LOOP_COUNT=0 for service in "${SALT_SERVICES[@]}"; do - systemctl stop "$service"; + systemctl stop "$service" >> "$setup_log" 2>&1 LOOP_COUNT=0 while check_service_status "$service"; do echo "$service still running" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 120 ]; then echo "$service could not be stopped in 120 seconds" >> "$setup_log" 2>&1 - whiptail_setup_failed + #whiptail_setup_failed exit 1 fi sleep 1; ((LOOP_COUNT+=1)) done - systemctl start "$service"; + systemctl start "$service" >> "$setup_log" 2>&1 LOOP_COUNT=0 while ! check_service_status "$service"; do echo "$service still not running" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 120 ]; then echo "$service could not be started in 120 seconds" >> "$setup_log" 2>&1 - whiptail_setup_failed + #whiptail_setup_failed exit 1 fi sleep 1; @@ -1444,9 +1446,9 @@ salt_checkin() { LOOP_COUNT=0 while check_salt_master_status; do echo "salt-minion cannot talk to salt-master" >> "$setup_log" 2>&1 - if [ $LOOP_COUNT -gt 20 ]; then - echo "salt-minion could not talk to salt-master after 20 attempts" >> "$setup_log" 2>&1 - whiptail_setup_failed + if [ $LOOP_COUNT -gt 120 ]; then + echo "salt-minion could not talk to salt-master after 120 attempts" >> "$setup_log" 2>&1 + #whiptail_setup_failed exit 1 fi sleep 1; From 6cf623e133948105ff444974966fd14c9dddc822 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 13 Aug 2020 16:52:39 -0400 Subject: [PATCH 04/21] some logic changes --- setup/so-functions | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index b93e556f2..5f6be3f8e 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1374,29 +1374,31 @@ check_service_status() { local service_name=$1 systemctl status $service_name > /dev/null 2>&1 - local service_status=$? + local status=$? + #true service is running false if not if [ $service_status -gt 0 ]; then - service_status=1 + status=false else - service_status=0 + status=true fi - return $service_status + return $status } check_salt_master_status() { echo "Checking salt-master status" >> "$setup_log" 2>&1 salt-call state.show_top >> "$setup_log" 2>&1 - local exit_code=$? + local status=$? + #true if we can talk to salt master false if not if [ $exit_code -gt 0 ]; then - exit_code=1 + status=false else - exit_code=0 + status=true fi - echo "$exit_code" >> "$setup_log" 2>&1 - return $exit_code + echo "$status" >> "$setup_log" 2>&1 + return $status } @@ -1421,7 +1423,6 @@ salt_checkin() { echo "$service still running" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 120 ]; then echo "$service could not be stopped in 120 seconds" >> "$setup_log" 2>&1 - #whiptail_setup_failed exit 1 fi sleep 1; @@ -1434,7 +1435,6 @@ salt_checkin() { echo "$service still not running" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 120 ]; then echo "$service could not be started in 120 seconds" >> "$setup_log" 2>&1 - #whiptail_setup_failed exit 1 fi sleep 1; @@ -1444,11 +1444,10 @@ salt_checkin() { done LOOP_COUNT=0 - while check_salt_master_status; do + while ! check_salt_master_status; do echo "salt-minion cannot talk to salt-master" >> "$setup_log" 2>&1 - if [ $LOOP_COUNT -gt 120 ]; then - echo "salt-minion could not talk to salt-master after 120 attempts" >> "$setup_log" 2>&1 - #whiptail_setup_failed + if [ $LOOP_COUNT -gt 40 ]; then + echo "salt-minion could not talk to salt-master after 40 attempts" >> "$setup_log" 2>&1 exit 1 fi sleep 1; From 829490da19f35cff0aaf9402d255cc18b9bc8568 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 13 Aug 2020 17:05:50 -0400 Subject: [PATCH 05/21] fix errors --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 5f6be3f8e..7537ceaa7 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1376,7 +1376,7 @@ check_service_status() { systemctl status $service_name > /dev/null 2>&1 local status=$? #true service is running false if not - if [ $service_status -gt 0 ]; then + if [ $status -gt 0 ]; then status=false else status=true @@ -1391,7 +1391,7 @@ check_salt_master_status() { salt-call state.show_top >> "$setup_log" 2>&1 local status=$? #true if we can talk to salt master false if not - if [ $exit_code -gt 0 ]; then + if [ $status -gt 0 ]; then status=false else status=true From f9f2744d3f2e043ebd501cfaf5cd1385f8e81242 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 13 Aug 2020 17:49:05 -0400 Subject: [PATCH 06/21] logic changes --- setup/so-functions | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 7537ceaa7..480c86604 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1377,9 +1377,9 @@ check_service_status() { local status=$? #true service is running false if not if [ $status -gt 0 ]; then - status=false + status=1 else - status=true + status=0 fi return $status @@ -1390,11 +1390,11 @@ check_salt_master_status() { echo "Checking salt-master status" >> "$setup_log" 2>&1 salt-call state.show_top >> "$setup_log" 2>&1 local status=$? - #true if we can talk to salt master false if not + #true if there is an issue talking to salt master if [ $status -gt 0 ]; then - status=false + status=1 else - status=true + status=0 fi echo "$status" >> "$setup_log" 2>&1 From 42c1e817fedb89da8add5cd4b83706474f4a7cb1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 13 Aug 2020 18:09:57 -0400 Subject: [PATCH 07/21] more logging and debugging --- setup/so-functions | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 480c86604..162c0e82b 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1373,7 +1373,8 @@ saltify() { check_service_status() { local service_name=$1 - systemctl status $service_name > /dev/null 2>&1 + echo "Checking service $service_name status" >> "$setup_log" 2>&1 + systemctl status $service_name >> "$setup_log" 2>&1 local status=$? #true service is running false if not if [ $status -gt 0 ]; then @@ -1454,6 +1455,9 @@ salt_checkin() { ((LOOP_COUNT+=1)) done + systemctl status salt-master; + systemctl status salt-minion; + echo " Confirming existence of the CA certificate" cat /etc/pki/ca.crt echo " Applyng a mine hack"; From 0eb0551b68968ca36711de64c3e3ef91b6e3d63a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 14 Aug 2020 10:15:54 -0400 Subject: [PATCH 08/21] add check if salt minion is returning jobs --- setup/so-functions | 51 +++++++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 162c0e82b..14c0fd671 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1388,7 +1388,7 @@ check_service_status() { } check_salt_master_status() { - echo "Checking salt-master status" >> "$setup_log" 2>&1 + echo "Checking if we can talk to the salt master" >> "$setup_log" 2>&1 salt-call state.show_top >> "$setup_log" 2>&1 local status=$? #true if there is an issue talking to salt master @@ -1398,11 +1398,26 @@ check_salt_master_status() { status=0 fi - echo "$status" >> "$setup_log" 2>&1 return $status } +check_salt_minion_status() { + echo "Checking if the salt minion will respond to jobs" >> "$setup_log" 2>&1 + salt "$MINION_ID" test.ping >> "$setup_log" 2>&1 + local status=$? + #true if there is an issue getting a job response from the minion + if [ $status -gt 0 ]; then + status=1 + else + status=0 + fi + + return $status + +} + + salt_checkin() { case "$install_type" in @@ -1422,8 +1437,8 @@ salt_checkin() { LOOP_COUNT=0 while check_service_status "$service"; do echo "$service still running" >> "$setup_log" 2>&1 - if [ $LOOP_COUNT -gt 120 ]; then - echo "$service could not be stopped in 120 seconds" >> "$setup_log" 2>&1 + if [ $LOOP_COUNT -gt 60 ]; then + echo "$service could not be stopped in 60 seconds, exiting" >> "$setup_log" 2>&1 exit 1 fi sleep 1; @@ -1434,8 +1449,8 @@ salt_checkin() { LOOP_COUNT=0 while ! check_service_status "$service"; do echo "$service still not running" >> "$setup_log" 2>&1 - if [ $LOOP_COUNT -gt 120 ]; then - echo "$service could not be started in 120 seconds" >> "$setup_log" 2>&1 + if [ $LOOP_COUNT -gt 60 ]; then + echo "$service could not be started in 60 seconds, exiting" >> "$setup_log" 2>&1 exit 1 fi sleep 1; @@ -1446,25 +1461,33 @@ salt_checkin() { LOOP_COUNT=0 while ! check_salt_master_status; do - echo "salt-minion cannot talk to salt-master" >> "$setup_log" 2>&1 - if [ $LOOP_COUNT -gt 40 ]; then - echo "salt-minion could not talk to salt-master after 40 attempts" >> "$setup_log" 2>&1 + echo "salt minion cannot talk to salt master" >> "$setup_log" 2>&1 + if [ $LOOP_COUNT -gt 30 ]; then + echo "salt minion could not talk to salt master after 30 attempts, exiting" >> "$setup_log" 2>&1 exit 1 fi sleep 1; ((LOOP_COUNT+=1)) done - systemctl status salt-master; - systemctl status salt-minion; + LOOP_COUNT=0 + while ! check_salt_minion_status; do + echo "salt master not getting job response from salt minion" >> "$setup_log" 2>&1 + if [ $LOOP_COUNT -gt 30 ]; then + echo "salt master not getting job response from salt minion after 30 attempts, exiting" >> "$setup_log" 2>&1 + exit 1 + fi + sleep 1; + ((LOOP_COUNT+=1)) + done echo " Confirming existence of the CA certificate" cat /etc/pki/ca.crt echo " Applyng a mine hack"; - salt '*' mine.send x509.get_pem_entries glob_path=/etc/pki/ca.crt; - salt '*' mine.update; + salt "$MINION_ID" mine.send x509.get_pem_entries glob_path=/etc/pki/ca.crt; + salt "$MINION_ID" mine.update; echo " Confirming salt mine now contain the certificate"; - salt '*' mine.get '*' x509.get_pem_entries; + salt "$MINION_ID" mine.get '*' x509.get_pem_entries; echo " Applying SSL state"; salt-call state.apply ssl; } >> "$setup_log" 2>&1 From e2fbe59b7c6ee956739b64edab1b8e8691c591c3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 14 Aug 2020 10:30:01 -0400 Subject: [PATCH 09/21] additional logging --- setup/so-functions | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 14c0fd671..2a34dd0a9 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1374,12 +1374,14 @@ check_service_status() { local service_name=$1 echo "Checking service $service_name status" >> "$setup_log" 2>&1 - systemctl status $service_name >> "$setup_log" 2>&1 + systemctl status $service_name > /dev/null 2>&1 local status=$? #true service is running false if not if [ $status -gt 0 ]; then + echo "$service_name is running" >> "$setup_log" 2>&1 status=1 else + echo "$service_name is not running" >> "$setup_log" 2>&1 status=0 fi @@ -1389,10 +1391,11 @@ check_service_status() { check_salt_master_status() { echo "Checking if we can talk to the salt master" >> "$setup_log" 2>&1 - salt-call state.show_top >> "$setup_log" 2>&1 + salt-call state.show_top > /dev/null 2>&1 local status=$? #true if there is an issue talking to salt master if [ $status -gt 0 ]; then + echo "Cannot talk to salt master" >> "$setup_log" 2>&1 status=1 else status=0 @@ -1408,6 +1411,7 @@ check_salt_minion_status() { local status=$? #true if there is an issue getting a job response from the minion if [ $status -gt 0 ]; then + echo "Not receiving job response from salt minion" >> "$setup_log" 2>&1 status=1 else status=0 @@ -1459,6 +1463,8 @@ salt_checkin() { done + #sleep 15; + LOOP_COUNT=0 while ! check_salt_master_status; do echo "salt minion cannot talk to salt master" >> "$setup_log" 2>&1 From cd1169b68d5b5811865dc4afdf76318f61793f01 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 14 Aug 2020 10:53:42 -0400 Subject: [PATCH 10/21] logging changes --- setup/so-functions | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 2a34dd0a9..987a71317 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1395,9 +1395,9 @@ check_salt_master_status() { local status=$? #true if there is an issue talking to salt master if [ $status -gt 0 ]; then - echo "Cannot talk to salt master" >> "$setup_log" 2>&1 status=1 else + echo "Can talk to salt master" >> "$setup_log" 2>&1 status=0 fi @@ -1411,9 +1411,9 @@ check_salt_minion_status() { local status=$? #true if there is an issue getting a job response from the minion if [ $status -gt 0 ]; then - echo "Not receiving job response from salt minion" >> "$setup_log" 2>&1 status=1 else + echo "Received job response from salt minion" >> "$setup_log" 2>&1 status=0 fi @@ -1421,7 +1421,6 @@ check_salt_minion_status() { } - salt_checkin() { case "$install_type" in @@ -1437,6 +1436,7 @@ salt_checkin() { ) local LOOP_COUNT=0 for service in "${SALT_SERVICES[@]}"; do + echo "Stopping service $service" >> "$setup_log" 2>&1 systemctl stop "$service" >> "$setup_log" 2>&1 LOOP_COUNT=0 while check_service_status "$service"; do @@ -1449,6 +1449,7 @@ salt_checkin() { ((LOOP_COUNT+=1)) done + echo "Starting service $service" >> "$setup_log" 2>&1 systemctl start "$service" >> "$setup_log" 2>&1 LOOP_COUNT=0 while ! check_service_status "$service"; do @@ -1478,9 +1479,9 @@ salt_checkin() { LOOP_COUNT=0 while ! check_salt_minion_status; do - echo "salt master not getting job response from salt minion" >> "$setup_log" 2>&1 + echo "salt master did not get a job response from salt minion" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 30 ]; then - echo "salt master not getting job response from salt minion after 30 attempts, exiting" >> "$setup_log" 2>&1 + echo "salt master did not get a job response from salt minion after 30 attempts, exiting" >> "$setup_log" 2>&1 exit 1 fi sleep 1; From ea5116700d19fbd1d16d668b696ca558b4b99366 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 14 Aug 2020 11:01:26 -0400 Subject: [PATCH 11/21] stop both service then start both --- setup/so-functions | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 987a71317..c92f6a152 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1431,8 +1431,8 @@ salt_checkin() { echo " *** Restarting Salt to fix any SSL errors. ***"; local SALT_SERVICES=(\ - "salt-minion" \ - "salt-master" + "salt-master" \ + "salt-minion" ) local LOOP_COUNT=0 for service in "${SALT_SERVICES[@]}"; do @@ -1448,7 +1448,11 @@ salt_checkin() { sleep 1; ((LOOP_COUNT+=1)) done + done + sleep 5; + + for service in "${SALT_SERVICES[@]}"; do echo "Starting service $service" >> "$setup_log" 2>&1 systemctl start "$service" >> "$setup_log" 2>&1 LOOP_COUNT=0 @@ -1461,7 +1465,6 @@ salt_checkin() { sleep 1; ((LOOP_COUNT+=1)) done - done #sleep 15; From 876c6c7cb0448f22a0033e35eb47c9e35ccb5ee0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 14 Aug 2020 11:16:56 -0400 Subject: [PATCH 12/21] logic changes --- setup/so-functions | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index c92f6a152..a4444481e 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1374,7 +1374,7 @@ check_service_status() { local service_name=$1 echo "Checking service $service_name status" >> "$setup_log" 2>&1 - systemctl status $service_name > /dev/null 2>&1 + systemctl status $service_name >> "$setup_log" 2>&1 local status=$? #true service is running false if not if [ $status -gt 0 ]; then @@ -1439,7 +1439,7 @@ salt_checkin() { echo "Stopping service $service" >> "$setup_log" 2>&1 systemctl stop "$service" >> "$setup_log" 2>&1 LOOP_COUNT=0 - while check_service_status "$service"; do + while (( check_service_status "$service" )); do echo "$service still running" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 60 ]; then echo "$service could not be stopped in 60 seconds, exiting" >> "$setup_log" 2>&1 @@ -1456,7 +1456,7 @@ salt_checkin() { echo "Starting service $service" >> "$setup_log" 2>&1 systemctl start "$service" >> "$setup_log" 2>&1 LOOP_COUNT=0 - while ! check_service_status "$service"; do + while ! (( check_service_status )) "$service"; do echo "$service still not running" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 60 ]; then echo "$service could not be started in 60 seconds, exiting" >> "$setup_log" 2>&1 @@ -1470,7 +1470,7 @@ salt_checkin() { #sleep 15; LOOP_COUNT=0 - while ! check_salt_master_status; do + while (( check_salt_master_status )); do echo "salt minion cannot talk to salt master" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 30 ]; then echo "salt minion could not talk to salt master after 30 attempts, exiting" >> "$setup_log" 2>&1 @@ -1481,7 +1481,7 @@ salt_checkin() { done LOOP_COUNT=0 - while ! check_salt_minion_status; do + while (( check_salt_minion_status )); do echo "salt master did not get a job response from salt minion" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 30 ]; then echo "salt master did not get a job response from salt minion after 30 attempts, exiting" >> "$setup_log" 2>&1 From aa2b0699d57614eb593523f4de4954db9765a266 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 14 Aug 2020 11:20:18 -0400 Subject: [PATCH 13/21] move parens --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index a4444481e..8e0beaa74 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1456,7 +1456,7 @@ salt_checkin() { echo "Starting service $service" >> "$setup_log" 2>&1 systemctl start "$service" >> "$setup_log" 2>&1 LOOP_COUNT=0 - while ! (( check_service_status )) "$service"; do + while ! (( check_service_status "$service" )); do echo "$service still not running" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 60 ]; then echo "$service could not be started in 60 seconds, exiting" >> "$setup_log" 2>&1 From 683e8a2a39f969bac6c75efd8078f543d15d1d93 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 14 Aug 2020 11:24:46 -0400 Subject: [PATCH 14/21] remove quotes --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 8e0beaa74..160ccdb37 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1439,7 +1439,7 @@ salt_checkin() { echo "Stopping service $service" >> "$setup_log" 2>&1 systemctl stop "$service" >> "$setup_log" 2>&1 LOOP_COUNT=0 - while (( check_service_status "$service" )); do + while (( check_service_status $service )); do echo "$service still running" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 60 ]; then echo "$service could not be stopped in 60 seconds, exiting" >> "$setup_log" 2>&1 @@ -1456,7 +1456,7 @@ salt_checkin() { echo "Starting service $service" >> "$setup_log" 2>&1 systemctl start "$service" >> "$setup_log" 2>&1 LOOP_COUNT=0 - while ! (( check_service_status "$service" )); do + while ! (( check_service_status $service )); do echo "$service still not running" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 60 ]; then echo "$service could not be started in 60 seconds, exiting" >> "$setup_log" 2>&1 From 69fd80375994bd359795498d8c26d30762900401 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 14 Aug 2020 11:30:10 -0400 Subject: [PATCH 15/21] change while --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 160ccdb37..0d912e82b 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1439,7 +1439,7 @@ salt_checkin() { echo "Stopping service $service" >> "$setup_log" 2>&1 systemctl stop "$service" >> "$setup_log" 2>&1 LOOP_COUNT=0 - while (( check_service_status $service )); do + while (( $(check_service_status $service) )); do echo "$service still running" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 60 ]; then echo "$service could not be stopped in 60 seconds, exiting" >> "$setup_log" 2>&1 @@ -1456,7 +1456,7 @@ salt_checkin() { echo "Starting service $service" >> "$setup_log" 2>&1 systemctl start "$service" >> "$setup_log" 2>&1 LOOP_COUNT=0 - while ! (( check_service_status $service )); do + while ! (( $(check_service_status $service) )); do echo "$service still not running" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 60 ]; then echo "$service could not be started in 60 seconds, exiting" >> "$setup_log" 2>&1 From e229cb49bcc283c63062cecca69b2e023692c554 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 14 Aug 2020 11:40:21 -0400 Subject: [PATCH 16/21] logic changes --- setup/so-functions | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 0d912e82b..e857f71e1 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1376,12 +1376,12 @@ check_service_status() { echo "Checking service $service_name status" >> "$setup_log" 2>&1 systemctl status $service_name >> "$setup_log" 2>&1 local status=$? - #true service is running false if not + #true if there is an issue with the service false if it is running properly if [ $status -gt 0 ]; then - echo "$service_name is running" >> "$setup_log" 2>&1 + echo "$service_name not is running" >> "$setup_log" 2>&1 status=1 else - echo "$service_name is not running" >> "$setup_log" 2>&1 + echo "$service_name is running" >> "$setup_log" 2>&1 status=0 fi @@ -1439,7 +1439,7 @@ salt_checkin() { echo "Stopping service $service" >> "$setup_log" 2>&1 systemctl stop "$service" >> "$setup_log" 2>&1 LOOP_COUNT=0 - while (( $(check_service_status $service) )); do + while ! (( $(check_service_status $service) )); do echo "$service still running" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 60 ]; then echo "$service could not be stopped in 60 seconds, exiting" >> "$setup_log" 2>&1 @@ -1456,7 +1456,7 @@ salt_checkin() { echo "Starting service $service" >> "$setup_log" 2>&1 systemctl start "$service" >> "$setup_log" 2>&1 LOOP_COUNT=0 - while ! (( $(check_service_status $service) )); do + while (( $(check_service_status $service) )); do echo "$service still not running" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 60 ]; then echo "$service could not be started in 60 seconds, exiting" >> "$setup_log" 2>&1 From 9d59fc23dd81b13b952fbcfd46f0f41fae62a625 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 14 Aug 2020 12:24:15 -0400 Subject: [PATCH 17/21] logic changes --- setup/so-functions | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index e857f71e1..5e2110ffe 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1378,7 +1378,7 @@ check_service_status() { local status=$? #true if there is an issue with the service false if it is running properly if [ $status -gt 0 ]; then - echo "$service_name not is running" >> "$setup_log" 2>&1 + echo "$service_name is not running" >> "$setup_log" 2>&1 status=1 else echo "$service_name is running" >> "$setup_log" 2>&1 @@ -1439,7 +1439,7 @@ salt_checkin() { echo "Stopping service $service" >> "$setup_log" 2>&1 systemctl stop "$service" >> "$setup_log" 2>&1 LOOP_COUNT=0 - while ! (( $(check_service_status $service) )); do + while ! check_service_status $service; do echo "$service still running" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 60 ]; then echo "$service could not be stopped in 60 seconds, exiting" >> "$setup_log" 2>&1 @@ -1456,7 +1456,7 @@ salt_checkin() { echo "Starting service $service" >> "$setup_log" 2>&1 systemctl start "$service" >> "$setup_log" 2>&1 LOOP_COUNT=0 - while (( $(check_service_status $service) )); do + while check_service_status $service; do echo "$service still not running" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 60 ]; then echo "$service could not be started in 60 seconds, exiting" >> "$setup_log" 2>&1 @@ -1470,7 +1470,7 @@ salt_checkin() { #sleep 15; LOOP_COUNT=0 - while (( check_salt_master_status )); do + while check_salt_master_status; do echo "salt minion cannot talk to salt master" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 30 ]; then echo "salt minion could not talk to salt master after 30 attempts, exiting" >> "$setup_log" 2>&1 @@ -1481,7 +1481,7 @@ salt_checkin() { done LOOP_COUNT=0 - while (( check_salt_minion_status )); do + while check_salt_minion_status; do echo "salt master did not get a job response from salt minion" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 30 ]; then echo "salt master did not get a job response from salt minion after 30 attempts, exiting" >> "$setup_log" 2>&1 From 4b21c1b492e2e99947db8ceae3f4ba48a1cffa4c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 14 Aug 2020 12:45:50 -0400 Subject: [PATCH 18/21] logic change --- setup/so-functions | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 5e2110ffe..95313c6ff 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1379,14 +1379,12 @@ check_service_status() { #true if there is an issue with the service false if it is running properly if [ $status -gt 0 ]; then echo "$service_name is not running" >> "$setup_log" 2>&1 - status=1 + echo 1; else echo "$service_name is running" >> "$setup_log" 2>&1 - status=0 + echo 0; fi - return $status - } check_salt_master_status() { @@ -1395,14 +1393,12 @@ check_salt_master_status() { local status=$? #true if there is an issue talking to salt master if [ $status -gt 0 ]; then - status=1 + echo 1; else echo "Can talk to salt master" >> "$setup_log" 2>&1 - status=0 + echo 0; fi - return $status - } check_salt_minion_status() { @@ -1411,14 +1407,12 @@ check_salt_minion_status() { local status=$? #true if there is an issue getting a job response from the minion if [ $status -gt 0 ]; then - status=1 + echo 1; else echo "Received job response from salt minion" >> "$setup_log" 2>&1 - status=0 + echo 0; fi - return $status - } salt_checkin() { @@ -1439,7 +1433,7 @@ salt_checkin() { echo "Stopping service $service" >> "$setup_log" 2>&1 systemctl stop "$service" >> "$setup_log" 2>&1 LOOP_COUNT=0 - while ! check_service_status $service; do + while ! $(check_service_status $service); do echo "$service still running" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 60 ]; then echo "$service could not be stopped in 60 seconds, exiting" >> "$setup_log" 2>&1 @@ -1456,7 +1450,7 @@ salt_checkin() { echo "Starting service $service" >> "$setup_log" 2>&1 systemctl start "$service" >> "$setup_log" 2>&1 LOOP_COUNT=0 - while check_service_status $service; do + while $(check_service_status $service); do echo "$service still not running" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 60 ]; then echo "$service could not be started in 60 seconds, exiting" >> "$setup_log" 2>&1 @@ -1470,7 +1464,7 @@ salt_checkin() { #sleep 15; LOOP_COUNT=0 - while check_salt_master_status; do + while $(check_salt_master_status); do echo "salt minion cannot talk to salt master" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 30 ]; then echo "salt minion could not talk to salt master after 30 attempts, exiting" >> "$setup_log" 2>&1 @@ -1481,7 +1475,7 @@ salt_checkin() { done LOOP_COUNT=0 - while check_salt_minion_status; do + while $(check_salt_minion_status); do echo "salt master did not get a job response from salt minion" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 30 ]; then echo "salt master did not get a job response from salt minion after 30 attempts, exiting" >> "$setup_log" 2>&1 From 4bb23a089e5ebe8ffe7b8920b86bc6d9580312dd Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 14 Aug 2020 12:48:52 -0400 Subject: [PATCH 19/21] add some parens --- setup/so-functions | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 95313c6ff..2e4e054b9 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1433,7 +1433,7 @@ salt_checkin() { echo "Stopping service $service" >> "$setup_log" 2>&1 systemctl stop "$service" >> "$setup_log" 2>&1 LOOP_COUNT=0 - while ! $(check_service_status $service); do + while ! (( $(check_service_status $service) )); do echo "$service still running" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 60 ]; then echo "$service could not be stopped in 60 seconds, exiting" >> "$setup_log" 2>&1 @@ -1450,7 +1450,7 @@ salt_checkin() { echo "Starting service $service" >> "$setup_log" 2>&1 systemctl start "$service" >> "$setup_log" 2>&1 LOOP_COUNT=0 - while $(check_service_status $service); do + while (( $(check_service_status $service) )); do echo "$service still not running" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 60 ]; then echo "$service could not be started in 60 seconds, exiting" >> "$setup_log" 2>&1 @@ -1464,7 +1464,7 @@ salt_checkin() { #sleep 15; LOOP_COUNT=0 - while $(check_salt_master_status); do + while (( $(check_salt_master_status) )); do echo "salt minion cannot talk to salt master" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 30 ]; then echo "salt minion could not talk to salt master after 30 attempts, exiting" >> "$setup_log" 2>&1 @@ -1475,7 +1475,7 @@ salt_checkin() { done LOOP_COUNT=0 - while $(check_salt_minion_status); do + while (( $(check_salt_minion_status) )); do echo "salt master did not get a job response from salt minion" >> "$setup_log" 2>&1 if [ $LOOP_COUNT -gt 30 ]; then echo "salt master did not get a job response from salt minion after 30 attempts, exiting" >> "$setup_log" 2>&1 From 6602ad32862189e068bfb867f347e7e966204230 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 14 Aug 2020 12:53:24 -0400 Subject: [PATCH 20/21] sleep for 5 seconds --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 2e4e054b9..3ca22c159 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1461,7 +1461,7 @@ salt_checkin() { done done - #sleep 15; + sleep 5; LOOP_COUNT=0 while (( $(check_salt_master_status) )); do From b7bfa6f9a9d712018dbf61066af5c963054581d6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 14 Aug 2020 12:55:54 -0400 Subject: [PATCH 21/21] move functions up --- setup/so-functions | 90 +++++++++++++++++++++++----------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 3ca22c159..59a8f6fe3 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -195,6 +195,51 @@ check_pass_match() { fi } +check_service_status() { + + local service_name=$1 + echo "Checking service $service_name status" >> "$setup_log" 2>&1 + systemctl status $service_name >> "$setup_log" 2>&1 + local status=$? + #true if there is an issue with the service false if it is running properly + if [ $status -gt 0 ]; then + echo "$service_name is not running" >> "$setup_log" 2>&1 + echo 1; + else + echo "$service_name is running" >> "$setup_log" 2>&1 + echo 0; + fi + +} + +check_salt_master_status() { + echo "Checking if we can talk to the salt master" >> "$setup_log" 2>&1 + salt-call state.show_top > /dev/null 2>&1 + local status=$? + #true if there is an issue talking to salt master + if [ $status -gt 0 ]; then + echo 1; + else + echo "Can talk to salt master" >> "$setup_log" 2>&1 + echo 0; + fi + +} + +check_salt_minion_status() { + echo "Checking if the salt minion will respond to jobs" >> "$setup_log" 2>&1 + salt "$MINION_ID" test.ping >> "$setup_log" 2>&1 + local status=$? + #true if there is an issue getting a job response from the minion + if [ $status -gt 0 ]; then + echo 1; + else + echo "Received job response from salt minion" >> "$setup_log" 2>&1 + echo 0; + fi + +} + check_soremote_pass() { check_pass_match "$SOREMOTEPASS1" "$SOREMOTEPASS2" "SCMATCH" } @@ -1370,51 +1415,6 @@ saltify() { } -check_service_status() { - - local service_name=$1 - echo "Checking service $service_name status" >> "$setup_log" 2>&1 - systemctl status $service_name >> "$setup_log" 2>&1 - local status=$? - #true if there is an issue with the service false if it is running properly - if [ $status -gt 0 ]; then - echo "$service_name is not running" >> "$setup_log" 2>&1 - echo 1; - else - echo "$service_name is running" >> "$setup_log" 2>&1 - echo 0; - fi - -} - -check_salt_master_status() { - echo "Checking if we can talk to the salt master" >> "$setup_log" 2>&1 - salt-call state.show_top > /dev/null 2>&1 - local status=$? - #true if there is an issue talking to salt master - if [ $status -gt 0 ]; then - echo 1; - else - echo "Can talk to salt master" >> "$setup_log" 2>&1 - echo 0; - fi - -} - -check_salt_minion_status() { - echo "Checking if the salt minion will respond to jobs" >> "$setup_log" 2>&1 - salt "$MINION_ID" test.ping >> "$setup_log" 2>&1 - local status=$? - #true if there is an issue getting a job response from the minion - if [ $status -gt 0 ]; then - echo 1; - else - echo "Received job response from salt minion" >> "$setup_log" 2>&1 - echo 0; - fi - -} - salt_checkin() { case "$install_type" in