From a2180a6721a77eb8c3e47742827f0eb3dc3aa9fc Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 11 Apr 2022 15:01:41 -0400 Subject: [PATCH 1/9] ensure salt-master service is running before proceeding with soup --- salt/common/tools/sbin/so-common | 24 ++++++++++++++++++++++++ salt/common/tools/sbin/soup | 6 ++++++ setup/so-functions | 30 ------------------------------ 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 5e1ecfbeb..3a289308e 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -120,6 +120,30 @@ check_elastic_license() { fi } +check_salt_master_status() { + local timeout=$1 + echo "Checking if we can talk to the salt master" + salt-call state.show_top -t $timeout concurrent=true + + return +} + +check_salt_minion_status() { + local timeout=$1 + echo "Checking if the salt minion will respond to jobs" >> "$setup_log" 2>&1 + salt "$MINION_ID" test.ping -t $timeout > /dev/null 2>&1 + local status=$? + if [ $status -gt 0 ]; then + echo " Minion did not respond" >> "$setup_log" 2>&1 + else + echo " Received job response from salt minion" >> "$setup_log" 2>&1 + fi + + retrun $status +} + + + copy_new_files() { # Copy new files over to the salt dir cd $UPDATE_DIR diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index efa6000a6..3ba03d012 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -1018,6 +1018,12 @@ main() { echo "" require_manager + if ! check_salt_master_status -t 30; then + echo "Could not talk to salt master" + echo "Please run `systemctl status salt-master` to ensure the salt-master service is running." + exit 1 + fi + check_pillar_items echo "Checking to see if this is an airgap install." diff --git a/setup/so-functions b/setup/so-functions index ec1b25a26..aac5f24d2 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -232,36 +232,6 @@ check_service_status() { } -check_salt_master_status() { - local timeout=$1 - echo "Checking if we can talk to the salt master" >> "$setup_log" 2>&1 - salt-call saltutil.kill_all_jobs > /dev/null 2>&1 - salt-call state.show_top -t $timeout > /dev/null 2>&1 - local status=$? - if [ $status -gt 0 ]; then - echo " Could not talk to salt master" >> "$setup_log" 2>&1 - return 1; - else - echo " Can talk to salt master" >> "$setup_log" 2>&1 - return 0; - fi - -} - -check_salt_minion_status() { - local timeout=$1 - echo "Checking if the salt minion will respond to jobs" >> "$setup_log" 2>&1 - salt "$MINION_ID" test.ping -t $timeout > /dev/null 2>&1 - local status=$? - if [ $status -gt 0 ]; then - echo " Minion did not respond" >> "$setup_log" 2>&1 - return 1; - else - echo " Received job response from salt minion" >> "$setup_log" 2>&1 - return 0; - fi -} - check_soremote_pass() { check_pass_match "$SOREMOTEPASS1" "$SOREMOTEPASS2" "SCMATCH" } From 79dc2374e024ef59125c12799f147a2336c07edb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 11 Apr 2022 15:09:00 -0400 Subject: [PATCH 2/9] check that salt-master is running before requiring manager --- salt/common/tools/sbin/soup | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 3ba03d012..72a8cd467 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -1014,16 +1014,16 @@ main() { echo "### Preparing soup at $(date) ###" echo "" - echo "Checking to see if this is a manager." - echo "" - require_manager - if ! check_salt_master_status -t 30; then echo "Could not talk to salt master" echo "Please run `systemctl status salt-master` to ensure the salt-master service is running." exit 1 fi + echo "Checking to see if this is a manager." + echo "" + require_manager + check_pillar_items echo "Checking to see if this is an airgap install." From 6ed8694008446e2f194974a97a7110a2d3695ca1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 11 Apr 2022 15:11:57 -0400 Subject: [PATCH 3/9] dont need to pass -t --- salt/common/tools/sbin/so-common | 2 +- salt/common/tools/sbin/soup | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 3a289308e..2bfc1c0de 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -139,7 +139,7 @@ check_salt_minion_status() { echo " Received job response from salt minion" >> "$setup_log" 2>&1 fi - retrun $status + return $status } diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 72a8cd467..770c86ae8 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -1014,7 +1014,7 @@ main() { echo "### Preparing soup at $(date) ###" echo "" - if ! check_salt_master_status -t 30; then + if ! check_salt_master_status 30; then echo "Could not talk to salt master" echo "Please run `systemctl status salt-master` to ensure the salt-master service is running." exit 1 From 01510c184a251d7ed4f624a70febdf06eb473644 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 11 Apr 2022 15:36:02 -0400 Subject: [PATCH 4/9] set_os and set_cron_service_name sooner --- salt/common/tools/sbin/so-common | 2 +- salt/common/tools/sbin/soup | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 2bfc1c0de..7b5f29c00 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -123,7 +123,7 @@ check_elastic_license() { check_salt_master_status() { local timeout=$1 echo "Checking if we can talk to the salt master" - salt-call state.show_top -t $timeout concurrent=true + salt-call state.show_top concurrent=true return } diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 770c86ae8..b5c1057ad 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -1014,7 +1014,9 @@ main() { echo "### Preparing soup at $(date) ###" echo "" - if ! check_salt_master_status 30; then + set_os + set_cron_service_name + if ! check_salt_master_status; then echo "Could not talk to salt master" echo "Please run `systemctl status salt-master` to ensure the salt-master service is running." exit 1 @@ -1055,8 +1057,6 @@ main() { echo "Verifying we have the latest soup script." verify_latest_update_script echo "" - set_os - set_cron_service_name set_palette check_elastic_license echo "" From 2a18059ad94f178abf9fdb3e48a5fc17bb1bb435 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 11 Apr 2022 15:37:07 -0400 Subject: [PATCH 5/9] use quotes --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index b5c1057ad..f98866c43 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -1018,7 +1018,7 @@ main() { set_cron_service_name if ! check_salt_master_status; then echo "Could not talk to salt master" - echo "Please run `systemctl status salt-master` to ensure the salt-master service is running." + echo "Please run 'systemctl status salt-master' to ensure the salt-master service is running." exit 1 fi From 464772d7d375041a0c2b20ea7be85de9241d914f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 11 Apr 2022 15:43:09 -0400 Subject: [PATCH 6/9] start salt-master and salt-minion service is soup fails and exits --- salt/common/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index f98866c43..547837dd6 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -93,6 +93,8 @@ check_err() { fi set +e systemctl_func "start" "$cron_service_name" + systemctl_func "start" "salt-master" + systemctl_func "start" "salt-minion" enable_highstate exit $exit_code fi From 9914148441b20e1e6a61801cd53faf2c0d0016d7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 11 Apr 2022 15:51:11 -0400 Subject: [PATCH 7/9] more verbose --- salt/common/tools/sbin/soup | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 547837dd6..7fe7f07c4 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -1020,10 +1020,13 @@ main() { set_cron_service_name if ! check_salt_master_status; then echo "Could not talk to salt master" - echo "Please run 'systemctl status salt-master' to ensure the salt-master service is running." + echo "Please run 'systemctl status salt-master' to ensure the salt-master service is running and check the log at /opt/so/log/salt/master." + echo "SOUP will now attempt to start the salt-master service and exit." exit 1 fi + echo "This node can communicate with the salt-master." + echo "Checking to see if this is a manager." echo "" require_manager From d102ca298d3691c77c8f0582990e841dd663bb6b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 11 Apr 2022 16:01:36 -0400 Subject: [PATCH 8/9] move messages about starting services on soup failure before exit message --- salt/common/tools/sbin/soup | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 7fe7f07c4..9b9358693 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -34,6 +34,13 @@ check_err() { local err_msg="Unhandled error occured, please check $SOUP_LOG for details." [[ $ERR_HANDLED == true ]] && exit $exit_code + + set +e + systemctl_func "start" "$cron_service_name" + systemctl_func "start" "salt-master" + systemctl_func "start" "salt-minion" + enable_highstate + if [[ $exit_code -ne 0 ]]; then printf '%s' "Soup failed with error $exit_code: " case $exit_code in @@ -91,11 +98,7 @@ check_err() { if [[ $exit_code -ge 64 && $exit_code -le 113 ]]; then echo "$err_msg" fi - set +e - systemctl_func "start" "$cron_service_name" - systemctl_func "start" "salt-master" - systemctl_func "start" "salt-minion" - enable_highstate + exit $exit_code fi From d68b6e7c9a2d630692697a7e028ef72aa7b7fb46 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 11 Apr 2022 16:03:00 -0400 Subject: [PATCH 9/9] only start if exit code != 0 --- salt/common/tools/sbin/soup | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 9b9358693..70e300cb0 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -35,13 +35,14 @@ check_err() { [[ $ERR_HANDLED == true ]] && exit $exit_code - set +e - systemctl_func "start" "$cron_service_name" - systemctl_func "start" "salt-master" - systemctl_func "start" "salt-minion" - enable_highstate - if [[ $exit_code -ne 0 ]]; then + + set +e + systemctl_func "start" "$cron_service_name" + systemctl_func "start" "salt-master" + systemctl_func "start" "salt-minion" + enable_highstate + printf '%s' "Soup failed with error $exit_code: " case $exit_code in 2)