diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index f754b34ef..c19d51a42 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -133,11 +133,23 @@ check_elastic_license() { } check_salt_master_status() { - local timeout=$1 - echo "Checking if we can talk to the salt master" - salt-call state.show_top concurrent=true - - return + local count=0 + local attempts="${1:- 10}" + current_time="$(date '+%b %d %H:%M:%S')" + echo "Checking if we can access the salt master and that it is ready at: ${current_time}" + while ! salt-call state.show_top -l error concurrent=true 1> /dev/null; do + current_time="$(date '+%b %d %H:%M:%S')" + echo "Can't access salt master or it is not ready at: ${current_time}" + ((count+=1)) + if [[ $count -eq $attempts ]]; then + # 10 attempts takes about 5.5 minutes + echo "Gave up trying to access salt-master" + return 1 + fi + done + current_time="$(date '+%b %d %H:%M:%S')" + echo "Successfully accessed and salt master ready at: ${current_time}" + return 0 } check_salt_minion_status() { diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 0e11276ea..84501bad5 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -736,14 +736,8 @@ main() { echo "" set_os - if ! check_salt_master_status; then - echo "Could not talk to salt master" - echo "Please run 'systemctl status salt-master' to ensure the salt-master service is running and check the log at /opt/so/log/salt/master." - echo "SOUP will now attempt to start the salt-master service and exit." - exit 1 - fi - echo "This node can communicate with the salt-master." + check_salt_master_status 1 || fail "Could not talk to salt master: Please run 'systemctl status salt-master' to ensure the salt-master service is running and check the log at /opt/so/log/salt/master." echo "Checking to see if this is a manager." echo "" @@ -881,7 +875,7 @@ main() { # Testing that salt-master is up by checking that is it connected to itself set +e echo "Waiting on the Salt Master service to be ready." - salt-call state.show_top -l error queue=True || fail "salt-master could not be reached. Check $SOUP_LOG for details." + check_salt_master_status || fail "Can't access salt master or it is not ready. Check $SOUP_LOG for details." set -e # update the salt-minion configs here and start the minion @@ -917,7 +911,7 @@ main() { set +e echo "Waiting on the Salt Master service to be ready." - salt-call state.show_top -l error queue=True || fail "salt-master could not be reached. Check $SOUP_LOG for details." + check_salt_master_status || fail "Can't access salt master or it is not ready. Check $SOUP_LOG for details." set -e echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes." diff --git a/setup/so-functions b/setup/so-functions index 42a4b4ac6..68fd01550 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2111,11 +2111,6 @@ saltify() { } -# Run a salt command to generate the minion key -salt_firstcheckin() { - salt-call state.show_top >> /dev/null 2>&1 # send output to /dev/null because we don't actually care about the ouput -} - salt_install_module_deps() { logCmd "salt-pip install docker --no-index --only-binary=:all: --find-links files/salt_module_deps/docker/" logCmd "salt-pip install pymysql --no-index --only-binary=:all: --find-links files/salt_module_deps/pymysql/" diff --git a/setup/so-setup b/setup/so-setup index dc209b756..4db24aa1a 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -715,6 +715,11 @@ if ! [[ -f $install_opt_file ]]; then logCmd "salt-call state.apply common.packages" logCmd "salt-call state.apply common" logCmd "salt-call state.apply salt.master" + + # wait here until we get a response from the salt-master since it may have just restarted + # exit setup after 5-6 minutes of trying + check_salt_master_status || fail "Can't access salt master or it is not ready" + logCmd "salt-call state.apply docker" firewall_generate_templates set_initial_firewall_policy