redo how we check if salt-master is ready and accessible

salt/common/tools/sbin/so-common

@@ -133,11 +133,23 @@ check_elastic_license() {
 }
 
 check_salt_master_status() {
-	local timeout=$1
+	local count=0
-	echo "Checking if we can talk to the salt master"
+    local attempts="${1:- 10}"
-	salt-call state.show_top concurrent=true
+	current_time="$(date '+%b %d %H:%M:%S')"
-
+	echo "Checking if we can access the salt master and that it is ready at: ${current_time}"
-	return
+    while ! salt-call state.show_top -l error concurrent=true 1> /dev/null; do
+        current_time="$(date '+%b %d %H:%M:%S')"
+        echo "Can't access salt master or it is not ready at: ${current_time}"
+        ((count+=1))
+        if [[ $count -eq $attempts ]]; then
+			# 10 attempts takes about 5.5 minutes
+            echo "Gave up trying to access salt-master"
+            return 1
+        fi
+    done
+	current_time="$(date '+%b %d %H:%M:%S')"
+	echo "Successfully accessed and salt master ready at: ${current_time}"
+    return 0
 }
 
 check_salt_minion_status() {

salt/manager/tools/sbin/soup

@@ -736,14 +736,8 @@ main() {
   echo ""
 
   set_os
-  if ! check_salt_master_status; then
-  	echo "Could not talk to salt master"
-    echo "Please run 'systemctl status salt-master' to ensure the salt-master service is running and check the log at /opt/so/log/salt/master."
-    echo "SOUP will now attempt to start the salt-master service and exit."
-    exit 1
-  fi
 
-  echo "This node can communicate with the salt-master."
+  check_salt_master_status 1 || fail  "Could not talk to salt master: Please run 'systemctl status salt-master' to ensure the salt-master service is running and check the log at /opt/so/log/salt/master."
 
   echo "Checking to see if this is a manager."
   echo ""
@@ -881,7 +875,7 @@ main() {
     # Testing that salt-master is up by checking that is it connected to itself
     set +e
     echo "Waiting on the Salt Master service to be ready."
-    salt-call state.show_top -l error queue=True || fail "salt-master could not be reached. Check $SOUP_LOG for details."
+    check_salt_master_status || fail  "Can't access salt master or it is not ready. Check $SOUP_LOG for details."
     set -e
 
     # update the salt-minion configs here and start the minion
@@ -917,7 +911,7 @@ main() {
 
     set +e
     echo "Waiting on the Salt Master service to be ready."
-    salt-call state.show_top -l error queue=True || fail "salt-master could not be reached. Check $SOUP_LOG for details."
+    check_salt_master_status || fail  "Can't access salt master or it is not ready. Check $SOUP_LOG for details."
     set -e
     
     echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."

setup/so-functions

@@ -2111,11 +2111,6 @@ saltify() {
 
 }
 
-# Run a salt command to generate the minion key
-salt_firstcheckin() {
-	salt-call state.show_top >> /dev/null 2>&1 # send output to /dev/null because we don't actually care about the ouput
-}
-
 salt_install_module_deps() {
     logCmd "salt-pip install docker --no-index --only-binary=:all: --find-links files/salt_module_deps/docker/"
     logCmd "salt-pip install pymysql --no-index --only-binary=:all: --find-links files/salt_module_deps/pymysql/"

setup/so-setup

@@ -715,6 +715,11 @@ if ! [[ -f $install_opt_file ]]; then
 		logCmd "salt-call state.apply common.packages"
 		logCmd "salt-call state.apply common"
 		logCmd "salt-call state.apply salt.master"
+
+		# wait here until we get a response from the salt-master since it may have just restarted
+		# exit setup after 5-6 minutes of trying
+		check_salt_master_status || fail  "Can't access salt master or it is not ready"
+
 		logCmd "salt-call state.apply docker"
 		firewall_generate_templates
 		set_initial_firewall_policy