block soup if all ES nodes are not online and reporting their ES version for compatibility check

salt/manager/tools/sbin/soup

@@ -1021,9 +1021,24 @@ verify_es_version_compatibility() {
         local retries=20
         local retry_count=0
         local delay=180
+        local expected_es_nodes
+        local searchnode_minions
         SEARCHNODE_ES_VERSIONS=""
 
         while [[ $retry_count -lt $retries ]]; do
+            if ! searchnode_minions=$(set -o pipefail; salt-key --out=json --list=accepted 2> /dev/null | jq -r '.minions[]? | select(endswith("searchnode"))'); then
+                echo "Failed to retrieve grid searchnodes via salt-key... Retrying in $delay seconds. Attempt $((retry_count + 1)) of $retries."
+                ((retry_count++))
+                sleep $delay
+                continue
+            fi
+            # Always add node running soup to expected es nodes
+            expected_es_nodes="${MINIONID%_*}"
+            while IFS= read -r searchnode_minion; do
+                [[ -z "$searchnode_minion" ]] && continue
+                expected_es_nodes+=$'\n'"${searchnode_minion%_searchnode}"
+            done <<< "$searchnode_minions"
+
             SEARCHNODE_ES_VERSIONS=$(so-elasticsearch-query _nodes/_all/version --retry 5 --retry-delay 10 --fail 2>&1)
             local exit_status=$?
 
@@ -1043,6 +1058,14 @@ verify_es_version_compatibility() {
                 fi
             done < <(echo "$SEARCHNODE_ES_VERSIONS" | jq -r '.nodes | to_entries[] | [.value.name, .value.version] | @tsv')
 
+            while IFS= read -r expected_es_node; do
+                [[ -z "$expected_es_node" ]] && continue
+                if ! echo "$SEARCHNODE_ES_VERSIONS" | jq -e --arg node "$expected_es_node" '.nodes | to_entries | any(.value.name == $node)' > /dev/null; then
+                    echo "Searchnode $expected_es_node did not report an Elasticsearch version. It may be offline or still upgrading."
+                    all_searchnodes_compatible=false
+                fi
+            done <<< "$expected_es_nodes"
+
             if [[ "$all_searchnodes_compatible" == true ]]; then
                 echo "All Searchnodes are upgradable to Elasticsearch $target_es_version."
                 return 0
@@ -1056,21 +1079,27 @@ verify_es_version_compatibility() {
         return 1
     }
 
-     # Gather heavynode version info and verify that each node is running a version compatible with the target ES version.
+    # Gather heavynode version info and verify that each node is running a version compatible with the target ES version.
     verify_heavynodes_es_target_compatibility() {
-        if ! salt-key -l accepted | grep -q 'heavynode$'; then
+        local heavynode_minions
-            echo "No heavynodes detected. Skipping heavynode Elasticsearch version compatibility check."
-            return 0
-        fi
-
-        echo -e "\nOne or more heavynodes detected. Verifying each is running an Elasticsearch version that is compatible with $target_es_version."
-
         local retries=20
         local retry_count=0
         local delay=180
         HEAVYNODE_ES_VERSIONS=""
 
         while [[ $retry_count -lt $retries ]]; do
+            if ! heavynode_minions=$(set -o pipefail; salt-key --out=json --list=accepted 2> /dev/null | jq -r '.minions[]? | select(endswith("heavynode"))'); then
+                echo "Failed to retrieve grid heavynodes via salt-key... Retrying in $delay seconds. Attempt $((retry_count + 1)) of $retries."
+                ((retry_count++))
+                sleep $delay
+                continue
+            fi
+
+            if [[ -z "$heavynode_minions" ]]; then
+                echo "No heavynodes detected. Skipping heavynode Elasticsearch version compatibility check."
+                return 0
+            fi
+
             HEAVYNODE_ES_VERSIONS=$(salt -C 'G@role:so-heavynode' cmd.run 'set -o pipefail; so-elasticsearch-query / --retry 5 --retry-delay 10 | jq -er ".version.number"' shell=/bin/bash --out=json 2> /dev/null)
             local exit_status=$?
 
@@ -1090,6 +1119,14 @@ verify_es_version_compatibility() {
                 fi
             done < <(echo "$HEAVYNODE_ES_VERSIONS" | jq -r 'to_entries[] | [.key, .value] | @tsv')
 
+            while IFS= read -r heavynode_minion; do
+                [[ -z "$heavynode_minion" ]] && continue
+                if ! echo "$HEAVYNODE_ES_VERSIONS" | jq -e --arg minion "$heavynode_minion" 'has($minion)' > /dev/null; then
+                    echo "Heavynode $heavynode_minion did not report an Elasticsearch version. It may be offline or still upgrading."
+                    all_heavynodes_compatible=false
+                fi
+            done <<< "$heavynode_minions"
+
             if [[ "$all_heavynodes_compatible" == true ]]; then
                 echo -e "\nAll heavynodes can upgrade to Elasticsearch $target_es_version."
                 return 0