disable kibana / elasticfleet health checks during soup

salt/elasticfleet/enabled.sls

@@ -8,6 +8,7 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+{%   set SOUP_SKIP_HEALTH_CHECKS = salt['pillar.get']('soup:skip_health_checks', False) %}
 
 {#   This value is generated during node install and stored in minion pillar #}
 {%   set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %}
@@ -113,6 +114,11 @@ wait_for_so-elastic-fleet:
     - request_interval: 15
     - require:
       - docker_container: so-elastic-fleet
+{% if SOUP_SKIP_HEALTH_CHECKS %}
+    # SOUP starts containers as part of its upgrade highstate; do not spend up to
+    # 5 minutes waiting for Fleet Server while that upgrade is in progress.
+    - onlyif: /bin/false
+{% endif %}
 {%   endif %}
 
 delete_so-elastic-fleet_so-status.disabled:

salt/kibana/enabled.sls

@@ -8,6 +8,7 @@
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   set SOUP_SKIP_HEALTH_CHECKS = salt['pillar.get']('soup:skip_health_checks', False) %}
 
 include:
   - kibana.config
@@ -74,6 +75,11 @@ wait_for_so-kibana:
     - request_interval: 15
     - require:
       - docker_container: so-kibana
+{% if SOUP_SKIP_HEALTH_CHECKS %}
+    # SOUP starts containers as part of its upgrade highstate; do not spend up to
+    # 10 minutes waiting for Kibana while that upgrade is in progress.
+    - onlyif: /bin/false
+{% endif %}
 
 delete_so-kibana_so-status.disabled:
   file.uncomment:

salt/manager/tools/sbin/soup

@@ -456,13 +456,19 @@ get_soup_script_hashes() {
 
 highstate() {
   # Run a highstate with a retry attempt.
-  if salt-call state.highstate -l info queue=True; then
+  local highstate_cmd=("salt-call state.highstate -l info queue=True")
+  if [[ "$es_version" != "$target_es_version" ]]; then
+    echo "Elasticsearch upgrade is scheduled. While the upgrade is in progress, skip health checks for services expected to be temporarily unavailable."
+    highstate_cmd+=('pillar={"soup":{"skip_health_checks":true}}')
+  fi
+  
+  if "${highstate_cmd[@]}"; then
     return 0
   fi
 
   echo "Initial highstate attempt had a problem; retrying in 30 seconds."
   sleep 30
-  salt-call state.highstate -l info queue=True
+  "${highstate_cmd[@]}"
 }
 
 push_grid_highstate() {