diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index feb83fc2f..3bddb5bd5 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -8,6 +8,7 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKERMERGED %} {% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} +{% set SOUP_SKIP_HEALTH_CHECKS = salt['pillar.get']('soup:skip_health_checks', False) %} {# This value is generated during node install and stored in minion pillar #} {% set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %} @@ -113,6 +114,11 @@ wait_for_so-elastic-fleet: - request_interval: 15 - require: - docker_container: so-elastic-fleet +{% if SOUP_SKIP_HEALTH_CHECKS %} + # SOUP starts containers as part of its upgrade highstate; do not spend up to + # 5 minutes waiting for Fleet Server while that upgrade is in progress. + - onlyif: /bin/false +{% endif %} {% endif %} delete_so-elastic-fleet_so-status.disabled: diff --git a/salt/kibana/enabled.sls b/salt/kibana/enabled.sls index 0e986dbc2..965b26bdc 100644 --- a/salt/kibana/enabled.sls +++ b/salt/kibana/enabled.sls @@ -8,6 +8,7 @@ {% from 'docker/docker.map.jinja' import DOCKERMERGED %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} {% from 'vars/globals.map.jinja' import GLOBALS %} +{% set SOUP_SKIP_HEALTH_CHECKS = salt['pillar.get']('soup:skip_health_checks', False) %} include: - kibana.config @@ -74,6 +75,11 @@ wait_for_so-kibana: - request_interval: 15 - require: - docker_container: so-kibana +{% if SOUP_SKIP_HEALTH_CHECKS %} + # SOUP starts containers as part of its upgrade highstate; do not spend up to + # 10 minutes waiting for Kibana while that upgrade is in progress. + - onlyif: /bin/false +{% endif %} delete_so-kibana_so-status.disabled: file.uncomment: diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 2068a8bb6..47c727ea1 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -456,13 +456,19 @@ get_soup_script_hashes() { highstate() { # Run a highstate with a retry attempt. - if salt-call state.highstate -l info queue=True; then + local highstate_cmd=("salt-call state.highstate -l info queue=True") + if [[ "$es_version" != "$target_es_version" ]]; then + echo "Elasticsearch upgrade is scheduled. While the upgrade is in progress, skip health checks for services expected to be temporarily unavailable." + highstate_cmd+=('pillar={"soup":{"skip_health_checks":true}}') + fi + + if "${highstate_cmd[@]}"; then return 0 fi echo "Initial highstate attempt had a problem; retrying in 30 seconds." sleep 30 - salt-call state.highstate -l info queue=True + "${highstate_cmd[@]}" } push_grid_highstate() {