during in-progress elasticsearch upgrades, hold kibana at current version until es cluster upgrade complete.

salt/kibana/config.sls

@@ -63,6 +63,12 @@ kibanaconfig:
         KIBANACONFIG: {{ KIBANAMERGED.config }}
     - show_changes: False
 
+so-kibana-es-upgrade-ready:
+  file.managed:
+    - name: /usr/sbin/so-kibana-es-upgrade-ready
+    - source: salt://kibana/files/so-kibana-es-upgrade-ready
+    - mode: '0755'
+
 kibanalogdir:
   file.directory:
     - name: /opt/so/log/kibana

salt/kibana/enabled.sls

@@ -8,6 +8,7 @@
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   set ELASTICSEARCH_UPGRADED = salt['pillar.get']('elasticsearch:upgraded', True) %}
 
 include:
   - kibana.config
@@ -61,6 +62,11 @@ so-kibana:
     {% endif %}
     - watch:
       - file: kibanaconfig
+    {% if not ELASTICSEARCH_UPGRADED %}
+    - onlyif: /usr/sbin/so-kibana-es-upgrade-ready {{ ELASTICSEARCHMERGED.version }}
+    {% endif %}
+    - require:
+      - file: so-kibana-es-upgrade-ready
 
 wait_for_so-kibana:
   http.wait_for_successful_query:
@@ -72,6 +78,9 @@ wait_for_so-kibana:
     - status: 200
     - wait_for: 600
     - request_interval: 15
+    {% if not ELASTICSEARCH_UPGRADED %}
+    - onlyif: /usr/sbin/so-kibana-es-upgrade-ready {{ ELASTICSEARCHMERGED.version }}
+    {% endif %}
     - require:
       - docker_container: so-kibana
 

salt/kibana/files/so-kibana-es-upgrade-ready

@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# Start Kibana only when all active Elasticsearch nodes match its configured
+# Elasticsearch version.
+
+PILLAR_FILE=/opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls
+
+set -o pipefail
+
+EXPECTED_VERSION=${1:-}
+if [[ ! "$EXPECTED_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+  echo "Usage: so-kibana-es-upgrade-ready <Elasticsearch version>" >&2
+  exit 1
+fi
+
+# Require every node active in Kibana's Elasticsearch cluster to be at the
+# target version. This matches Kibana's own nodes.info compatibility check.
+if ! cluster_versions=$(timeout 90 so-elasticsearch-query _nodes/_all/version --fail 2>/dev/null); then
+  echo "Kibana is held: unable to query Elasticsearch cluster node versions." >&2
+  exit 1
+fi
+
+if ! jq -e --arg expected "$EXPECTED_VERSION" \
+    '(.nodes | length) > 0 and all(.nodes[]; .version == $expected)' \
+    <<< "$cluster_versions" > /dev/null; then
+  echo "Kibana is held: one or more Elasticsearch cluster nodes are not running $EXPECTED_VERSION." >&2
+  exit 1
+fi
+
+if ! /usr/sbin/so-yaml.py replace "$PILLAR_FILE" elasticsearch.upgraded true; then
+  echo "Kibana is held: unable to mark the Elasticsearch upgrade complete." >&2
+  exit 1
+fi
+
+echo "All active Elasticsearch nodes are running $EXPECTED_VERSION; releasing Kibana."

salt/manager/tools/sbin/soup

@@ -1466,7 +1466,7 @@ verify_es_version_compatibility() {
             done <<< "$expected_es_nodes"
 
             if [[ "$all_searchnodes_compatible" == true ]]; then
-                echo "All Searchnodes are upgradable to Elasticsearch $target_es_version."
+                echo -e "\t- All Searchnodes are upgradable to Elasticsearch $target_es_version."
                 return 0
             fi
 
@@ -1536,11 +1536,11 @@ verify_es_version_compatibility() {
             done <<< "$heavynode_minions"
 
             if [[ "$all_heavynodes_compatible" == true ]]; then
-                echo -e "\nAll heavynodes can upgrade to Elasticsearch $target_es_version."
+                echo -e "\t- All heavynodes are upgradable to Elasticsearch $target_es_version."
                 return 0
             fi
 
-            echo "One or more heavynodes cannot upgrade directly to Elasticsearch $target_es_version. Rechecking in $delay seconds. Attempt $((retry_count + 1)) of $retries."
+            echo -e "\t- One or more heavynodes cannot upgrade directly to Elasticsearch $target_es_version. Rechecking in $delay seconds. Attempt $((retry_count + 1)) of $retries."
             ((retry_count++))
             sleep $delay
         done
@@ -1976,7 +1976,7 @@ main() {
   upgrade_check
   upgrade_space
 
-  echo "Verifying Elasticsearch version compatibility across the grid before upgrading."
+  echo -e "\nVerifying Elasticsearch version compatibility across the grid before upgrading."
   verify_es_version_compatibility
 
   # Pre-flight health checks: confirm the grid is in a good state before we change
@@ -2080,6 +2080,12 @@ main() {
     copy_new_files
     echo ""
     create_local_directories "/opt/so/saltstack/default"
+    if [[ "$es_version" != "$target_es_version" ]]; then
+      elasticsearch_pillar_file=/opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls
+      echo "Elasticsearch upgrade scheduled. Kibana will be held at $es_version until cluster has upgraded to $target_es_version."
+      so-yaml.py replace "$elasticsearch_pillar_file" elasticsearch.upgraded false \
+        || fail "Unable to mark the Elasticsearch upgrade as incomplete."
+    fi
     # Seed the resume marker before the highstate stamps /etc/soversion to the new
     # version, so an interrupted upgrade is detectable as "not finished" on re-run.
     # POSTVERSION still holds the pre-upgrade (or prior resume) version here.