From e4857d5856ebddffa4612a644e2e2522d999f943 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Sun, 23 Aug 2026 23:38:52 -0500 Subject: [PATCH] during in-progress elasticsearch upgrades, hold kibana at current version until es cluster upgrade complete. --- salt/kibana/config.sls | 6 ++++ salt/kibana/enabled.sls | 9 +++++ salt/kibana/files/so-kibana-es-upgrade-ready | 35 ++++++++++++++++++++ salt/manager/tools/sbin/soup | 14 +++++--- 4 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 salt/kibana/files/so-kibana-es-upgrade-ready diff --git a/salt/kibana/config.sls b/salt/kibana/config.sls index 433fc9601..20e1ce2e7 100644 --- a/salt/kibana/config.sls +++ b/salt/kibana/config.sls @@ -63,6 +63,12 @@ kibanaconfig: KIBANACONFIG: {{ KIBANAMERGED.config }} - show_changes: False +so-kibana-es-upgrade-ready: + file.managed: + - name: /usr/sbin/so-kibana-es-upgrade-ready + - source: salt://kibana/files/so-kibana-es-upgrade-ready + - mode: '0755' + kibanalogdir: file.directory: - name: /opt/so/log/kibana diff --git a/salt/kibana/enabled.sls b/salt/kibana/enabled.sls index 0e986dbc2..6806a559d 100644 --- a/salt/kibana/enabled.sls +++ b/salt/kibana/enabled.sls @@ -8,6 +8,7 @@ {% from 'docker/docker.map.jinja' import DOCKERMERGED %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} {% from 'vars/globals.map.jinja' import GLOBALS %} +{% set ELASTICSEARCH_UPGRADED = salt['pillar.get']('elasticsearch:upgraded', True) %} include: - kibana.config @@ -61,6 +62,11 @@ so-kibana: {% endif %} - watch: - file: kibanaconfig + {% if not ELASTICSEARCH_UPGRADED %} + - onlyif: /usr/sbin/so-kibana-es-upgrade-ready {{ ELASTICSEARCHMERGED.version }} + {% endif %} + - require: + - file: so-kibana-es-upgrade-ready wait_for_so-kibana: http.wait_for_successful_query: @@ -72,6 +78,9 @@ wait_for_so-kibana: - status: 200 - wait_for: 600 - request_interval: 15 + {% if not ELASTICSEARCH_UPGRADED %} + - onlyif: /usr/sbin/so-kibana-es-upgrade-ready {{ ELASTICSEARCHMERGED.version }} + {% endif %} - require: - docker_container: so-kibana diff --git a/salt/kibana/files/so-kibana-es-upgrade-ready b/salt/kibana/files/so-kibana-es-upgrade-ready new file mode 100644 index 000000000..c07b0e7a5 --- /dev/null +++ b/salt/kibana/files/so-kibana-es-upgrade-ready @@ -0,0 +1,35 @@ +#!/bin/bash +# +# Start Kibana only when all active Elasticsearch nodes match its configured +# Elasticsearch version. + +PILLAR_FILE=/opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls + +set -o pipefail + +EXPECTED_VERSION=${1:-} +if [[ ! "$EXPECTED_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Usage: so-kibana-es-upgrade-ready " >&2 + exit 1 +fi + +# Require every node active in Kibana's Elasticsearch cluster to be at the +# target version. This matches Kibana's own nodes.info compatibility check. +if ! cluster_versions=$(timeout 90 so-elasticsearch-query _nodes/_all/version --fail 2>/dev/null); then + echo "Kibana is held: unable to query Elasticsearch cluster node versions." >&2 + exit 1 +fi + +if ! jq -e --arg expected "$EXPECTED_VERSION" \ + '(.nodes | length) > 0 and all(.nodes[]; .version == $expected)' \ + <<< "$cluster_versions" > /dev/null; then + echo "Kibana is held: one or more Elasticsearch cluster nodes are not running $EXPECTED_VERSION." >&2 + exit 1 +fi + +if ! /usr/sbin/so-yaml.py replace "$PILLAR_FILE" elasticsearch.upgraded true; then + echo "Kibana is held: unable to mark the Elasticsearch upgrade complete." >&2 + exit 1 +fi + +echo "All active Elasticsearch nodes are running $EXPECTED_VERSION; releasing Kibana." diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 2068a8bb6..c6ce01b18 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1466,7 +1466,7 @@ verify_es_version_compatibility() { done <<< "$expected_es_nodes" if [[ "$all_searchnodes_compatible" == true ]]; then - echo "All Searchnodes are upgradable to Elasticsearch $target_es_version." + echo -e "\t- All Searchnodes are upgradable to Elasticsearch $target_es_version." return 0 fi @@ -1536,11 +1536,11 @@ verify_es_version_compatibility() { done <<< "$heavynode_minions" if [[ "$all_heavynodes_compatible" == true ]]; then - echo -e "\nAll heavynodes can upgrade to Elasticsearch $target_es_version." + echo -e "\t- All heavynodes are upgradable to Elasticsearch $target_es_version." return 0 fi - echo "One or more heavynodes cannot upgrade directly to Elasticsearch $target_es_version. Rechecking in $delay seconds. Attempt $((retry_count + 1)) of $retries." + echo -e "\t- One or more heavynodes cannot upgrade directly to Elasticsearch $target_es_version. Rechecking in $delay seconds. Attempt $((retry_count + 1)) of $retries." ((retry_count++)) sleep $delay done @@ -1976,7 +1976,7 @@ main() { upgrade_check upgrade_space - echo "Verifying Elasticsearch version compatibility across the grid before upgrading." + echo -e "\nVerifying Elasticsearch version compatibility across the grid before upgrading." verify_es_version_compatibility # Pre-flight health checks: confirm the grid is in a good state before we change @@ -2080,6 +2080,12 @@ main() { copy_new_files echo "" create_local_directories "/opt/so/saltstack/default" + if [[ "$es_version" != "$target_es_version" ]]; then + elasticsearch_pillar_file=/opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls + echo "Elasticsearch upgrade scheduled. Kibana will be held at $es_version until cluster has upgraded to $target_es_version." + so-yaml.py replace "$elasticsearch_pillar_file" elasticsearch.upgraded false \ + || fail "Unable to mark the Elasticsearch upgrade as incomplete." + fi # Seed the resume marker before the highstate stamps /etc/soversion to the new # version, so an interrupted upgrade is detectable as "not finished" on re-run. # POSTVERSION still holds the pre-upgrade (or prior resume) version here.