From d33eb70af64f4776a7d8e93a9d6bfefe6e23e797 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:12:38 -0500 Subject: [PATCH 1/7] reverts 83aaa76 #15985 - allow full highstate on manager when locked --- salt/manager/tools/sbin/soup | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index c0927c28a..43fd9b84f 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -483,11 +483,10 @@ push_grid_highstate() { masterlock() { echo "Locking Salt Master" mv -v $TOPFILE $BACKUPTOPFILE - # Render the real top file only for the host running soup; every other - # minion gets an empty top (no states) while the master is upgrading. - echo "{% if grains['id'] == '$MINIONID' %}" > $TOPFILE - cat $BACKUPTOPFILE >> $TOPFILE - echo "{% endif %}" >> $TOPFILE + echo "base:" > $TOPFILE + echo " $MINIONID:" >> $TOPFILE + echo " - ca" >> $TOPFILE + echo " - elasticsearch" >> $TOPFILE } masterunlock() { From fae1754feca56f6d7264f1ea80350806db2e9fcf Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:50:32 -0500 Subject: [PATCH 2/7] clean elasticsearch transform prior to elasticsearch integration package upgrade to prevent fleet automatic rollback --- .../so-elastic-fleet-package-upgrade | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade index d645403d5..1b1b79e56 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade @@ -11,16 +11,48 @@ PKG_LOAD_FAILURES=0 PKG_LOAD_FAILURES_NAMES=() +cleanup_elasticsearch_fleet_transforms() { + local transforms transform_id attempt + + if ! transforms=$(so-elasticsearch-query "_transform/logs-elasticsearch.index_pivot-default-*" --retry 1 --retry-delay 5); then + return 0 + fi + + while IFS= read -r transform_id; do + [ -n "$transform_id" ] || continue + for attempt in {1..3}; do + if so-elasticsearch-query "_transform/$transform_id?force=true" -XDELETE --fail --retry 1 --retry-delay 5; then + break + fi + sleep 5 + done + done < <(jq -r '.transforms[]?.id' <<< "$transforms") +} + {%- for PACKAGE in SUPPORTED_PACKAGES %} if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && LATEST_VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then if [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then echo "{{ PACKAGE }} integration version $INSTALLED_VERSION is already at the reported latest version $LATEST_VERSION, skipping upgrade." else +{%- if PACKAGE == 'elasticsearch' %} + cleanup_elasticsearch_fleet_transforms +{%- endif %} echo "Upgrading {{ PACKAGE }} package from $INSTALLED_VERSION to version $LATEST_VERSION..." if ! elastic_fleet_package_install "{{ PACKAGE }}" "$LATEST_VERSION"; then PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1)) PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}") + # check that package has upgraded to the expected version after install command + elif ! LATEST_VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then + echo "ERROR: Failed to get latest version information for integration {{ PACKAGE }} after upgrade attempt" + PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1)) + PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}") + elif INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then + echo "{{ PACKAGE }} integration upgraded to version $LATEST_VERSION." + else + echo "ERROR: {{ PACKAGE }} integration still at ${INSTALLED_VERSION:-unknown}; expected $LATEST_VERSION" + PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1)) + PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}") fi fi else From cb8e576d6b70116189ebc21eb82deebb57cbab8d Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:52:59 -0500 Subject: [PATCH 3/7] run elasticsearch state on remote minions when there is an ES upgrade. Prior to manager completing its first full highstate that includes kibana / elasticfleet --- salt/manager/tools/sbin/soup | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 43fd9b84f..3399f8e07 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -455,6 +455,19 @@ highstate() { salt-call state.highstate -l info queue=True } +highstate_elastic() { + # Run the elasticsearch state across the true elastic cluster (non-heavy) with a retry attempt + # Excludes the manager, so that kibana & elasticfleet are not upgraded until searchnodes are upgraded. + echo "Getting ready to upgrade Elasticsearch across the grid. This may take a while..." + if salt -C "I@elasticsearch:enabled and not G@role:so-heavynode and not G@id:${MINIONID}" state.apply elasticsearch queue=True batch=10%; then + return 0 + fi + + echo "Initial elasticsearch state attempt had a problem; retrying in 30 seconds." + sleep 30 + salt -C "I@elasticsearch:enabled and not G@role:so-heavynode and not G@id:${MINIONID}" state.apply elasticsearch queue=True batch=10% +} + push_grid_highstate() { # Drive a batched, role-tiered highstate across the rest of the grid so remote minions # pick up this upgrade now instead of waiting up to ~2.5 hours for their own scheduled @@ -2117,7 +2130,17 @@ main() { # ensure the mine is updated and populated before highstates run, following the salt-master restart update_salt_mine - + + # kick off a searchnode elasticsearch upgrade + set +e + if [[ "$es_version" != "$target_es_version" ]]; then + if salt-key -L accepted | grep -q "_searchnode$" 2>/dev/null; then + # only run if there is atleast 1 searchnode + highstate_elastic + fi + fi + set -e + highstate check_saltmaster_status postupgrade_changes From 85ab4c69e52f8767c68a0020205fab695ac091c3 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:17:29 -0500 Subject: [PATCH 4/7] rename --- salt/manager/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 3399f8e07..672941bee 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -455,7 +455,7 @@ highstate() { salt-call state.highstate -l info queue=True } -highstate_elastic() { +upgrade_searchnode_elasticsearch() { # Run the elasticsearch state across the true elastic cluster (non-heavy) with a retry attempt # Excludes the manager, so that kibana & elasticfleet are not upgraded until searchnodes are upgraded. echo "Getting ready to upgrade Elasticsearch across the grid. This may take a while..." @@ -2136,7 +2136,7 @@ main() { if [[ "$es_version" != "$target_es_version" ]]; then if salt-key -L accepted | grep -q "_searchnode$" 2>/dev/null; then # only run if there is atleast 1 searchnode - highstate_elastic + upgrade_searchnode_elasticsearch fi fi set -e From d771fbc444b9d84415a68d1665211870d32ed0c3 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:18:39 -0500 Subject: [PATCH 5/7] upgrade integration policies directly after integration package upgrade --- salt/elasticfleet/manager.sls | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/salt/elasticfleet/manager.sls b/salt/elasticfleet/manager.sls index 4ae64314b..fbef61e1f 100644 --- a/salt/elasticfleet/manager.sls +++ b/salt/elasticfleet/manager.sls @@ -68,6 +68,15 @@ so-elastic-fleet-package-upgrade: - require: - http: wait_for_so-kibana +so-elastic-fleet-integration-upgrade: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-integration-upgrade + - retry: + attempts: 3 + interval: 10 + - require: + - cmd: so-elastic-fleet-package-upgrade + so-elastic-fleet-integrations: cmd.run: - name: /usr/sbin/so-elastic-fleet-integration-policy-load @@ -86,15 +95,6 @@ so-elastic-agent-grid-upgrade: - require: - http: wait_for_so-kibana -so-elastic-fleet-integration-upgrade: - cmd.run: - - name: /usr/sbin/so-elastic-fleet-integration-upgrade - - retry: - attempts: 3 - interval: 10 - - require: - - http: wait_for_so-kibana - {# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #} so-elastic-fleet-addon-integrations: cmd.run: From 4786d359fb2b0b76098e0e632d43973b6109528b Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:39:46 -0500 Subject: [PATCH 6/7] exclude telegraf error during elasticsearch upgrade / master election --- salt/common/tools/sbin/so-log-check | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 4a8a1db14..29cf3708c 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -134,6 +134,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Redis may have been restarted" # Redis likely restarted by salt EXCLUDED_ERRORS="$EXCLUDED_ERRORS|file already closed" # Go logging race condition during container restart EXCLUDED_ERRORS="$EXCLUDED_ERRORS|relation \"audit_settings\" does not exist" # salt checking for changes before SOC starts + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error in plugin: elasticsearch: Unable to retrieve master node information" # expected error while ES is upgrading/electing a master fi if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then From f8401bef374c55aa7ca2de3f728e42afbd1a917d Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 27 Aug 2026 21:17:27 -0500 Subject: [PATCH 7/7] exclude elasticsearch indexing error during upgrade for temporarily outdated policies --- salt/common/tools/sbin/so-log-check | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 29cf3708c..39f36eacc 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -241,6 +241,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459 EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown" # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1 + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Could not index event to Elasticsearch.*\"version\" => \"9.0.8\"" # Expected during Elastic upgrade temporarily, as policies referencing older pipelines are updated fi RESULT=0 @@ -305,4 +306,4 @@ else echo -e "\nResult: One or more errors found" fi -exit $RESULT +exit $RESULT \ No newline at end of file