diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 4a8a1db14..39f36eacc 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -134,6 +134,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Redis may have been restarted" # Redis likely restarted by salt EXCLUDED_ERRORS="$EXCLUDED_ERRORS|file already closed" # Go logging race condition during container restart EXCLUDED_ERRORS="$EXCLUDED_ERRORS|relation \"audit_settings\" does not exist" # salt checking for changes before SOC starts + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error in plugin: elasticsearch: Unable to retrieve master node information" # expected error while ES is upgrading/electing a master fi if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then @@ -240,6 +241,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459 EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown" # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1 + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Could not index event to Elasticsearch.*\"version\" => \"9.0.8\"" # Expected during Elastic upgrade temporarily, as policies referencing older pipelines are updated fi RESULT=0 @@ -304,4 +306,4 @@ else echo -e "\nResult: One or more errors found" fi -exit $RESULT +exit $RESULT \ No newline at end of file diff --git a/salt/elasticfleet/manager.sls b/salt/elasticfleet/manager.sls index 4ae64314b..fbef61e1f 100644 --- a/salt/elasticfleet/manager.sls +++ b/salt/elasticfleet/manager.sls @@ -68,6 +68,15 @@ so-elastic-fleet-package-upgrade: - require: - http: wait_for_so-kibana +so-elastic-fleet-integration-upgrade: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-integration-upgrade + - retry: + attempts: 3 + interval: 10 + - require: + - cmd: so-elastic-fleet-package-upgrade + so-elastic-fleet-integrations: cmd.run: - name: /usr/sbin/so-elastic-fleet-integration-policy-load @@ -86,15 +95,6 @@ so-elastic-agent-grid-upgrade: - require: - http: wait_for_so-kibana -so-elastic-fleet-integration-upgrade: - cmd.run: - - name: /usr/sbin/so-elastic-fleet-integration-upgrade - - retry: - attempts: 3 - interval: 10 - - require: - - http: wait_for_so-kibana - {# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #} so-elastic-fleet-addon-integrations: cmd.run: diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade index d645403d5..1b1b79e56 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade @@ -11,16 +11,48 @@ PKG_LOAD_FAILURES=0 PKG_LOAD_FAILURES_NAMES=() +cleanup_elasticsearch_fleet_transforms() { + local transforms transform_id attempt + + if ! transforms=$(so-elasticsearch-query "_transform/logs-elasticsearch.index_pivot-default-*" --retry 1 --retry-delay 5); then + return 0 + fi + + while IFS= read -r transform_id; do + [ -n "$transform_id" ] || continue + for attempt in {1..3}; do + if so-elasticsearch-query "_transform/$transform_id?force=true" -XDELETE --fail --retry 1 --retry-delay 5; then + break + fi + sleep 5 + done + done < <(jq -r '.transforms[]?.id' <<< "$transforms") +} + {%- for PACKAGE in SUPPORTED_PACKAGES %} if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && LATEST_VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then if [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then echo "{{ PACKAGE }} integration version $INSTALLED_VERSION is already at the reported latest version $LATEST_VERSION, skipping upgrade." else +{%- if PACKAGE == 'elasticsearch' %} + cleanup_elasticsearch_fleet_transforms +{%- endif %} echo "Upgrading {{ PACKAGE }} package from $INSTALLED_VERSION to version $LATEST_VERSION..." if ! elastic_fleet_package_install "{{ PACKAGE }}" "$LATEST_VERSION"; then PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1)) PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}") + # check that package has upgraded to the expected version after install command + elif ! LATEST_VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then + echo "ERROR: Failed to get latest version information for integration {{ PACKAGE }} after upgrade attempt" + PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1)) + PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}") + elif INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then + echo "{{ PACKAGE }} integration upgraded to version $LATEST_VERSION." + else + echo "ERROR: {{ PACKAGE }} integration still at ${INSTALLED_VERSION:-unknown}; expected $LATEST_VERSION" + PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1)) + PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}") fi fi else diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 43fd9b84f..672941bee 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -455,6 +455,19 @@ highstate() { salt-call state.highstate -l info queue=True } +upgrade_searchnode_elasticsearch() { + # Run the elasticsearch state across the true elastic cluster (non-heavy) with a retry attempt + # Excludes the manager, so that kibana & elasticfleet are not upgraded until searchnodes are upgraded. + echo "Getting ready to upgrade Elasticsearch across the grid. This may take a while..." + if salt -C "I@elasticsearch:enabled and not G@role:so-heavynode and not G@id:${MINIONID}" state.apply elasticsearch queue=True batch=10%; then + return 0 + fi + + echo "Initial elasticsearch state attempt had a problem; retrying in 30 seconds." + sleep 30 + salt -C "I@elasticsearch:enabled and not G@role:so-heavynode and not G@id:${MINIONID}" state.apply elasticsearch queue=True batch=10% +} + push_grid_highstate() { # Drive a batched, role-tiered highstate across the rest of the grid so remote minions # pick up this upgrade now instead of waiting up to ~2.5 hours for their own scheduled @@ -2117,7 +2130,17 @@ main() { # ensure the mine is updated and populated before highstates run, following the salt-master restart update_salt_mine - + + # kick off a searchnode elasticsearch upgrade + set +e + if [[ "$es_version" != "$target_es_version" ]]; then + if salt-key -L accepted | grep -q "_searchnode$" 2>/dev/null; then + # only run if there is atleast 1 searchnode + upgrade_searchnode_elasticsearch + fi + fi + set -e + highstate check_saltmaster_status postupgrade_changes