diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 4a8a1db14..39f36eacc 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -134,6 +134,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Redis may have been restarted" # Redis likely restarted by salt EXCLUDED_ERRORS="$EXCLUDED_ERRORS|file already closed" # Go logging race condition during container restart EXCLUDED_ERRORS="$EXCLUDED_ERRORS|relation \"audit_settings\" does not exist" # salt checking for changes before SOC starts + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error in plugin: elasticsearch: Unable to retrieve master node information" # expected error while ES is upgrading/electing a master fi if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then @@ -240,6 +241,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459 EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown" # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1 + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Could not index event to Elasticsearch.*\"version\" => \"9.0.8\"" # Expected during Elastic upgrade temporarily, as policies referencing older pipelines are updated fi RESULT=0 @@ -304,4 +306,4 @@ else echo -e "\nResult: One or more errors found" fi -exit $RESULT +exit $RESULT \ No newline at end of file diff --git a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json index 7d64fd1ab..ecb522ce9 100644 --- a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json +++ b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json @@ -5,7 +5,7 @@ "package": { "name": "endpoint", "title": "Elastic Defend", - "version": "9.3.1", + "version": "9.4.1", "requires_root": true }, "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index be05d59e0..723370ba6 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -29,7 +29,7 @@ "\\.gz$" ], "include_files": [], - "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.20.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.8.3\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.20.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.20.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.8.3\n- add_fields:\n target: data_stream\n fields:\n dataset: import", + "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.22.3\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.9.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.22.3\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.22.3\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.9.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import", "tags": [ "import" ], diff --git a/salt/elasticfleet/integration-defaults.map.jinja b/salt/elasticfleet/integration-defaults.map.jinja index eeb85123a..25e307331 100644 --- a/salt/elasticfleet/integration-defaults.map.jinja +++ b/salt/elasticfleet/integration-defaults.map.jinja @@ -16,7 +16,6 @@ 'awsfirehose.metrics': 'aws.cloudwatch', 'cribl.logs': 'cribl', 'cribl.metrics': 'cribl', - 'sentinel_one_cloud_funnel.logins': 'sentinel_one_cloud_funnel.login', 'azure_application_insights.app_insights': 'azure.app_insights', 'azure_application_insights.app_state': 'azure.app_state', 'azure_billing.billing': 'azure.billing', diff --git a/salt/elasticfleet/manager.sls b/salt/elasticfleet/manager.sls index 4ae64314b..9a27ea6e9 100644 --- a/salt/elasticfleet/manager.sls +++ b/salt/elasticfleet/manager.sls @@ -68,6 +68,24 @@ so-elastic-fleet-package-upgrade: - require: - http: wait_for_so-kibana +# initial so-elasticsearch-templates run is earlier, but it can skip over templates that have component templates not yet installed to avoid elasticsearch rejecting the template. +so-elasticsearch-templates-after-fleet-packages: + cmd.run: + - name: /usr/sbin/so-elasticsearch-templates-load + - cwd: /opt/so + - unless: test -f /opt/so/state/estemplates.txt + - require: + - cmd: so-elastic-fleet-package-upgrade + +so-elastic-fleet-integration-upgrade: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-integration-upgrade + - retry: + attempts: 3 + interval: 10 + - require: + - cmd: so-elastic-fleet-package-upgrade + so-elastic-fleet-integrations: cmd.run: - name: /usr/sbin/so-elastic-fleet-integration-policy-load @@ -86,21 +104,13 @@ so-elastic-agent-grid-upgrade: - require: - http: wait_for_so-kibana -so-elastic-fleet-integration-upgrade: - cmd.run: - - name: /usr/sbin/so-elastic-fleet-integration-upgrade - - retry: - attempts: 3 - interval: 10 - - require: - - http: wait_for_so-kibana - {# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #} so-elastic-fleet-addon-integrations: cmd.run: - name: /usr/sbin/so-elastic-fleet-optional-integrations-load - require: - http: wait_for_so-kibana + - cmd: so-elasticsearch-templates-after-fleet-packages {% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %} so-elastic-defend-manage-filters-file-watch: diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade index d645403d5..8a63859e5 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade @@ -10,6 +10,25 @@ PKG_LOAD_FAILURES=0 PKG_LOAD_FAILURES_NAMES=() +PKG_UPGRADED=0 + +cleanup_elasticsearch_fleet_transforms() { + local transforms transform_id attempt + + if ! transforms=$(so-elasticsearch-query "_transform/logs-elasticsearch.index_pivot-default-*" --retry 1 --retry-delay 5); then + return 0 + fi + + while IFS= read -r transform_id; do + [ -n "$transform_id" ] || continue + for attempt in {1..3}; do + if so-elasticsearch-query "_transform/$transform_id?force=true" -XDELETE --fail --retry 1 --retry-delay 5; then + break + fi + sleep 5 + done + done < <(jq -r '.transforms[]?.id' <<< "$transforms") +} {%- for PACKAGE in SUPPORTED_PACKAGES %} if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && LATEST_VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then @@ -17,10 +36,25 @@ if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && L if [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then echo "{{ PACKAGE }} integration version $INSTALLED_VERSION is already at the reported latest version $LATEST_VERSION, skipping upgrade." else +{%- if PACKAGE == 'elasticsearch' %} + cleanup_elasticsearch_fleet_transforms +{%- endif %} echo "Upgrading {{ PACKAGE }} package from $INSTALLED_VERSION to version $LATEST_VERSION..." if ! elastic_fleet_package_install "{{ PACKAGE }}" "$LATEST_VERSION"; then PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1)) PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}") + # check that package has upgraded to the expected version after install command + elif ! LATEST_VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then + echo "ERROR: Failed to get latest version information for integration {{ PACKAGE }} after upgrade attempt" + PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1)) + PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}") + elif INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then + echo "{{ PACKAGE }} integration upgraded to version $LATEST_VERSION." + PKG_UPGRADED=$((PKG_UPGRADED + 1)) + else + echo "ERROR: {{ PACKAGE }} integration still at ${INSTALLED_VERSION:-unknown}; expected $LATEST_VERSION" + PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1)) + PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}") fi fi else @@ -30,6 +64,11 @@ else fi {%- endfor %} +if [ $PKG_UPGRADED -gt 0 ]; then + echo "Elasticsearch template statefiles cleared after $PKG_UPGRADED package upgrade(s), so templates can reload." + rm -f /opt/so/state/estemplates.txt /opt/so/state/addon_estemplates.txt +fi + if [ $PKG_LOAD_FAILURES -gt 0 ]; then echo "ERROR: Failed to upgrade $PKG_LOAD_FAILURES package(s):" for PKG in "${PKG_LOAD_FAILURES_NAMES[@]}"; do diff --git a/salt/elasticsearch/cluster.sls b/salt/elasticsearch/cluster.sls index a8ccd3780..45dcf4111 100644 --- a/salt/elasticsearch/cluster.sls +++ b/salt/elasticsearch/cluster.sls @@ -98,6 +98,13 @@ so-es-cluster-settings: - docker_container: so-elasticsearch - file: elasticsearch_sbin_jinja - http: wait_for_so-elasticsearch + +so-elasticsearch-system-indices-patch: + cmd.run: + - name: /usr/sbin/so-elasticsearch-system-indices-patch + - require: + - http: wait_for_so-elasticsearch + - file: so-elasticsearch-system-indices-patch-script {% endif %} # heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml) diff --git a/salt/elasticsearch/config.sls b/salt/elasticsearch/config.sls index 8a4674c71..cbfdca406 100644 --- a/salt/elasticsearch/config.sls +++ b/salt/elasticsearch/config.sls @@ -42,6 +42,16 @@ elasticsearch_sbin: - file_mode: 755 - exclude_pat: - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state + - so-elasticsearch-system-indices-patch + - show_changes: False + +so-elasticsearch-system-indices-patch-script: + file.managed: + - name: /usr/sbin/so-elasticsearch-system-indices-patch + - source: salt://elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch + - user: 930 + - group: 939 + - mode: 755 - show_changes: False elasticsearch_sbin_jinja: diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 80c5e012f..f67a01df9 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,7 +1,7 @@ elasticsearch: enabled: false esheap: '600m' - version: 9.3.7 + version: 9.4.5 index_clean: true data_retention_method: DLM vm: diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch b/salt/elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch new file mode 100644 index 000000000..578485bd8 --- /dev/null +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch @@ -0,0 +1,199 @@ +#!/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +set -eo pipefail + +SETTINGS='{"index":{"auto_expand_replicas":"0-1"}}' +KIBANA_PASSWORD= +INDEX_PATTERNS=( + '.entity_analytics.risk_score.lookup-*' + '.entity_analytics.watchlists.*' + '.entity_analytics.monitoring.users-*' + '.entity_analytics.entity-leads-*' + '.asset-criticality.asset-criticality-*' + '.workflows-executions' + '.workflows-step-executions' + '.entities.v2.latest.security_*' + '.entities.v2.history.security_*' + 'risk-score.risk-score-latest-*' +) +DATA_STREAM_PATTERNS=( + '.entities.v2.updates.security_*' + 'risk-score.risk-score-*' + '.rule-events' + '.alert-actions' +) +TEMPLATE_PATTERNS=( + 'entities_v2_latest_security_default_index_template' + 'entities_v2_history_security_default_index_template' + '.entities_v2_updates_security_default_index_template' + '.risk-score.risk-score-default-index-template' + '.rule-events' + '.alert-actions' +) + +query_es() { + if so-elasticsearch-query "$@" --fail --retry 3 --retry-delay 5; then + return 0 + fi + + # retry failed attempts with so_kibana user (system managed indices reject so_elastic user) + local query_path="$1" + shift + + if [[ -z "$KIBANA_PASSWORD" ]]; then + KIBANA_PASSWORD=$(salt-call pillar.get elasticsearch:auth:users:so_kibana_user:pass --out=newline_values_only) + fi + [[ -n "$KIBANA_PASSWORD" ]] || return 1 + + echo "Retrying ${query_path} as so_kibana." >&2 + curl -K /opt/so/conf/elasticsearch/curl.config --user "so_kibana:${KIBANA_PASSWORD}" \ + -s -k -L --fail --retry 3 --retry-delay 5 -H 'Content-Type: application/json' "https://localhost:9200/${query_path}" "$@" +} + +# add auto_expand_replicas=0-1 to given index +set_auto_expand_replicas() { + local index="$1" + + echo "Setting auto_expand_replicas to 0-1 on ${index}." + query_es "${index}/_settings" -XPUT -d "$SETTINGS" >/dev/null +} + +# resolve index patterns and find each index with an unassigned replica +unassigned_replicas() { + local pattern="$1" + local resolved_indices response index + + if ! resolved_indices=$(query_es "_resolve/index/${pattern}?expand_wildcards=all" 2>/dev/null); then + return 0 + fi + + while read -r index; do + if ! response=$(query_es "_cat/shards/${index}?format=json&h=index,prirep,state" 2>/dev/null); then + continue + fi + jq -r '.[]? | objects | select(.prirep == "r" and .state == "UNASSIGNED") | .index' <<<"$response" + done < <(jq -r '.indices[]?.name' <<<"$resolved_indices") +} + +data_stream_indices() { + local pattern="$1" + local response + + if ! response=$(query_es "_data_stream/${pattern}?expand_wildcards=all" 2>/dev/null); then + return 0 + fi + jq -r '.data_streams[]?.indices[]?.index_name' <<<"$response" +} + +update_system_indices() { + local pattern="$1" + local index + + while read -r index; do + [[ -n "$index" ]] && set_auto_expand_replicas "$index" + done < <(unassigned_replicas "$pattern") +} + +# update data stream backing indices with unassigned replicas +update_system_ds() { + local pattern="$1" + local index + + while read -r index; do + while read -r unassigned_index; do + [[ -n "$unassigned_index" ]] && set_auto_expand_replicas "$unassigned_index" + done < <(unassigned_replicas "$index") + done < <(data_stream_indices "$pattern") +} + +has_unassigned_replicas() { + local pattern="$1" + local index + + index=$(unassigned_replicas "$pattern" | sed -n '1p') + [[ -n "$index" ]] +} + +data_stream_has_unassigned_replicas() { + local pattern="$1" + local index + while read -r index; do + has_unassigned_replicas "$index" && return 0 + done < <(data_stream_indices "$pattern") + + return 1 +} + +needs_patch() { + local pattern + for pattern in "${INDEX_PATTERNS[@]}"; do + has_unassigned_replicas "$pattern" && return 0 + done + for pattern in "${DATA_STREAM_PATTERNS[@]}"; do + data_stream_has_unassigned_replicas "$pattern" && return 0 + done + + return 1 +} + +# get index templates, update with auto_expand_replicas=0-1, and PUT back. Keeping mappings/settings/aliases in-place +update_system_templates() { + local pattern="$1" + local templates name response template auto_expand_replicas + + if ! templates=$(query_es "_index_template/${pattern}" 2>/dev/null); then + return 0 + fi + while read -r name; do + response=$(query_es "_index_template/${name}") + template=$(jq -c '.index_templates[0].index_template' <<<"$response") + auto_expand_replicas=$(jq -r '.template.settings["index.auto_expand_replicas"] // .template.settings.index.auto_expand_replicas // empty' <<<"$template") + [[ "$auto_expand_replicas" == "0-1" ]] && continue + + template=$(jq ' + if (.template.settings.index | type) == "object" then + .template.settings.index.auto_expand_replicas = "0-1" + else + .template.settings["index.auto_expand_replicas"] = "0-1" + end + | del(.created_date_millis, .modified_date_millis) + ' <<<"$template") + echo "Setting auto_expand_replicas to 0-1 on index template ${name}." + query_es "_index_template/${name}" -XPUT -d "$template" >/dev/null + done < <(jq -r '.index_templates[]?.name' <<<"$templates") +} + +if [[ "${1:-}" == "--check" ]]; then + needs_patch + exit $? +fi + +if [[ $# -ne 0 ]]; then + echo "Usage: $0 [--check]" >&2 + exit 1 +fi + +patched=false +for pattern in "${INDEX_PATTERNS[@]}"; do + if has_unassigned_replicas "$pattern"; then + update_system_indices "$pattern" + patched=true + fi +done + +for pattern in "${DATA_STREAM_PATTERNS[@]}"; do + if data_stream_has_unassigned_replicas "$pattern"; then + update_system_ds "$pattern" + patched=true + fi +done + +if [[ "$patched" == true ]]; then + for pattern in "${TEMPLATE_PATTERNS[@]}"; do + update_system_templates "$pattern" + done +fi diff --git a/salt/kibana/defaults.yaml b/salt/kibana/defaults.yaml index 2cf6fe92f..a4b141e7d 100644 --- a/salt/kibana/defaults.yaml +++ b/salt/kibana/defaults.yaml @@ -22,7 +22,7 @@ kibana: - default - file migrations: - discardCorruptObjects: "9.3.7" + discardCorruptObjects: "9.4.5" telemetry: enabled: False xpack: diff --git a/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults b/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults index d0447f514..dd92253ab 100755 --- a/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults +++ b/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults @@ -9,5 +9,5 @@ SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http: # Disable certain Features from showing up in the Kibana UI echo echo "Setting up default Kibana Space:" -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV1","entityManager","streams","cloudConnect","slo"]} ' >> /opt/so/log/kibana/misc.log +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","securitySolutionCasesV3","inventory","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV4","securitySolutionAlertsV1","entityManager","slo","streams","anonymization","searchInferenceEndpoints","cloudConnect","queryActivity","automatic_import","stackAlerts","monitoring","dataQuality","actions"]} ' >> /opt/so/log/kibana/misc.log echo diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index b6be60709..eec561bd5 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -455,6 +455,19 @@ highstate() { salt-call state.highstate -l info queue=True } +upgrade_searchnode_elasticsearch() { + # Run the elasticsearch state across the true elastic cluster (non-heavy) with a retry attempt + # Excludes the manager, so that kibana & elasticfleet are not upgraded until searchnodes are upgraded. + echo "Getting ready to upgrade Elasticsearch across the grid. This may take a while..." + if salt -C "I@elasticsearch:enabled and not G@role:so-heavynode and not G@id:${MINIONID}" state.apply elasticsearch queue=True batch=10%; then + return 0 + fi + + echo "Initial elasticsearch state attempt had a problem; retrying in 30 seconds." + sleep 30 + salt -C "I@elasticsearch:enabled and not G@role:so-heavynode and not G@id:${MINIONID}" state.apply elasticsearch queue=True batch=10% +} + push_grid_highstate() { # Drive a batched, role-tiered highstate across the rest of the grid so remote minions # pick up this upgrade now instead of waiting up to ~2.5 hours for their own scheduled @@ -483,11 +496,10 @@ push_grid_highstate() { masterlock() { echo "Locking Salt Master" mv -v $TOPFILE $BACKUPTOPFILE - # Render the real top file only for the host running soup; every other - # minion gets an empty top (no states) while the master is upgrading. - echo "{% if grains['id'] == '$MINIONID' %}" > $TOPFILE - cat $BACKUPTOPFILE >> $TOPFILE - echo "{% endif %}" >> $TOPFILE + echo "base:" > $TOPFILE + echo " $MINIONID:" >> $TOPFILE + echo " - ca" >> $TOPFILE + echo " - elasticsearch" >> $TOPFILE } masterunlock() { @@ -1337,11 +1349,12 @@ verify_es_version_compatibility() { local is_active_intermediate_upgrade=1 # supported upgrade paths for SO-ES versions declare -A es_upgrade_map=( - ["8.18.4"]="8.18.6 8.18.8 9.0.8" + ["8.18.4"]="8.18.6 8.18.8 9.0.8" ["8.18.6"]="8.18.8 9.0.8" ["8.18.8"]="9.0.8" - ["9.0.8"]="9.3.3 9.3.7" - ["9.3.3"]="9.3.7" + ["9.0.8"]="9.3.3 9.3.7 9.4.5" + ["9.3.3"]="9.3.7 9.4.5" + ["9.3.7"]="9.4.5" ) # Elasticsearch MUST upgrade through these versions @@ -2117,12 +2130,28 @@ main() { # ensure the mine is updated and populated before highstates run, following the salt-master restart update_salt_mine - + + # kick off a searchnode elasticsearch upgrade + set +e + if [[ "$es_version" != "$target_es_version" ]]; then + if salt-key -L accepted | grep -q "_searchnode$" 2>/dev/null; then + # only run if there is atleast 1 searchnode + upgrade_searchnode_elasticsearch + fi + fi + set -e + highstate check_saltmaster_status postupgrade_changes [[ $is_airgap -eq 0 ]] && unmount_update - + + if [[ "$es_version" != "$target_es_version" ]]; then + # Run final elasticsearch / fleet state on manager to ensure addon index templates are created/regenerated and loaded + echo "Running final Elastic states at $(date +"%T.%6N"), after upgrade to $NEWVERSION" + salt-call state.apply elasticsearch,elasticfleet queue=True + fi + echo "" echo "Upgrade to $NEWVERSION complete."