add pause allowing fleet server to come up prior to health check

merge reyesj2/es945

VERSION

@@ -1 +1 @@
-3.3.0
+3.0.0-foxtrot

salt/elasticfleet/enabled.sls

@@ -8,6 +8,7 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+{%   set SOUP_SKIP_HEALTH_CHECKS = salt['pillar.get']('soup:skip_health_checks', False) %}
 
 {#   This value is generated during node install and stored in minion pillar #}
 {%   set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %}
@@ -113,6 +114,11 @@ wait_for_so-elastic-fleet:
     - request_interval: 15
     - require:
       - docker_container: so-elastic-fleet
+{% if SOUP_SKIP_HEALTH_CHECKS %}
+    # SOUP starts containers as part of its upgrade highstate; do not spend up to
+    # 5 minutes waiting for Fleet Server while that upgrade is in progress.
+    - onlyif: /bin/false
+{% endif %}
 {%   endif %}
 
 delete_so-elastic-fleet_so-status.disabled:

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -5,7 +5,7 @@
   "package": {
     "name": "endpoint",
     "title": "Elastic Defend",
-    "version": "9.3.1",
+    "version": "9.4.1",
     "requires_root": true
   },
   "enabled": true,

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -29,7 +29,7 @@
               "\\.gz$"
             ],
             "include_files": [],
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.20.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.3\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.20.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.20.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.3\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.22.3\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.9.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.22.3\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.22.3\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.9.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
             ],

salt/elasticfleet/integration-defaults.map.jinja

@@ -16,7 +16,6 @@
     'awsfirehose.metrics': 'aws.cloudwatch',
     'cribl.logs': 'cribl',
     'cribl.metrics': 'cribl',
-    'sentinel_one_cloud_funnel.logins': 'sentinel_one_cloud_funnel.login',
     'azure_application_insights.app_insights': 'azure.app_insights',
     'azure_application_insights.app_state': 'azure.app_state',
     'azure_billing.billing': 'azure.billing',

salt/elasticsearch/cluster.sls

@@ -98,6 +98,13 @@ so-es-cluster-settings:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
       - http: wait_for_so-elasticsearch
+
+so-elasticsearch-system-indices-patch:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-system-indices-patch
+    - require:
+      - http: wait_for_so-elasticsearch
+      - file: so-elasticsearch-system-indices-patch-script
 {%  endif %}
 
 # heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)

salt/elasticsearch/config.sls

@@ -42,6 +42,16 @@ elasticsearch_sbin:
     - file_mode: 755
     - exclude_pat:
       - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state
+      - so-elasticsearch-system-indices-patch
+    - show_changes: False
+
+so-elasticsearch-system-indices-patch-script:
+  file.managed:
+    - name: /usr/sbin/so-elasticsearch-system-indices-patch
+    - source: salt://elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch
+    - user: 930
+    - group: 939
+    - mode: 755
     - show_changes: False
 
 elasticsearch_sbin_jinja:

salt/elasticsearch/defaults.yaml

@@ -1,7 +1,7 @@
 elasticsearch:
   enabled: false
   esheap: '600m'
-  version: 9.3.7
+  version: 9.4.5
   index_clean: true
   data_retention_method: DLM
   vm:

salt/elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch

@@ -0,0 +1,199 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+set -eo pipefail
+
+SETTINGS='{"index":{"auto_expand_replicas":"0-1"}}'
+KIBANA_PASSWORD=
+INDEX_PATTERNS=(
+  '.entity_analytics.risk_score.lookup-*'
+  '.entity_analytics.watchlists.*'
+  '.entity_analytics.monitoring.users-*'
+  '.entity_analytics.entity-leads-*'
+  '.asset-criticality.asset-criticality-*'
+  '.workflows-executions'
+  '.workflows-step-executions'
+  '.entities.v2.latest.security_*'
+  '.entities.v2.history.security_*'
+  'risk-score.risk-score-latest-*'
+)
+DATA_STREAM_PATTERNS=(
+  '.entities.v2.updates.security_*'
+  'risk-score.risk-score-*'
+  '.rule-events'
+  '.alert-actions'
+)
+TEMPLATE_PATTERNS=(
+  'entities_v2_latest_security_default_index_template'
+  'entities_v2_history_security_default_index_template'
+  '.entities_v2_updates_security_default_index_template'
+  '.risk-score.risk-score-default-index-template'
+  '.rule-events'
+  '.alert-actions'
+)
+
+query_es() {
+  if so-elasticsearch-query "$@" --fail --retry 3 --retry-delay 5; then
+    return 0
+  fi
+
+  # retry failed attempts with so_kibana user (system managed indices reject so_elastic user)
+  local query_path="$1"
+  shift
+
+  if [[ -z "$KIBANA_PASSWORD" ]]; then
+    KIBANA_PASSWORD=$(salt-call pillar.get elasticsearch:auth:users:so_kibana_user:pass --out=newline_values_only)
+  fi
+  [[ -n "$KIBANA_PASSWORD" ]] || return 1
+
+  echo "Retrying ${query_path} as so_kibana." >&2
+  curl -K /opt/so/conf/elasticsearch/curl.config --user "so_kibana:${KIBANA_PASSWORD}" \
+    -s -k -L --fail --retry 3 --retry-delay 5 -H 'Content-Type: application/json' "https://localhost:9200/${query_path}" "$@"
+}
+
+# add auto_expand_replicas=0-1 to given index
+set_auto_expand_replicas() {
+  local index="$1"
+
+  echo "Setting auto_expand_replicas to 0-1 on ${index}."
+  query_es "${index}/_settings" -XPUT -d "$SETTINGS" >/dev/null
+}
+
+# resolve index patterns and find each index with an unassigned replica
+unassigned_replicas() {
+  local pattern="$1"
+  local resolved_indices response index
+
+  if ! resolved_indices=$(query_es "_resolve/index/${pattern}?expand_wildcards=all" 2>/dev/null); then
+    return 0
+  fi
+
+  while read -r index; do
+    if ! response=$(query_es "_cat/shards/${index}?format=json&h=index,prirep,state" 2>/dev/null); then
+      continue
+    fi
+    jq -r '.[]? | objects | select(.prirep == "r" and .state == "UNASSIGNED") | .index' <<<"$response"
+  done < <(jq -r '.indices[]?.name' <<<"$resolved_indices")
+}
+
+data_stream_indices() {
+  local pattern="$1"
+  local response
+
+  if ! response=$(query_es "_data_stream/${pattern}?expand_wildcards=all" 2>/dev/null); then
+    return 0
+  fi
+  jq -r '.data_streams[]?.indices[]?.index_name' <<<"$response"
+}
+
+update_system_indices() {
+  local pattern="$1"
+  local index
+
+  while read -r index; do
+    [[ -n "$index" ]] && set_auto_expand_replicas "$index"
+  done < <(unassigned_replicas "$pattern")
+}
+
+# update data stream backing indices with unassigned replicas
+update_system_ds() {
+  local pattern="$1"
+  local index
+
+  while read -r index; do
+    while read -r unassigned_index; do
+      [[ -n "$unassigned_index" ]] && set_auto_expand_replicas "$unassigned_index"
+    done < <(unassigned_replicas "$index")
+  done < <(data_stream_indices "$pattern")
+}
+
+has_unassigned_replicas() {
+  local pattern="$1"
+  local index
+
+  index=$(unassigned_replicas "$pattern" | sed -n '1p')
+  [[ -n "$index" ]]
+}
+
+data_stream_has_unassigned_replicas() {
+  local pattern="$1"
+  local index
+  while read -r index; do
+    has_unassigned_replicas "$index" && return 0
+  done < <(data_stream_indices "$pattern")
+
+  return 1
+}
+
+needs_patch() {
+  local pattern
+  for pattern in "${INDEX_PATTERNS[@]}"; do
+    has_unassigned_replicas "$pattern" && return 0
+  done
+  for pattern in "${DATA_STREAM_PATTERNS[@]}"; do
+    data_stream_has_unassigned_replicas "$pattern" && return 0
+  done
+
+  return 1
+}
+
+# get index templates, update with auto_expand_replicas=0-1, and PUT back. Keeping mappings/settings/aliases in-place
+update_system_templates() {
+  local pattern="$1"
+  local templates name response template auto_expand_replicas
+
+  if ! templates=$(query_es "_index_template/${pattern}" 2>/dev/null); then
+    return 0
+  fi
+  while read -r name; do
+    response=$(query_es "_index_template/${name}")
+    template=$(jq -c '.index_templates[0].index_template' <<<"$response")
+    auto_expand_replicas=$(jq -r '.template.settings["index.auto_expand_replicas"] // .template.settings.index.auto_expand_replicas // empty' <<<"$template")
+    [[ "$auto_expand_replicas" == "0-1" ]] && continue
+
+    template=$(jq '
+            if (.template.settings.index | type) == "object" then
+                .template.settings.index.auto_expand_replicas = "0-1"
+            else
+                .template.settings["index.auto_expand_replicas"] = "0-1"
+            end
+            | del(.created_date_millis, .modified_date_millis)
+        ' <<<"$template")
+    echo "Setting auto_expand_replicas to 0-1 on index template ${name}."
+    query_es "_index_template/${name}" -XPUT -d "$template" >/dev/null
+  done < <(jq -r '.index_templates[]?.name' <<<"$templates")
+}
+
+if [[ "${1:-}" == "--check" ]]; then
+  needs_patch
+  exit $?
+fi
+
+if [[ $# -ne 0 ]]; then
+  echo "Usage: $0 [--check]" >&2
+  exit 1
+fi
+
+patched=false
+for pattern in "${INDEX_PATTERNS[@]}"; do
+  if has_unassigned_replicas "$pattern"; then
+    update_system_indices "$pattern"
+    patched=true
+  fi
+done
+
+for pattern in "${DATA_STREAM_PATTERNS[@]}"; do
+  if data_stream_has_unassigned_replicas "$pattern"; then
+    update_system_ds "$pattern"
+    patched=true
+  fi
+done
+
+if [[ "$patched" == true ]]; then
+  for pattern in "${TEMPLATE_PATTERNS[@]}"; do
+    update_system_templates "$pattern"
+  done
+fi

salt/kibana/defaults.yaml

@@ -22,7 +22,7 @@ kibana:
           - default
           - file
     migrations:
-      discardCorruptObjects: "9.3.7"
+      discardCorruptObjects: "9.4.5"
     telemetry:
       enabled: False
     xpack:

salt/kibana/enabled.sls

@@ -8,6 +8,7 @@
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   set SOUP_SKIP_HEALTH_CHECKS = salt['pillar.get']('soup:skip_health_checks', False) %}
 
 include:
   - kibana.config
@@ -74,6 +75,11 @@ wait_for_so-kibana:
     - request_interval: 15
     - require:
       - docker_container: so-kibana
+{% if SOUP_SKIP_HEALTH_CHECKS %}
+    # SOUP starts containers as part of its upgrade highstate; do not spend up to
+    # 10 minutes waiting for Kibana while that upgrade is in progress.
+    - onlyif: /bin/false
+{% endif %}
 
 delete_so-kibana_so-status.disabled:
   file.uncomment:

salt/kibana/tools/sbin_jinja/so-kibana-space-defaults

@@ -9,5 +9,5 @@ SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http:
 # Disable certain Features from showing up in the Kibana UI
 echo
 echo "Setting up default Kibana Space:"
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV1","entityManager","streams","cloudConnect","slo"]} ' >> /opt/so/log/kibana/misc.log
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","securitySolutionCasesV3","inventory","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV4","securitySolutionAlertsV1","entityManager","slo","streams","anonymization","searchInferenceEndpoints","cloudConnect","queryActivity","automatic_import","stackAlerts","monitoring","dataQuality","actions"]} ' >> /opt/so/log/kibana/misc.log
 echo

salt/manager/tools/sbin/soup

@@ -352,6 +352,8 @@ check_fleet_server() {
     docker stop so-elastic-fleet; docker rm -f so-elastic-fleet
     printf "\nUpdated sysctl fs.protected_symlinks. Restarting fleet before running health check and continuing with soup.\n"
     salt-call state.apply elasticfleet queue=True
+    echo "Fleet restarted, waiting 2 minutes before checking fleet health and continuing SOUP...";
+    for i in {1..12}; do printf "."; sleep 10; done
   fi
   
   # Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls,
@@ -456,13 +458,19 @@ get_soup_script_hashes() {
 
 highstate() {
   # Run a highstate with a retry attempt.
-  if salt-call state.highstate -l info queue=True; then
+  local highstate_cmd=(salt-call state.highstate -l info queue=True)
+  if [[ "$es_version" != "$target_es_version" ]]; then
+    echo "Elasticsearch upgrade is scheduled. While the upgrade is in progress, skip health checks for services expected to be temporarily unavailable."
+    highstate_cmd+=('pillar={"soup":{"skip_health_checks":true}}')
+  fi
+  
+  if "${highstate_cmd[@]}"; then
     return 0
   fi
 
   echo "Initial highstate attempt had a problem; retrying in 30 seconds."
   sleep 30
-  salt-call state.highstate -l info queue=True
+  "${highstate_cmd[@]}"
 }
 
 push_grid_highstate() {
@@ -1359,11 +1367,12 @@ verify_es_version_compatibility() {
     local is_active_intermediate_upgrade=1
     # supported upgrade paths for SO-ES versions
     declare -A es_upgrade_map=(
-        ["8.18.4"]="8.18.6 8.18.8 9.0.8"
+      ["8.18.4"]="8.18.6 8.18.8 9.0.8"
 	    ["8.18.6"]="8.18.8 9.0.8"
 	    ["8.18.8"]="9.0.8"
-        ["9.0.8"]="9.3.3 9.3.7"
+      ["9.0.8"]="9.3.3 9.3.7 9.4.5"
-        ["9.3.3"]="9.3.7"
+      ["9.3.3"]="9.3.7 9.4.5"
+      ["9.3.7"]="9.4.5"
     )
 
     # Elasticsearch MUST upgrade through these versions

salt/soc/defaults.yaml

@@ -2671,7 +2671,7 @@ soc:
               # The id (UUIDv4) is pregenerated and can safely be used.
               # Click "Convert" to convert the Sigma rule to use Security Onion field mappings within an EQL query
               #
-              # Rule Creation Guide: https://github.com/SigmaHQ/sigma/wiki/Rule-Creation-Guide
+              # Rule Creation Guide: https://github.com/SigmaHQ/sigma/wiki/Rule-Creation-High%E2%80%90Level-Guide
               # Logsources: https://sigmahq.io/docs/basics/log-sources.html
 
               title: 'A Short Capitalized Title With Less Than 50 Characters'
@@ -2683,7 +2683,7 @@ soc:
               references:
                 - 'https://local.invalid'
               author: '@SecurityOnion'
-              date: 'YYYY/MM/DD'
+              date: '[today]'
               tags:
                 - detection.threat_hunting
                 - attack.technique_id