Merge pull request #16172 from Security-Onion-Solutions/3/dev

3/dev

VERSION

@@ -1 +1 @@
-3.3.0
+3.0.0-foxtrot

salt/elasticfleet/integration-defaults.map.jinja

@@ -16,7 +16,6 @@
     'awsfirehose.metrics': 'aws.cloudwatch',
     'cribl.logs': 'cribl',
     'cribl.metrics': 'cribl',
-    'sentinel_one_cloud_funnel.logins': 'sentinel_one_cloud_funnel.login',
     'azure_application_insights.app_insights': 'azure.app_insights',
     'azure_application_insights.app_state': 'azure.app_state',
     'azure_billing.billing': 'azure.billing',

salt/elasticsearch/cluster.sls

@@ -98,6 +98,15 @@ so-es-cluster-settings:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
       - http: wait_for_so-elasticsearch
+      
+so-elasticsearch-system-indices-patch:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-system-indices-patch
+    - require:
+      - http: wait_for_so-elasticsearch
+      - file: so-elasticsearch-system-indices-patch-script
+    - onchanges:
+      - file: so-elasticsearch-system-indices-patch-script
 {%  endif %}
 
 # heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)

salt/elasticsearch/config.sls

@@ -42,6 +42,16 @@ elasticsearch_sbin:
     - file_mode: 755
     - exclude_pat:
       - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state
+      - so-elasticsearch-system-indices-patch
+    - show_changes: False
+
+so-elasticsearch-system-indices-patch-script:
+  file.managed:
+    - name: /usr/sbin/so-elasticsearch-system-indices-patch
+    - source: salt://elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch
+    - user: 930
+    - group: 939
+    - mode: 755
     - show_changes: False
 
 elasticsearch_sbin_jinja:

salt/elasticsearch/defaults.yaml

@@ -1,7 +1,7 @@
 elasticsearch:
   enabled: false
   esheap: '600m'
-  version: 9.3.7
+  version: 9.4.5
   index_clean: true
   data_retention_method: DLM
   vm:

salt/elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch

@@ -0,0 +1,123 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+set -eo pipefail
+
+SETTINGS='{"index":{"auto_expand_replicas":"0-1"}}'
+KIBANA_PASSWORD=
+INDEX_PATTERNS=(
+    '.entity_analytics.risk_score.lookup-*'
+    '.entity_analytics.watchlists.*'
+    '.workflows-executions'
+    '.workflows-step-executions'
+    '.entities.v2.latest.security_*'
+    '.entities.v2.history.security_*'
+    'risk-score.risk-score-latest-*'
+)
+DATA_STREAM_PATTERNS=(
+    '.entities.v2.updates.security_*'
+    'risk-score.risk-score-*'
+)
+TEMPLATE_PATTERNS=(
+    'entities_v2_latest_security_default_index_template'
+    'entities_v2_history_security_default_index_template'
+    '.entities_v2_updates_security_default_index_template'
+    '.risk-score.risk-score-default-index-template'
+)
+
+query_es() {
+    if so-elasticsearch-query "$@" --fail --retry 3 --retry-delay 5; then
+        return 0
+    fi
+    
+    # retry failed attempts with so_kibana user (system managed indices reject so_elastic user)
+    local query_path="$1"
+    shift
+
+    if [[ -z "$KIBANA_PASSWORD" ]]; then
+        KIBANA_PASSWORD=$(salt-call pillar.get elasticsearch:auth:users:so_kibana_user:pass --out=newline_values_only)
+    fi
+    [[ -n "$KIBANA_PASSWORD" ]] || return 1
+
+    echo "Retrying ${query_path} as so_kibana." >&2
+    curl -K /opt/so/conf/elasticsearch/curl.config --user "so_kibana:${KIBANA_PASSWORD}" \
+        -s -k -L --fail --retry 3 --retry-delay 5 -H 'Content-Type: application/json' "https://localhost:9200/${query_path}" "$@"
+}
+
+# add auto_expand_replicas=0-1 to given index
+set_auto_expand_replicas() {
+    local index="$1"
+
+    echo "Setting auto_expand_replicas to 0-1 on ${index}."
+    query_es "${index}/_settings" -XPUT -d "$SETTINGS" >/dev/null
+}
+
+# resolve index patterns and run set_auto_expand_replicas on each index
+update_system_indices() {
+    local pattern="$1"
+    local response index
+
+    if ! response=$(query_es "_resolve/index/${pattern}?expand_wildcards=all" 2>/dev/null); then
+        return 0
+    fi
+
+    while read -r index; do
+        [[ -n "$index" ]] && set_auto_expand_replicas "$index"
+    done < <(jq -r '.indices[]?.name' <<<"$response")
+}
+
+# get data stream backing indices and run set_auto_expand_replicas on each index
+update_system_ds() {
+    local pattern="$1"
+    local response index
+
+    if ! response=$(query_es "_data_stream/${pattern}?expand_wildcards=all" 2>/dev/null); then
+        return 0
+    fi
+    # find backing indices for each data stream and update with $SETTINGS
+    while read -r index; do
+        [[ -n "$index" ]] && set_auto_expand_replicas "$index"
+    done < <(jq -r '.data_streams[]?.indices[]?.index_name' <<<"$response")
+}
+
+# get index templates, update with auto_expand_replicas=0-1, and PUT back. Keeping mappings/settings/aliases in-place
+update_system_templates() {
+    local pattern="$1"
+    local templates name response template auto_expand_replicas
+
+    if ! templates=$(query_es "_index_template/${pattern}" 2>/dev/null); then
+        return 0
+    fi
+    while read -r name; do
+        response=$(query_es "_index_template/${name}")
+        template=$(jq -c '.index_templates[0].index_template' <<<"$response")
+        auto_expand_replicas=$(jq -r '.template.settings["index.auto_expand_replicas"] // .template.settings.index.auto_expand_replicas // empty' <<<"$template")
+        [[ "$auto_expand_replicas" == "0-1" ]] && continue
+
+        template=$(jq '
+            if (.template.settings.index | type) == "object" then
+                .template.settings.index.auto_expand_replicas = "0-1"
+            else
+                .template.settings["index.auto_expand_replicas"] = "0-1"
+            end
+            | del(.created_date_millis, .modified_date_millis)
+        ' <<<"$template")
+        echo "Setting auto_expand_replicas to 0-1 on index template ${name}."
+        query_es "_index_template/${name}" -XPUT -d "$template" >/dev/null
+    done < <(jq -r '.index_templates[]?.name' <<<"$templates")
+}
+
+for pattern in "${INDEX_PATTERNS[@]}"; do
+    update_system_indices "$pattern"
+done
+
+for pattern in "${DATA_STREAM_PATTERNS[@]}"; do
+    update_system_ds "$pattern"
+done
+
+for pattern in "${TEMPLATE_PATTERNS[@]}"; do
+    update_system_templates "$pattern"
+done