ft

include fleet state
Merge branch 'reyesj2/reworksoup' into reyesj2/es945
2026-08-30 19:29:19 +02:00 · 2026-08-28 13:46:50 -05:00 · 2026-08-28 13:45:55 -05:00 · 2026-08-28 13:28:53 -05:00 · 2026-08-28 13:27:44 -05:00 · 2026-08-28 12:19:20 -05:00
32 changed files with 361 additions and 1083 deletions
@@ -1 +1 @@
-3.3.0
+3.0.0-foxtrot
@@ -141,20 +141,6 @@ pin_nic_names:
      - file: common_sbin
      - file: statedir

-# Once a node is actually running UEK8, the stock EL9 (RHCK) kernel packages are dead weight.
-# They can't be removed any earlier -- dnf protects the running kernel -- so the cleanup waits
-# for the reboot, which makes the highstate the natural place to catch it: fresh installs
-# reboot at the end of setup, and upgraded nodes reboot whenever the admin schedules it.
-# so-kernel-upgrade --cleanup checks rpm before touching dnf, so this costs an rpm query on
-# every highstate after the first pass. The package list lives in the script only, so there
-# is nothing here to drift out of sync with it.
-remove_stock_kernel:
-  cmd.run:
-    - name: /usr/sbin/so-kernel-upgrade --cleanup
-    - onlyif: 'uname -r | grep -qE "^6\.[0-9]+.*uek"'
-    - require:
-      - file: common_sbin
-
 common_sbin_jinja:
  file.recurse:
    - name: /usr/sbin
@@ -5,11 +5,10 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 #
-# so-kernel-upgrade — install the UEK8 (6.x) kernel, make it the boot default, and once the
-# node is running it, remove the stock EL9 kernel.
+# so-kernel-upgrade — install the UEK8 (6.x) kernel and make it the boot default.
 #
 # Security Onion is moving off the EL9 stock kernel (RHCK, 5.14) and UEK7 (5.15) onto UEK8
-# (6.x). Four things have to happen, and the tool has to drive each one:
+# (6.x). Three things have to happen, and the tool has to drive each one:
 #
 #   1. Populate. The manager mirrors the UEK8 packages into /nsm/kernelrepo via so-repo-sync,
 #      and serves them to the grid over https://<manager>/kernelrepo. Until that sync runs the
@@ -27,21 +26,10 @@
 #        - From the stock EL9 kernel (RHCK, 5.14, no UEK) it is a flavor CROSS that is NOT
 #          auto-promoted, so the box keeps booting RHCK until grubby is told otherwise.
 #      This tool inspects the running kernel and only runs 'grubby --set-default' for RHCK.
-#   4. Clean up. Once the node is actually RUNNING UEK8 the stock kernel packages are dead
-#      weight -- disk in /boot and a stale GRUB entry. They cannot come off any earlier:
-#      dnf's protect_running_kernel refuses to erase the booted kernel-core, so the removal
-#      has to wait for the reboot. Waiting is also the safer sequencing on its own terms --
-#      the node has proven it comes up on UEK8 before its fallback is deleted. That is why
-#      the removal does not happen in the uek7 branch either, where dnf would allow it.
 #
 # Every one of those failure modes is silent by default. This tool handles each case and fails
 # loudly when it cannot, rather than reporting success while changing nothing.
 #
-# Invocation: with no arguments it drives the whole sequence for whatever kernel the node is
-# on. With --cleanup it does the step 4 removal ONLY, and no-ops on a node that isn't running
-# UEK8 yet -- that is the form the common highstate calls (remove_stock_kernel in
-# salt/common/init.sls) so the cleanup lands grid-wide after each node reboots.
-#
 # Manager vs minion: only the manager owns /nsm/kernelrepo, so only the manager can populate
 # it. If the repo is empty here, a manager runs so-repo-sync itself; a minion has no way to
 # fix it and exits non-zero telling the admin to sync the manager first.
@@ -61,11 +49,6 @@ KERNEL_REPO_DIR="/nsm/kernelrepo"
 REPOSYNC_CONF="/opt/so/conf/reposync/repodownload.conf"
 GLOBAL_PILLAR="/opt/so/saltstack/local/pillar/global/soc_global.sls"

-# Stock EL9 (RHCK) kernel packages, removed only once the node is running UEK8 (see step 4
-# in the header). Left deliberately narrow: UEK7 kernel-uek builds age out on their own via
-# installonly_limit=3, and kernel-devel/kernel-headers are not touched.
-RHCK_PKGS="kernel kernel-core kernel-modules kernel-modules-core kernel-tools kernel-tools-libs"
-
 log() { echo "[so-kernel-upgrade] $*"; }
 die() { echo "[so-kernel-upgrade] ERROR: $*" >&2; exit 1; }

@@ -166,13 +149,8 @@ ensure_kernel_repo() {
 }

 reboot_notice() {
-    [ "$(uname -r)" = "$(basename "$1" | sed 's/^vmlinuz-//')" ] && return 0
-    log "REBOOT REQUIRED to start using the UEK8 kernel (currently running $(uname -r))."
-    # The stock kernel can't be removed until it stops being the running one, so say when
-    # that will happen rather than leaving the admin to wonder if it was missed.
-    [ -n "$(rhck_installed)" ] \
-        && log "The stock EL9 kernel is left in place until then; it is removed by the next highstate after the reboot."
-    return 0
+    [ "$(uname -r)" = "$(basename "$1" | sed 's/^vmlinuz-//')" ] \
+        || log "REBOOT REQUIRED to start using the UEK8 kernel (currently running $(uname -r))."
 }

 # Keep future kernel updates on the UEK line rather than falling back to RHCK. Oracle ships
@@ -184,32 +162,6 @@ set_default_kernel_conf() {
    fi
 }

-# Which of RHCK_PKGS are actually installed, one per line. rpm -qa treats each argument as a
-# name glob and prints only what it finds, so a package that was never installed (or is
-# already gone) simply doesn't appear -- no "not installed" noise and no non-zero exit.
-rhck_installed() {
-    rpm -qa $RHCK_PKGS 2>/dev/null
-}
-
-# Remove the stock EL9 kernel. Only ever called once the running kernel is UEK8. The rpm
-# check above is the idempotency guard, so this is a cheap no-op on every highstate after
-# the first one -- it costs an rpm query, not a dnf transaction.
-remove_rhck() {
-    local installed; installed="$(rhck_installed)"
-    if [ -z "$installed" ]; then
-        log "no stock EL9 (RHCK) kernel packages installed; nothing to remove."
-        return 0
-    fi
-
-    log "running UEK8; removing the stock EL9 (RHCK) kernel packages:"
-    echo "$installed" | sed 's/^/[so-kernel-upgrade]   /'
-    dnf -y remove $RHCK_PKGS || die "failed to remove the stock EL9 kernel packages"
-
-    installed="$(rhck_installed)"
-    [ -z "$installed" ] || die "dnf reported success but these remain: $(echo $installed)"
-    log "stock EL9 kernel packages removed."
-}
-
 # Make sure a UEK8 kernel is installed, leaving its boot entry in INSTALLED_UEK8. If one is
 # already present we leave the repo alone -- it may be disabled or empty and we don't need it
 # just to flip the boot default. Otherwise install the explicit NEVRA, not the bare package
@@ -232,38 +184,12 @@ ensure_uek8_installed() {
    log "installed UEK8 kernel: $INSTALLED_UEK8"
 }

-# --cleanup does step 4 and nothing else. It exits 0 rather than failing on a node that
-# isn't on UEK8 yet: the highstate gates on 'uname -r' before calling this, and a state that
-# fails whenever that gate races would be worse than one that says what it's waiting for.
-case "$1" in
-"")
-    ;;
--cleanup)
-    if [ "$(running_flavor)" != uek8 ]; then
-        log "not running a UEK8 kernel yet (currently $(uname -r)); leaving the stock EL9 kernel in place."
-        log "Run so-kernel-upgrade with no arguments to install UEK8, then reboot."
-        exit 0
-    fi
-    set_default_kernel_conf
-    remove_rhck
-    exit 0
-    ;;
-*)
-    echo "Usage: so-kernel-upgrade [--cleanup]" >&2
-    echo "  (no arguments)  install UEK8, make it the boot default, clean up once it's running" >&2
-    echo "  --cleanup       remove the stock EL9 kernel; no-op unless already running UEK8" >&2
-    exit 1
-    ;;
-esac
-
 case "$(running_flavor)" in
 uek8)
    # Already on the 6.x UEK line. A plain 'dnf update' keeps this node current within the
-    # lineage and auto-promotes newer builds, so there is no install or grubby work left --
-    # only the step 4 cleanup, which this is the first point in the sequence that can run it.
-    log "already running a UEK8 kernel ($(uname -r)); no kernel install needed."
-    set_default_kernel_conf
-    remove_rhck
+    # lineage and auto-promotes newer builds, so there is nothing for this tool to do.
+    log "already running a UEK8 kernel ($(uname -r)); nothing to do."
+    exit 0
    ;;

 uek7)
@@ -134,6 +134,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Redis may have been restarted" # Redis likely restarted by salt
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|file already closed"          # Go logging race condition during container restart
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|relation \"audit_settings\" does not exist" # salt checking for changes before SOC starts
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error in plugin: elasticsearch: Unable to retrieve master node information" # expected error while ES is upgrading/electing a master 
 fi

 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
@@ -240,6 +241,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown"             # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Could not index event to Elasticsearch.*\"version\" => \"9.0.8\"" # Expected during Elastic upgrade temporarily, as policies referencing older pipelines are updated
 fi

 RESULT=0
@@ -304,4 +306,4 @@ else
    echo -e "\nResult: One or more errors found"
 fi

-exit $RESULT
+exit $RESULT
@@ -5,7 +5,7 @@
  "package": {
    "name": "endpoint",
    "title": "Elastic Defend",
-    "version": "9.3.1",
+    "version": "9.4.1",
    "requires_root": true
  },
  "enabled": true,
@@ -29,7 +29,7 @@
              "\\.gz$"
            ],
            "include_files": [],
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.20.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.3\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.20.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.20.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.3\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.22.3\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.9.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.22.3\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.22.3\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.9.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
            "tags": [
              "import"
            ],
@@ -16,7 +16,6 @@
    'awsfirehose.metrics': 'aws.cloudwatch',
    'cribl.logs': 'cribl',
    'cribl.metrics': 'cribl',
-    'sentinel_one_cloud_funnel.logins': 'sentinel_one_cloud_funnel.login',
    'azure_application_insights.app_insights': 'azure.app_insights',
    'azure_application_insights.app_state': 'azure.app_state',
    'azure_billing.billing': 'azure.billing',
@@ -68,6 +68,24 @@ so-elastic-fleet-package-upgrade:
    - require:
      - http: wait_for_so-kibana

+# initial so-elasticsearch-templates run is earlier, but it can skip over templates that have component templates not yet installed to avoid elasticsearch rejecting the template.
+so-elasticsearch-templates-after-fleet-packages:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-templates-load
+    - cwd: /opt/so
+    - unless: test -f /opt/so/state/estemplates.txt
+    - require:
+      - cmd: so-elastic-fleet-package-upgrade
+
+so-elastic-fleet-integration-upgrade:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-integration-upgrade
+    - retry:
+        attempts: 3
+        interval: 10
+    - require:
+      - cmd: so-elastic-fleet-package-upgrade
+
 so-elastic-fleet-integrations:
  cmd.run:
    - name: /usr/sbin/so-elastic-fleet-integration-policy-load
@@ -86,21 +104,13 @@ so-elastic-agent-grid-upgrade:
    - require:
      - http: wait_for_so-kibana

-so-elastic-fleet-integration-upgrade:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-integration-upgrade
-    - retry:
-        attempts: 3
-        interval: 10
-    - require:
-      - http: wait_for_so-kibana
-
 {# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
 so-elastic-fleet-addon-integrations:
  cmd.run:
    - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
    - require:
      - http: wait_for_so-kibana
+      - cmd: so-elasticsearch-templates-after-fleet-packages

 {% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
 so-elastic-defend-manage-filters-file-watch:
@@ -10,6 +10,25 @@

 PKG_LOAD_FAILURES=0
 PKG_LOAD_FAILURES_NAMES=()
+PKG_UPGRADED=0
+
+cleanup_elasticsearch_fleet_transforms() {
+    local transforms transform_id attempt
+    
+    if ! transforms=$(so-elasticsearch-query "_transform/logs-elasticsearch.index_pivot-default-*" --retry 1 --retry-delay 5); then
+        return 0
+    fi
+
+    while IFS= read -r transform_id; do
+        [ -n "$transform_id" ] || continue
+        for attempt in {1..3}; do
+            if so-elasticsearch-query "_transform/$transform_id?force=true" -XDELETE --fail --retry 1 --retry-delay 5; then
+                break
+            fi
+            sleep 5
+        done
+    done < <(jq -r '.transforms[]?.id' <<< "$transforms")
+}

 {%- for PACKAGE in SUPPORTED_PACKAGES %}
 if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && LATEST_VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
@@ -17,10 +36,25 @@ if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && L
    if [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then
        echo "{{ PACKAGE }} integration version $INSTALLED_VERSION is already at the reported latest version $LATEST_VERSION, skipping upgrade."
    else
+{%- if PACKAGE == 'elasticsearch' %}
+        cleanup_elasticsearch_fleet_transforms
+{%- endif %}
        echo "Upgrading {{ PACKAGE }} package from $INSTALLED_VERSION to version $LATEST_VERSION..."
        if ! elastic_fleet_package_install "{{ PACKAGE }}" "$LATEST_VERSION"; then
            PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
            PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
+        # check that package has upgraded to the expected version after install command
+        elif ! LATEST_VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
+            echo "ERROR: Failed to get latest version information for integration {{ PACKAGE }} after upgrade attempt"
+            PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
+            PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
+        elif INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then
+            echo "{{ PACKAGE }} integration upgraded to version $LATEST_VERSION."
+            PKG_UPGRADED=$((PKG_UPGRADED + 1))
+        else
+            echo "ERROR: {{ PACKAGE }} integration still at ${INSTALLED_VERSION:-unknown}; expected $LATEST_VERSION"
+            PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
+            PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
        fi
    fi
 else
@@ -30,6 +64,11 @@ else
 fi
 {%- endfor %}

+if [ $PKG_UPGRADED -gt 0 ]; then
+    echo "Elasticsearch template statefiles cleared after $PKG_UPGRADED package upgrade(s), so templates can reload."
+    rm -f /opt/so/state/estemplates.txt /opt/so/state/addon_estemplates.txt
+fi
+
 if [ $PKG_LOAD_FAILURES -gt 0 ]; then
    echo "ERROR: Failed to upgrade $PKG_LOAD_FAILURES package(s):"
    for PKG in "${PKG_LOAD_FAILURES_NAMES[@]}"; do
@@ -98,6 +98,13 @@ so-es-cluster-settings:
      - docker_container: so-elasticsearch
      - file: elasticsearch_sbin_jinja
      - http: wait_for_so-elasticsearch
+
+so-elasticsearch-system-indices-patch:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-system-indices-patch
+    - require:
+      - http: wait_for_so-elasticsearch
+      - file: so-elasticsearch-system-indices-patch-script
 {%  endif %}

 # heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)
@@ -42,6 +42,16 @@ elasticsearch_sbin:
    - file_mode: 755
    - exclude_pat:
      - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state
+      - so-elasticsearch-system-indices-patch
+    - show_changes: False
+
+so-elasticsearch-system-indices-patch-script:
+  file.managed:
+    - name: /usr/sbin/so-elasticsearch-system-indices-patch
+    - source: salt://elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch
+    - user: 930
+    - group: 939
+    - mode: 755
    - show_changes: False

 elasticsearch_sbin_jinja:
@@ -1,7 +1,7 @@
 elasticsearch:
  enabled: false
  esheap: '600m'
-  version: 9.3.7
+  version: 9.4.5
  index_clean: true
  data_retention_method: DLM
  vm:
@@ -0,0 +1,199 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+set -eo pipefail
+
+SETTINGS='{"index":{"auto_expand_replicas":"0-1"}}'
+KIBANA_PASSWORD=
+INDEX_PATTERNS=(
+  '.entity_analytics.risk_score.lookup-*'
+  '.entity_analytics.watchlists.*'
+  '.entity_analytics.monitoring.users-*'
+  '.entity_analytics.entity-leads-*'
+  '.asset-criticality.asset-criticality-*'
+  '.workflows-executions'
+  '.workflows-step-executions'
+  '.entities.v2.latest.security_*'
+  '.entities.v2.history.security_*'
+  'risk-score.risk-score-latest-*'
+)
+DATA_STREAM_PATTERNS=(
+  '.entities.v2.updates.security_*'
+  'risk-score.risk-score-*'
+  '.rule-events'
+  '.alert-actions'
+)
+TEMPLATE_PATTERNS=(
+  'entities_v2_latest_security_default_index_template'
+  'entities_v2_history_security_default_index_template'
+  '.entities_v2_updates_security_default_index_template'
+  '.risk-score.risk-score-default-index-template'
+  '.rule-events'
+  '.alert-actions'
+)
+
+query_es() {
+  if so-elasticsearch-query "$@" --fail --retry 3 --retry-delay 5; then
+    return 0
+  fi
+
+  # retry failed attempts with so_kibana user (system managed indices reject so_elastic user)
+  local query_path="$1"
+  shift
+
+  if [[ -z "$KIBANA_PASSWORD" ]]; then
+    KIBANA_PASSWORD=$(salt-call pillar.get elasticsearch:auth:users:so_kibana_user:pass --out=newline_values_only)
+  fi
+  [[ -n "$KIBANA_PASSWORD" ]] || return 1
+
+  echo "Retrying ${query_path} as so_kibana." >&2
+  curl -K /opt/so/conf/elasticsearch/curl.config --user "so_kibana:${KIBANA_PASSWORD}" \
+    -s -k -L --fail --retry 3 --retry-delay 5 -H 'Content-Type: application/json' "https://localhost:9200/${query_path}" "$@"
+}
+
+# add auto_expand_replicas=0-1 to given index
+set_auto_expand_replicas() {
+  local index="$1"
+
+  echo "Setting auto_expand_replicas to 0-1 on ${index}."
+  query_es "${index}/_settings" -XPUT -d "$SETTINGS" >/dev/null
+}
+
+# resolve index patterns and find each index with an unassigned replica
+unassigned_replicas() {
+  local pattern="$1"
+  local resolved_indices response index
+
+  if ! resolved_indices=$(query_es "_resolve/index/${pattern}?expand_wildcards=all" 2>/dev/null); then
+    return 0
+  fi
+
+  while read -r index; do
+    if ! response=$(query_es "_cat/shards/${index}?format=json&h=index,prirep,state" 2>/dev/null); then
+      continue
+    fi
+    jq -r '.[]? | objects | select(.prirep == "r" and .state == "UNASSIGNED") | .index' <<<"$response"
+  done < <(jq -r '.indices[]?.name' <<<"$resolved_indices")
+}
+
+data_stream_indices() {
+  local pattern="$1"
+  local response
+
+  if ! response=$(query_es "_data_stream/${pattern}?expand_wildcards=all" 2>/dev/null); then
+    return 0
+  fi
+  jq -r '.data_streams[]?.indices[]?.index_name' <<<"$response"
+}
+
+update_system_indices() {
+  local pattern="$1"
+  local index
+
+  while read -r index; do
+    [[ -n "$index" ]] && set_auto_expand_replicas "$index"
+  done < <(unassigned_replicas "$pattern")
+}
+
+# update data stream backing indices with unassigned replicas
+update_system_ds() {
+  local pattern="$1"
+  local index
+
+  while read -r index; do
+    while read -r unassigned_index; do
+      [[ -n "$unassigned_index" ]] && set_auto_expand_replicas "$unassigned_index"
+    done < <(unassigned_replicas "$index")
+  done < <(data_stream_indices "$pattern")
+}
+
+has_unassigned_replicas() {
+  local pattern="$1"
+  local index
+
+  index=$(unassigned_replicas "$pattern" | sed -n '1p')
+  [[ -n "$index" ]]
+}
+
+data_stream_has_unassigned_replicas() {
+  local pattern="$1"
+  local index
+  while read -r index; do
+    has_unassigned_replicas "$index" && return 0
+  done < <(data_stream_indices "$pattern")
+
+  return 1
+}
+
+needs_patch() {
+  local pattern
+  for pattern in "${INDEX_PATTERNS[@]}"; do
+    has_unassigned_replicas "$pattern" && return 0
+  done
+  for pattern in "${DATA_STREAM_PATTERNS[@]}"; do
+    data_stream_has_unassigned_replicas "$pattern" && return 0
+  done
+
+  return 1
+}
+
+# get index templates, update with auto_expand_replicas=0-1, and PUT back. Keeping mappings/settings/aliases in-place
+update_system_templates() {
+  local pattern="$1"
+  local templates name response template auto_expand_replicas
+
+  if ! templates=$(query_es "_index_template/${pattern}" 2>/dev/null); then
+    return 0
+  fi
+  while read -r name; do
+    response=$(query_es "_index_template/${name}")
+    template=$(jq -c '.index_templates[0].index_template' <<<"$response")
+    auto_expand_replicas=$(jq -r '.template.settings["index.auto_expand_replicas"] // .template.settings.index.auto_expand_replicas // empty' <<<"$template")
+    [[ "$auto_expand_replicas" == "0-1" ]] && continue
+
+    template=$(jq '
+            if (.template.settings.index | type) == "object" then
+                .template.settings.index.auto_expand_replicas = "0-1"
+            else
+                .template.settings["index.auto_expand_replicas"] = "0-1"
+            end
+            | del(.created_date_millis, .modified_date_millis)
+        ' <<<"$template")
+    echo "Setting auto_expand_replicas to 0-1 on index template ${name}."
+    query_es "_index_template/${name}" -XPUT -d "$template" >/dev/null
+  done < <(jq -r '.index_templates[]?.name' <<<"$templates")
+}
+
+if [[ "${1:-}" == "--check" ]]; then
+  needs_patch
+  exit $?
+fi
+
+if [[ $# -ne 0 ]]; then
+  echo "Usage: $0 [--check]" >&2
+  exit 1
+fi
+
+patched=false
+for pattern in "${INDEX_PATTERNS[@]}"; do
+  if has_unassigned_replicas "$pattern"; then
+    update_system_indices "$pattern"
+    patched=true
+  fi
+done
+
+for pattern in "${DATA_STREAM_PATTERNS[@]}"; do
+  if data_stream_has_unassigned_replicas "$pattern"; then
+    update_system_ds "$pattern"
+    patched=true
+  fi
+done
+
+if [[ "$patched" == true ]]; then
+  for pattern in "${TEMPLATE_PATTERNS[@]}"; do
+    update_system_templates "$pattern"
+  done
+fi
@@ -22,7 +22,7 @@ kibana:
          - default
          - file
    migrations:
-      discardCorruptObjects: "9.3.7"
+      discardCorruptObjects: "9.4.5"
    telemetry:
      enabled: False
    xpack:
@@ -9,5 +9,5 @@ SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http:
 # Disable certain Features from showing up in the Kibana UI
 echo
 echo "Setting up default Kibana Space:"
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV1","entityManager","streams","cloudConnect","slo"]} ' >> /opt/so/log/kibana/misc.log
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","securitySolutionCasesV3","inventory","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV4","securitySolutionAlertsV1","entityManager","slo","streams","anonymization","searchInferenceEndpoints","cloudConnect","queryActivity","automatic_import","stackAlerts","monitoring","dataQuality","actions"]} ' >> /opt/so/log/kibana/misc.log
 echo
@@ -81,14 +81,6 @@ ls_custom_pipeline_conf_{{assigned_pipeline}}_{{pipeline}}:


 {% for assigned_pipeline in ASSIGNED_PIPELINES %}
-{# a blank per-pipeline setting falls back to the global logstash.yml value #}
-{% set PARSED_OVERRIDES = LOGSTASH_MERGED.get('pipeline_settings', {}).get(assigned_pipeline, {}) %}
-{% if PARSED_OVERRIDES is not mapping %}
-{%   do salt.log.warning('logstash: ignoring malformed pipeline_settings for pipeline ' ~ assigned_pipeline ~ '; expected a set of settings') %}
-{% endif %}
-{% set PIPELINE_OVERRIDES = PARSED_OVERRIDES if PARSED_OVERRIDES is mapping else {} %}
-{% set THREADS = PIPELINE_OVERRIDES.get('pipeline_x_workers') or LOGSTASH_MERGED.config.pipeline_x_workers %}
-{% set BATCH = PIPELINE_OVERRIDES.get('pipeline_x_batch_x_size') or LOGSTASH_MERGED.config.pipeline_x_batch_x_size %}
    {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %}
 ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}:
  file.managed:
@@ -100,8 +92,8 @@ ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_")
        GLOBALS: {{ GLOBALS }}
        ES_USER: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') }}"
        ES_PASS: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') }}"
-        THREADS: {{ THREADS }}
-        BATCH: {{ BATCH }}
+        THREADS: {{ LOGSTASH_MERGED.config.pipeline_x_workers }}
+        BATCH: {{ LOGSTASH_MERGED.config.pipeline_x_batch_x_size }}
      {% else %}
    - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1]}}
      {% endif %}
@@ -133,14 +125,6 @@ lspipelinesyml:
    - defaults:
        ASSIGNED_PIPELINES: {{ ASSIGNED_PIPELINES }}

-lslog4j2:
-  file.managed:
-    - name: /opt/so/conf/logstash/etc/log4j2.properties
-    - source: salt://logstash/etc/log4j2.properties.jinja
-    - template: jinja
-    - user: 931
-    - group: 939
-
 lsetcsync:
  file.recurse:
    - name: /opt/so/conf/logstash/etc
@@ -149,11 +133,7 @@ lsetcsync:
    - group: 939
    - template: jinja
    - clean: True
-{#- both names are matched: the .jinja source so the recurse does not copy it verbatim,
-    and the rendered file so clean: True does not delete what lslog4j2 wrote #}
-    - exclude_pat:
-      - pipelines*
-      - log4j2.properties*
+    - exclude_pat: pipelines*
    - defaults:
        LOGSTASH_MERGED: {{ LOGSTASH_MERGED }}

@@ -42,11 +42,6 @@ logstash:
    custom2: []
    custom3: []
    custom4: []
-    custom5: []
-    custom6: []
-    custom7: []
-    custom8: []
-    custom9: []
  pipeline_config:
    custom001: |-
      filter {
@@ -65,405 +60,10 @@ logstash:
    custom008: PLACEHOLDER
    custom009: PLACEHOLDER
    custom010: PLACEHOLDER
-  pipeline_settings:
-    fleet:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    manager:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    receiver:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    search:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom0:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom1:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom2:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom3:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom4:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom5:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom6:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom7:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom8:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom9:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
  settings:
    lsheap: 500m
  config:
    api_x_http_x_host: 0.0.0.0
-    log_x_level: info
-    log_x_format: plain
    path_x_logs: /var/log/logstash
    pipeline_x_workers: 1
    pipeline_x_batch_x_size: 125
@@ -105,8 +105,6 @@ so-logstash:
    {% endif %}
    - watch:
      - file: lsetcsync
-      - file: lslog4j2
-      - file: lspipelinesyml
      - file: trusttheca
      {% if GLOBALS.is_manager %}
      - file: elasticsearch_cacerts
@@ -1,4 +1,3 @@
-{%- from 'logstash/map.jinja' import LOGSTASH_MERGED -%}
 status = error
 name = LogstashPropertiesConfig

@@ -17,14 +16,8 @@ name = LogstashPropertiesConfig
 appender.rolling.type = RollingFile
 appender.rolling.name = rolling
 appender.rolling.fileName = /var/log/logstash/logstash.log
-{%- if LOGSTASH_MERGED.config.get('log_x_format', 'plain') == 'json' %}
-appender.rolling.layout.type = JSONLayout
-appender.rolling.layout.compact = true
-appender.rolling.layout.eventEol = true
-{%- else %}
 appender.rolling.layout.type = PatternLayout
 appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %.10000m%n
-{%- endif %}
 appender.rolling.filePattern = /var/log/logstash/logstash-%d{yyyy-MM-dd}.log.gz
 appender.rolling.policies.type = Policies
 appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
@@ -39,5 +32,7 @@ appender.rolling.strategy.action.condition.type = IfFileName
 appender.rolling.strategy.action.condition.glob = *.gz
 appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified
 appender.rolling.strategy.action.condition.nested_condition.age = 7D
-rootLogger.level = ${sys:ls.log.level}
+rootLogger.level = info
 rootLogger.appenderRef.rolling.ref = rolling
+#rootLogger.level = ${sys:ls.log.level}
+#rootLogger.appenderRef.console.ref = ${sys:ls.log.format}_console
@@ -1,17 +1,4 @@
-{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
-{%- set PIPELINE_SETTINGS = LOGSTASH_MERGED.get('pipeline_settings', {}) %}
 {%- for assigned_pipeline in ASSIGNED_PIPELINES %}
 - pipeline.id: {{ assigned_pipeline }}
  path.config: "/usr/share/logstash/pipelines/{{ assigned_pipeline }}/"
-{%-   set extra = PIPELINE_SETTINGS.get(assigned_pipeline, {}) %}
-{%-   if extra is mapping %}
-{#-     values are emitted unquoted so yaml re-infers the type logstash expects:
-        4 as an integer, false as a boolean, 1024mb and auto as strings #}
-{%-     for key, value in extra | dictsort %}
-{%-       set rendered = key | replace('_x_', '.') %}
-{%-       if value not in ['', None] and rendered not in ['pipeline.id', 'path.config'] %}
-  {{ rendered }}: {{ value }}
-{%-       endif %}
-{%-     endfor %}
-{%-   endif %}
 {% endfor -%}
@@ -16,7 +16,6 @@ logstash:
      heavynode: *assigned_pipelines
      searchnode: *assigned_pipelines
      manager: *assigned_pipelines
-      managerhype: *assigned_pipelines
      managersearch: *assigned_pipelines
      fleet: *assigned_pipelines
  defined_pipelines:
@@ -35,11 +34,6 @@ logstash:
    custom2: *defined_pipelines
    custom3: *defined_pipelines
    custom4: *defined_pipelines
-    custom5: *defined_pipelines
-    custom6: *defined_pipelines
-    custom7: *defined_pipelines
-    custom8: *defined_pipelines
-    custom9: *defined_pipelines
  pipeline_config:
    custom001: &pipeline_config
      description: Pipeline configuration for Logstash
@@ -57,351 +51,6 @@ logstash:
    custom008: *pipeline_config
    custom009: *pipeline_config
    custom010: *pipeline_config
-  pipeline_settings:
-    manager: &pipeline_settings
-      pipeline_x_workers:
-        description: >-
-          Number of worker threads that run filters and outputs for this pipeline. May be set higher
-          than the CPU core count when outputs spend time waiting on I/O. Leave blank to use the value
-          from logstash.yml.
-        title: pipeline.workers
-        regex: '^$|^[1-9][0-9]*$'
-        regexFailureMessage: Must be blank, or a positive whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_batch_x_size:
-        description: >-
-          Maximum number of events an individual worker thread collects before running filters and
-          outputs. Larger batches are more efficient but increase heap use; total in-flight events is
-          workers multiplied by batch size. Leave blank to use the value from logstash.yml.
-        title: pipeline.batch.size
-        regex: '^$|^[1-9][0-9]*$'
-        regexFailureMessage: Must be blank, or a positive whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_batch_x_delay:
-        description: >-
-          Milliseconds a worker waits for the next event before running a batch that is not yet full.
-          Leave blank to use the value from logstash.yml.
-        title: pipeline.batch.delay
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_batch_x_metrics_x_sampling_mode:
-        description: >-
-          Controls how often batch size metrics are collected for this pipeline, which helps tune
-          pipeline.batch.size to the batch sizes actually being processed. Fuller sampling consumes
-          additional heap. Elastic marks this setting as a technical preview that may change in a
-          future release. Leave blank to use the value from logstash.yml.
-        title: pipeline.batch.metrics.sampling_mode
-        options:
-        - ''
-        - 'disabled'
-        - 'minimal'
-        - 'full'
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_ordered:
-        description: >-
-          Whether event order is preserved through this pipeline. auto enables ordering only when
-          pipeline.workers is explicitly set to 1, and does nothing otherwise. Setting this to true
-          requires pipeline.workers to be 1 as well; with more workers this pipeline fails to start.
-          Leave blank to use the value from logstash.yml.
-        title: pipeline.ordered
-        options:
-        - ''
-        - 'auto'
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_ecs_compatibility:
-        description: >-
-          Elastic Common Schema compatibility mode for plugins in this pipeline. Security Onion sets
-          this globally and it should rarely be changed per pipeline. Elastic considers values other
-          than disabled to be BETA, and they may produce unintended consequences when upgrading
-          Logstash. Leave blank to use the value from logstash.yml.
-        title: pipeline.ecs_compatibility
-        options:
-        - ''
-        - 'disabled'
-        - 'v1'
-        - 'v8'
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_reloadable:
-        description: >-
-          Whether this pipeline may be reloaded when its configuration changes. Leave blank to use the
-          value from logstash.yml.
-        title: pipeline.reloadable
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_type:
-        description: >-
-          Queue backing this pipeline. persisted buffers events to disk under /nsm/logstash so they
-          survive a restart, at some throughput cost; memory does not. Leave blank to use the value
-          from logstash.yml.
-        title: queue.type
-        options:
-        - ''
-        - 'memory'
-        - 'persisted'
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_max_bytes:
-        description: >-
-          Total capacity of the persistent queue for this pipeline, in bytes. Only applies when
-          queue.type is persisted. The disk backing /nsm/logstash must have room for this much data or
-          the pipeline fails to start, reporting that it was unable to allocate the space. If both
-          queue.max_events and queue.max_bytes are set, whichever is reached first applies. Leave
-          blank to use the value from logstash.yml.
-        title: queue.max_bytes
-        regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$'
-        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb, or 64k. Units are lowercase.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_page_capacity:
-        description: >-
-          Size of the individual append-only page data files that make up the persistent queue for
-          this pipeline. Only applies when queue.type is persisted. Leave blank to use the value from
-          logstash.yml.
-        title: queue.page_capacity
-        regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$'
-        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb, or 64k. Units are lowercase.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_max_events:
-        description: >-
-          Maximum number of unread events in the persistent queue for this pipeline. 0 means
-          unlimited. Only applies when queue.type is persisted. Leave blank to use the value from
-          logstash.yml.
-        title: queue.max_events
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_checkpoint_x_acks:
-        description: >-
-          Maximum number of acknowledged events before a checkpoint is forced. 0 means unlimited. Only
-          applies when queue.type is persisted. Leave blank to use the value from logstash.yml.
-        title: queue.checkpoint.acks
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_checkpoint_x_writes:
-        description: >-
-          Maximum number of written events before a checkpoint is forced. Setting this to 1 gives
-          maximum durability at a severe performance cost. 0 means unlimited. Only applies when
-          queue.type is persisted. Leave blank to use the value from logstash.yml.
-        title: queue.checkpoint.writes
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_checkpoint_x_interval:
-        description: >-
-          Milliseconds between forced checkpoints on the persistent queue head page. 0 eliminates
-          periodic checkpoints. Deprecated by Elastic as of Logstash 9.1. Only applies when queue.type
-          is persisted. Leave blank to use the value from logstash.yml.
-        title: queue.checkpoint.interval
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_checkpoint_x_retry:
-        description: >-
-          When enabled, Logstash retries four times per attempted checkpoint write that fails; later
-          errors are not retried. Elastic describes this as a workaround for failed checkpoint writes
-          seen only on Windows and on filesystems with non-standard behaviour such as SANs, and does
-          not recommend enabling it otherwise. Only applies when queue.type is persisted. Leave blank
-          to use the value from logstash.yml.
-        title: queue.checkpoint.retry
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_compression:
-        description: >-
-          Compression applied to persistent queue pages for this pipeline, trading CPU for disk: speed
-          favours the fastest operation, size the smallest files, and balanced sits between them. Once
-          compressed events have been written, that queue cannot be read by Logstash releases earlier
-          than 9.2. Only applies when queue.type is persisted. Leave blank to use the value from
-          logstash.yml.
-        title: queue.compression
-        options:
-        - ''
-        - 'none'
-        - 'speed'
-        - 'balanced'
-        - 'size'
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_drain:
-        description: >-
-          When enabled, Logstash waits for the persistent queue to drain before shutting down this
-          pipeline. Draining a large queue makes shutdown take considerably longer. Only applies when
-          queue.type is persisted. Leave blank to use the value from logstash.yml.
-        title: queue.drain
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_enable:
-        description: >-
-          Whether events this pipeline cannot process are written to a dead letter queue instead of
-          being dropped. Leave blank to use the value from logstash.yml.
-        title: dead_letter_queue.enable
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_max_bytes:
-        description: >-
-          Total capacity of the dead letter queue for this pipeline, in bytes. Only applies when
-          dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
-        title: dead_letter_queue.max_bytes
-        regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$'
-        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb, or 64k. Units are lowercase.
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_flush_interval:
-        description: >-
-          Milliseconds before an incomplete dead letter queue segment is flushed and made available to
-          the dead_letter_queue input. Lower values write more, smaller segment files; higher values
-          add latency before events can be read. Only applies when dead_letter_queue.enable is true.
-          Leave blank to use the value from logstash.yml.
-        title: dead_letter_queue.flush_interval
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_flush_check_interval:
-        description: >-
-          Milliseconds between checks for a stale dead letter queue segment needing a flush. Cannot be
-          set lower than 1000. Smaller values rotate segments sooner at the cost of CPU. Only applies
-          when dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
-        title: dead_letter_queue.flush_check_interval
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_storage_policy:
-        description: >-
-          Action taken when dead_letter_queue.max_bytes is reached: drop_newer stops accepting new
-          events, drop_older removes the oldest events to make room. Only applies when
-          dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
-        title: dead_letter_queue.storage_policy
-        options:
-        - ''
-        - 'drop_newer'
-        - 'drop_older'
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_retain_x_age:
-        description: >-
-          How long an event is kept in the dead letter queue before Logstash removes it, such as 5d.
-          Units are d, h, m and s; there is no default unit, so one must be given. Only applies when
-          dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
-        title: dead_letter_queue.retain.age
-        regex: '^$|^[0-9]+\s*[dhms]$'
-        regexFailureMessage: Must be blank, or a number followed by d, h, m, or s, such as 5d.
-        advanced: True
-        global: False
-        helpLink: logstash
-      path_x_queue:
-        description: >-
-          Directory inside the Logstash container holding the persistent queue for this pipeline. The
-          default lives under the /nsm/logstash bind mount; a path outside it will not survive a
-          container restart. Logstash creates the directory if it is missing, requires it to be
-          writable, and refuses to start if the path is a symlink. Only applies when queue.type is
-          persisted. Leave blank to use the value from logstash.yml.
-        title: path.queue
-        advanced: True
-        global: False
-        helpLink: logstash
-      path_x_dead_letter_queue:
-        description: >-
-          Directory inside the Logstash container holding the dead letter queue for this pipeline. The
-          default lives under the /nsm/logstash bind mount; a path outside it will not survive a
-          container restart. Logstash creates the directory if it is missing, requires it to be
-          writable, and refuses to start if the path is a symlink. Only applies when
-          dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
-        title: path.dead_letter_queue
-        advanced: True
-        global: False
-        helpLink: logstash
-      config_x_debug:
-        description: >-
-          Whether the fully compiled configuration for this pipeline is written to the log. The output
-          may contain sensitive values from the pipeline configuration. Leave blank to use the value
-          from logstash.yml.
-        title: config.debug
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      config_x_support_escapes:
-        description: >-
-          Whether escape sequences such as \n and \t in this pipeline's quoted strings are
-          interpreted. Leave blank to use the value from logstash.yml.
-        title: config.support_escapes
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-    fleet: *pipeline_settings
-    receiver: *pipeline_settings
-    search: *pipeline_settings
-    custom0: *pipeline_settings
-    custom1: *pipeline_settings
-    custom2: *pipeline_settings
-    custom3: *pipeline_settings
-    custom4: *pipeline_settings
-    custom5: *pipeline_settings
-    custom6: *pipeline_settings
-    custom7: *pipeline_settings
-    custom8: *pipeline_settings
-    custom9: *pipeline_settings
  settings:
    lsheap: 
      description: Heap size to use for logstash
@@ -413,35 +62,6 @@ logstash:
      helpLink: logstash
      readonly: True
      advanced: True
-    log_x_level:
-      description: >-
-        Verbosity of the Logstash log at /opt/so/log/logstash/logstash.log. debug and trace produce
-        a very large volume of log data on a busy node and should be used only while troubleshooting;
-        the log rotates at 1GB and rotated files are deleted after 7 days. Setting this to debug is
-        also what makes the per-pipeline config.debug setting emit anything.
-      title: log.level
-      options:
-      - 'fatal'
-      - 'error'
-      - 'warn'
-      - 'info'
-      - 'debug'
-      - 'trace'
-      advanced: True
-      global: False
-      helpLink: logstash
-    log_x_format:
-      description: >-
-        Layout of the Logstash log. plain writes human readable lines; json writes one JSON object
-        per line, which is easier to parse but harder to read directly. The file name and location
-        do not change.
-      title: log.format
-      options:
-      - 'plain'
-      - 'json'
-      advanced: True
-      global: False
-      helpLink: logstash
    path_x_logs: 
      description: Path inside the container to wrote logs.
      helpLink: logstash
@@ -344,16 +344,6 @@ check_cluster_health() {

 check_fleet_server() {
  echo "Checking that Elastic Fleet Server is responding."
-  # Before checking fleet health, check for and fix known issue with elastic-agent container and fs.protected_symlinks
-  local protected_symlinks=$(sysctl -b fs.protected_symlinks)
-  if [[ "$protected_symlinks" == "1" ]]; then
-    # disable fs.protected_symlinks and restart elasticfleet
-    sysctl -w fs.protected_symlinks=0
-    docker stop so-elastic-fleet; docker rm -f so-elastic-fleet
-    printf "\nUpdated sysctl fs.protected_symlinks. Restarting fleet before running health check and continuing with soup.\n"
-    salt-call state.apply elasticfleet queue=True
-  fi
-  
  # Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls,
  # which waits for HTTP 200 from the Fleet Server status API.
  if curl -sk --fail --retry 3 --retry-delay 10 --max-time 30 "https://localhost:8220/api/status" > /dev/null 2>&1; then
@@ -465,6 +455,19 @@ highstate() {
  salt-call state.highstate -l info queue=True
 }

+upgrade_searchnode_elasticsearch() {
+  # Run the elasticsearch state across the true elastic cluster (non-heavy) with a retry attempt
+  #   Excludes the manager, so that kibana & elasticfleet are not upgraded until searchnodes are upgraded.
+  echo "Getting ready to upgrade Elasticsearch across the grid. This may take a while..."
+  if salt -C "I@elasticsearch:enabled and not G@role:so-heavynode and not G@id:${MINIONID}" state.apply elasticsearch queue=True batch=10%; then
+    return 0
+  fi
+  
+  echo "Initial elasticsearch state attempt had a problem; retrying in 30 seconds."
+  sleep 30
+  salt -C "I@elasticsearch:enabled and not G@role:so-heavynode and not G@id:${MINIONID}" state.apply elasticsearch queue=True batch=10%
+}
+
 push_grid_highstate() {
  # Drive a batched, role-tiered highstate across the rest of the grid so remote minions
  # pick up this upgrade now instead of waiting up to ~2.5 hours for their own scheduled
@@ -493,11 +496,10 @@ push_grid_highstate() {
 masterlock() {
  echo "Locking Salt Master"
  mv -v $TOPFILE $BACKUPTOPFILE
-  # Render the real top file only for the host running soup; every other
-  # minion gets an empty top (no states) while the master is upgrading.
-  echo "{% if grains['id'] == '$MINIONID' %}" > $TOPFILE
-  cat $BACKUPTOPFILE >> $TOPFILE
-  echo "{% endif %}" >> $TOPFILE
+  echo "base:" > $TOPFILE
+  echo "  $MINIONID:" >> $TOPFILE
+  echo "    - ca" >> $TOPFILE
+  echo "    - elasticsearch" >> $TOPFILE
 }

 masterunlock() {
@@ -1046,20 +1048,8 @@ post_to_3.2.0() {
 }
 ### 3.2.0 End ###

-### 3.3.0 Scripts ###
-
- # Sets fs.protected_symlinks=0
- #
- # Elastic Agent docker image chowns its directory to the running UID 
- # but does not chown the elastic-agent launcher symlink. 
- # Preventing non-root users from following that launcher symlink.
-disable_sysctl_fs_protected_symlink() {
-  salt -C 'I@stig:enabled' state.single sysctl.present name=fs.protected_symlinks value=0 config=/etc/sysctl.conf || true
-}
-    
+### 3.2.0 Scripts ###
 up_to_3.3.0() {
-  disable_sysctl_fs_protected_symlink
-  
  INSTALLEDVERSION=3.3.0
 }

@@ -1359,11 +1349,12 @@ verify_es_version_compatibility() {
    local is_active_intermediate_upgrade=1
    # supported upgrade paths for SO-ES versions
    declare -A es_upgrade_map=(
-        ["8.18.4"]="8.18.6 8.18.8 9.0.8"
+      ["8.18.4"]="8.18.6 8.18.8 9.0.8"
 	    ["8.18.6"]="8.18.8 9.0.8"
 	    ["8.18.8"]="9.0.8"
-        ["9.0.8"]="9.3.3 9.3.7"
-        ["9.3.3"]="9.3.7"
+      ["9.0.8"]="9.3.3 9.3.7 9.4.5"
+      ["9.3.3"]="9.3.7 9.4.5"
+      ["9.3.7"]="9.4.5"
    )

    # Elasticsearch MUST upgrade through these versions
@@ -2139,12 +2130,28 @@ main() {

    # ensure the mine is updated and populated before highstates run, following the salt-master restart
    update_salt_mine
-
+    
+    # kick off a searchnode elasticsearch upgrade
+    set +e
+    if [[ "$es_version" != "$target_es_version" ]]; then
+      if salt-key -L accepted | grep -q "_searchnode$" 2>/dev/null; then
+        # only run if there is atleast 1 searchnode
+        upgrade_searchnode_elasticsearch
+      fi
+    fi
+    set -e
+    
    highstate
    check_saltmaster_status
    postupgrade_changes
    [[ $is_airgap -eq 0 ]] && unmount_update
-
+    
+    if [[ "$es_version" != "$target_es_version" ]]; then
+      # Run final elasticsearch / fleet state on manager to ensure addon index templates are created/regenerated and loaded
+      echo "Running final Elastic states at $(date +"%T.%6N"), after upgrade to $NEWVERSION"
+      salt-call state.apply elasticsearch,elasticfleet queue=True
+    fi
+    
    echo ""
    echo "Upgrade to $NEWVERSION complete."

@@ -29,8 +29,6 @@ psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-E
    -- revoking CONNECT closes the soft edge entirely.
    REVOKE CONNECT ON DATABASE "$POSTGRES_DB" FROM PUBLIC;
    GRANT CONNECT ON DATABASE "$POSTGRES_DB" TO "$SO_POSTGRES_USER";
-
-    CREATE EXTENSION IF NOT EXISTS vector;
 EOSQL

 # Bootstrap the Telegraf metrics database. Per-minion roles + schemas are
@@ -3,8 +3,6 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-{% from 'vars/globals.map.jinja' import GLOBALS %}
-
 # Manages /etc/systemd/system/so-boot-highstate.service, a Type=oneshot
 # RemainAfterExit=yes unit that runs `salt-call state.highstate` exactly once
 # per system boot. Replaces the legacy `startup_states: highstate` minion
@@ -21,25 +19,9 @@ so_boot_highstate_unit_file:
    - onchanges_in:
      - module: systemd_reload

-# Non-managers never apply salt.minion during setup, so reaching this state means
-# setup is finished and the marker is safe to write unconditionally. This also
-# heals nodes installed before this fix, which have no marker and no legacy
-# startup_states line to grep for. Managers do highstate mid-setup, so they only
-# get the marker from the legacy upgrade signal; fresh installs get it from
-# mark_setup_complete in setup/so-functions.
-mark_setup_complete:
-  file.managed:
-    - name: /opt/so/state/setup-complete
-    - replace: false
-    - makedirs: True
-{% if GLOBALS.is_manager %}
-    - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion"
-{% endif %}
-    - require_in:
-      - service: so_boot_highstate_service
-
 # Only enable once setup is complete. Until then the gate file is missing and
-# the unit's own ConditionPathExists would no-op it anyway.
+# the unit's own ConditionPathExists would no-op it anyway -- this just keeps
+# `systemctl is-enabled` honest for the sync_es_users gate.
 so_boot_highstate_service:
  service.enabled:
    - name: so-boot-highstate.service
@@ -87,15 +87,27 @@ set_log_levels:
 # so-boot-highstate.service (managed in salt.minion.boot_highstate), which
 # runs once per system boot only. Strip the line from /etc/salt/minion on
 # upgrade; both the commented and uncommented forms historically existed.
-# Ordered after mark_setup_complete (salt.minion.boot_highstate); the manager
-# gate there greps for this line, so it must run before we delete it.
 remove_startup_states:
  file.line:
    - name: /etc/salt/minion
    - match: 'startup_states: highstate'
    - mode: delete
-    - require:
-      - file: mark_setup_complete
+
+# Upgrade-path bridge: systems that already passed setup under the old gate
+# (`grep -x 'startup_states: highstate' /etc/salt/minion`) get a /opt/so/state/setup-complete
+# marker so so-boot-highstate.service can be enabled and the so-user_sync cron
+# in sync_es_users.sls keeps installing. Setup-in-progress systems instead get
+# the marker from `mark_setup_complete` in setup/so-functions at the right
+# moment. `replace: false` means we never overwrite a marker once written.
+mark_setup_complete_for_upgrades:
+  file.managed:
+    - name: /opt/so/state/setup-complete
+    - replace: false
+    - makedirs: True
+    - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion"
+    - require_in:
+      - file: remove_startup_states
+      - service: so_boot_highstate_service

 {% endif %}

@@ -8,15 +8,6 @@ set_role_grain:
    - name: role
    - value: so-{{ grains.id.split("_") | last }}

-# salt-cloud guests never run so-setup, so nothing else marks them setup-complete.
-# Replaces the 'startup_states: highstate' line this state used to append. No
-# GLOBALS import -- this runs before the guest's pillars exist.
-mark_setup_complete_vm_guest:
-  file.managed:
-    - name: /opt/so/state/setup-complete
-    - replace: false
-    - makedirs: True
-
 enable_salt_minion:
  service.enabled:
    - name: salt-minion
@@ -1537,18 +1537,6 @@ soc:
            Orchestrator: sonnet@SOAI
            Investigator: gemma@SOAI
            DetectionEngineer: gemma@SOAI
-          useMemory: true
-          useMemoryScanner: false
-          memoryScanIntervalSeconds: 300
-          memoryProximityThreshold: 0.8
-          messageProximityThreshold: 0.5
-          maxUserMemoriesToInclude: 5
-          maxGlobalMemoriesToInclude: 5
-          maxUserMemoriesToReconcile: 20
-          maxGlobalMemoriesToReconcile: 20
-          memoryModel: gemma@SOAI
-          embedModel: amazon.titan-embed-text-v2@SOAI
-          reconcileModel: gemma@SOAI
        onionconfig:
          saltstackDir: /opt/so/saltstack
          bypassEnabled: false
@@ -2683,7 +2671,7 @@ soc:
              # The id (UUIDv4) is pregenerated and can safely be used.
              # Click "Convert" to convert the Sigma rule to use Security Onion field mappings within an EQL query
              #
-              # Rule Creation Guide: https://github.com/SigmaHQ/sigma/wiki/Rule-Creation-High%E2%80%90Level-Guide
+              # Rule Creation Guide: https://github.com/SigmaHQ/sigma/wiki/Rule-Creation-Guide
              # Logsources: https://sigmahq.io/docs/basics/log-sources.html

              title: 'A Short Capitalized Title With Less Than 50 Characters'
@@ -2695,7 +2683,7 @@ soc:
              references:
                - 'https://local.invalid'
              author: '@SecurityOnion'
-              date: '[today]'
+              date: 'YYYY/MM/DD'
              tags:
                - detection.threat_hunting
                - attack.technique_id
@@ -2739,14 +2727,5 @@ soc:
              enabled: true
              adapter: SOAI
              charsPerTokenEstimate: 4
-            - id: amazon.titan-embed-text-v2
-              displayName: amazon.titan-embed-text-v2
-              origin: USA
-              contextLimitSmall: 8192
-              contextLimitLarge: 8192
-              lowBalanceColorAlert: 500000
-              enabled: true
-              adapter: SOAI
-              charsPerTokenEstimate: 4
              
            
@@ -845,45 +845,6 @@ soc:
            DetectionEngineer:
              description: This agent manages detections and their overrides, including tuning noisy rules and authoring rule content.
              global: True
-          useMemory:
-            description: Enables the Memory system for OnionAI
-            global: True
-            forcedType: bool
-          useMemoryScanner:
-            description: Enables the memory scanner for automatic memory extraction from historical sessions.
-            global: True
-            forcedType: bool
-          memoryScanIntervalSeconds:
-            description: How long to wait in seconds between attempts to scan sessions for new memories.
-            global: True
-          memoryProximityThreshold:
-            description: Describes how close memories need to be on a floating point scale from 0.0 to 1.0 to be considered when reconciling new memories with old ones. This value is usually higher than messageProximityThreshold.
-            global: True
-          messageProximityThreshold:
-            description: Describes how close a memory needs to be to a user's message on a floating point scale from 0.0 to 1.0 to be included in the context. This value is usually lower than memoryProximityThreshold.
-            global: True
-          maxUserMemoriesToInclude:
-            description: Specify the max number of user-specific memories to include in the prompt when a user sends a message.
-            global: True
-          maxGlobalMemoriesToInclude:
-            description: Specify the max number of global memories to include in the prompt when a user sends a message.
-            global: True
-          maxUserMemoriesToReconcile:
-            description: When reconciling new user-specific memories with existing user-specific memories, this determines how many old memories may be considered.
-            global: True
-          maxGlobalMemoriesToReconcile:
-            description: When reconciling new global memories with existing global memories, this determines how many old memories may be considered.
-            global: True
-          memoryModel:
-            description: The model to use when extracting memories from sessions.
-            global: True
-          embedModel: 
-            description: The model to use when embedding a memory as a vector. Note that only memories embedded using the same model may be compared and only memories created with the model specified here will be considered when informing an agent of existing memories.
-            global: True
-            advanced: True
-          reconcileModel:
-            description: The model to use when reconciling memories that contain nearly the same content.
-            global: True
      client:
        assistant:
          enabled:
@@ -65,15 +65,6 @@ run_remediate:
    - success_retcodes:
      - 2

-# Elastic Agent docker image chowns its directory to the running UID but does not
-#  chown the elastic-agent launcher symlink. fs.protected_symlinks=1 then prevents
-#  non-root users from following that launcher symlink.
-{# OSCAP rule id: xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks #}
-fs.protected_symlinks:
-  sysctl.present:
-    - value: 0
-    - config: /etc/sysctl.conf
-
 {# OSCAP rule id: xccdf_org.ssgproject.content_rule_disable_ctrlaltdel_burstaction #}
 disable_ctrl_alt_del_action:
  file.replace:
@@ -1601,7 +1601,7 @@ DISA STIG for Oracle Linux 9 V1R3.</xccdf-1.2:description>
        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sudoers_validate_passwd" selected="true"/>
        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_crypto_fips_enabled" selected="true"/>
        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_hardlinks" selected="true"/>
-        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="false"/>
+        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="true"/>
        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_core_pattern" selected="true"/>
        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_dmesg_restrict" selected="true"/>
        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_exec_shield" selected="true"/>
@@ -2202,7 +2202,7 @@ standard DISA STIG for Oracle Linux 9 profile.</xccdf-1.2:description>
        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sudoers_validate_passwd" selected="true"/>
        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_crypto_fips_enabled" selected="true"/>
        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_hardlinks" selected="true"/>
-        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="false"/>
+        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="true"/>
        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_core_pattern" selected="true"/>
        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_dmesg_restrict" selected="true"/>
        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_exec_shield" selected="true"/>
@@ -335,7 +335,7 @@
 {%-   do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('sostatus.sh') %}
 [[inputs.exec]]
   commands = [
-      ["/scripts/sostatus.sh"]
+      "/scripts/sostatus.sh"
   ]
   data_format = "influx"
   timeout = "15s"
@@ -346,7 +346,7 @@
 [[inputs.exec]]
   commands = [
 {%-   for script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %}
-      ["/scripts/{{script}}"]{% if not loop.last %},{% endif %}
+      "/scripts/{{script}}"{% if not loop.last %},{% endif %}
 {%-   endfor %}
   ]
  data_format = "influx"
@@ -375,7 +375,7 @@
 {%- if GLOBALS.is_manager or GLOBALS.role == 'so-heavynode' %}
 [[ inputs.exec ]]
  commands = [
-    ["/scripts/esindexsize.sh"]
+    "/scripts/esindexsize.sh"
  ]
  data_format = "influx"
  interval = "1h"
@@ -833,7 +833,6 @@ if ! [[ -f $install_opt_file ]]; then
 		check_sos_appliance
 		drop_install_options
 		hypervisor_local_states
-		mark_setup_complete
 		verify_setup
 	fi
Author	SHA1	Message	Date
reyesj2	3fee98d9ce	ft	2026-08-28 13:46:50 -05:00
reyesj2	dd035beec4	include fleet state	2026-08-28 13:45:55 -05:00
reyesj2	0dbb7803ef	Merge branch 'reyesj2/reworksoup' into reyesj2/es945	2026-08-28 13:28:53 -05:00
reyesj2	30deb00277	use correct version variable	2026-08-28 13:27:44 -05:00
reyesj2	192363bc2f	Merge branch 'reyesj2/reworksoup' into reyesj2/es945	2026-08-28 12:19:20 -05:00
reyesj2	3d8f86883a	after an ES upgrade run a final elasticsearch state to create/regenerate any needed addon index templates	2026-08-28 12:16:08 -05:00
reyesj2	fdb975fdef	Merge branch 'reyesj2/reworksoup' into reyesj2/es945	2026-08-27 22:12:44 -05:00
reyesj2	f8401bef37	exclude elasticsearch indexing error during upgrade for temporarily outdated policies	2026-08-27 21:17:27 -05:00
reyesj2	4786d359fb	exclude telegraf error during elasticsearch upgrade / master election	2026-08-27 14:39:46 -05:00
reyesj2	d771fbc444	upgrade integration policies directly after integration package upgrade	2026-08-27 14:18:39 -05:00
reyesj2	85ab4c69e5	rename	2026-08-27 14:17:29 -05:00
reyesj2	cb8e576d6b	run elasticsearch state on remote minions when there is an ES upgrade. Prior to manager completing its first full highstate that includes kibana / elasticfleet	2026-08-27 12:52:59 -05:00
reyesj2	fae1754fec	clean elasticsearch transform prior to elasticsearch integration package upgrade to prevent fleet automatic rollback	2026-08-27 12:50:32 -05:00
reyesj2	d33eb70af6	reverts `83aaa76` #15985 - allow full highstate on manager when locked	2026-08-27 12:12:38 -05:00
reyesj2	0ee8aa8079	revert manager running two highstates	2026-08-26 09:11:39 -05:00
reyesj2	99c3c7f8aa	ES 9.4.5 soup es compatibility update	2026-08-21 12:46:46 -05:00
reyesj2	cef1dcfcee	add additional problematic indices / templates. Also only update index templates if the script patched any index / data stream	2026-08-21 12:16:41 -05:00
reyesj2	247d9cdb34	kibana spaces 9.4.5	2026-08-20 17:00:53 -05:00
reyesj2	088b761190	9.4.5 policy updates	2026-08-20 16:35:46 -05:00
reyesj2	2dcc81ea7d	run so-elasticsearch-systems-indices-patch script every highstate with no op if no unassigned replicas are found for known problematic indices	2026-08-20 14:30:53 -05:00
reyesj2	35f545a858	find known problematic system indices and add missing auto_expand_replicas configuration to prevent yellow cluster	2026-08-19 16:30:38 -05:00
reyesj2	c9a041ddb4	upstream sentinel_one_cloud_funnel integration patched data stream name ref: https://github.com/elastic/integrations/commit/9f1513423ca26e6fded414c4f5a3a89409efb736	2026-08-17 22:08:19 -05:00
reyesj2	de3306e73c	ES 9.4.5	2026-08-17 16:45:41 -05:00