Merge pull request #16201 from Security-Onion-Solutions/reyesj2-patch-1

fix salt batching command

salt/common/tools/sbin/so-log-check

@@ -134,6 +134,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Redis may have been restarted" # Redis likely restarted by salt
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|file already closed"          # Go logging race condition during container restart
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|relation \"audit_settings\" does not exist" # salt checking for changes before SOC starts
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error in plugin: elasticsearch: Unable to retrieve master node information" # expected error while ES is upgrading/electing a master 
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
@@ -240,6 +241,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown"             # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Could not index event to Elasticsearch.*\"version\" => \"9.0.8\"" # Expected during Elastic upgrade temporarily, as policies referencing older pipelines are updated
 fi
 
 RESULT=0
@@ -304,4 +306,4 @@ else
     echo -e "\nResult: One or more errors found"
 fi
 
-exit $RESULT
+exit $RESULT

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -5,7 +5,7 @@
   "package": {
     "name": "endpoint",
     "title": "Elastic Defend",
-    "version": "9.3.1",
+    "version": "9.4.1",
     "requires_root": true
   },
   "enabled": true,

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -29,7 +29,7 @@
               "\\.gz$"
             ],
             "include_files": [],
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.20.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.3\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.20.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.20.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.3\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.22.3\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.9.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.22.3\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.22.3\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.9.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
             ],

salt/elasticfleet/integration-defaults.map.jinja

@@ -16,7 +16,6 @@
     'awsfirehose.metrics': 'aws.cloudwatch',
     'cribl.logs': 'cribl',
     'cribl.metrics': 'cribl',
-    'sentinel_one_cloud_funnel.logins': 'sentinel_one_cloud_funnel.login',
     'azure_application_insights.app_insights': 'azure.app_insights',
     'azure_application_insights.app_state': 'azure.app_state',
     'azure_billing.billing': 'azure.billing',

salt/elasticfleet/manager.sls

@@ -68,6 +68,24 @@ so-elastic-fleet-package-upgrade:
     - require:
       - http: wait_for_so-kibana
 
+# initial so-elasticsearch-templates run is earlier, but it can skip over templates that have component templates not yet installed to avoid elasticsearch rejecting the template.
+so-elasticsearch-templates-after-fleet-packages:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-templates-load
+    - cwd: /opt/so
+    - unless: test -f /opt/so/state/estemplates.txt
+    - require:
+      - cmd: so-elastic-fleet-package-upgrade
+
+so-elastic-fleet-integration-upgrade:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-integration-upgrade
+    - retry:
+        attempts: 3
+        interval: 10
+    - require:
+      - cmd: so-elastic-fleet-package-upgrade
+
 so-elastic-fleet-integrations:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-integration-policy-load
@@ -86,21 +104,13 @@ so-elastic-agent-grid-upgrade:
     - require:
       - http: wait_for_so-kibana
 
-so-elastic-fleet-integration-upgrade:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-integration-upgrade
-    - retry:
-        attempts: 3
-        interval: 10
-    - require:
-      - http: wait_for_so-kibana
-
 {# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
 so-elastic-fleet-addon-integrations:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
     - require:
       - http: wait_for_so-kibana
+      - cmd: so-elasticsearch-templates-after-fleet-packages
 
 {% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
 so-elastic-defend-manage-filters-file-watch:

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade

@@ -10,6 +10,25 @@
 
 PKG_LOAD_FAILURES=0
 PKG_LOAD_FAILURES_NAMES=()
+PKG_UPGRADED=0
+
+cleanup_elasticsearch_fleet_transforms() {
+    local transforms transform_id attempt
+    
+    if ! transforms=$(so-elasticsearch-query "_transform/logs-elasticsearch.index_pivot-default-*" --retry 1 --retry-delay 5); then
+        return 0
+    fi
+
+    while IFS= read -r transform_id; do
+        [ -n "$transform_id" ] || continue
+        for attempt in {1..3}; do
+            if so-elasticsearch-query "_transform/$transform_id?force=true" -XDELETE --fail --retry 1 --retry-delay 5; then
+                break
+            fi
+            sleep 5
+        done
+    done < <(jq -r '.transforms[]?.id' <<< "$transforms")
+}
 
 {%- for PACKAGE in SUPPORTED_PACKAGES %}
 if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && LATEST_VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
@@ -17,10 +36,25 @@ if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && L
     if [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then
         echo "{{ PACKAGE }} integration version $INSTALLED_VERSION is already at the reported latest version $LATEST_VERSION, skipping upgrade."
     else
+{%- if PACKAGE == 'elasticsearch' %}
+        cleanup_elasticsearch_fleet_transforms
+{%- endif %}
         echo "Upgrading {{ PACKAGE }} package from $INSTALLED_VERSION to version $LATEST_VERSION..."
         if ! elastic_fleet_package_install "{{ PACKAGE }}" "$LATEST_VERSION"; then
             PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
             PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
+        # check that package has upgraded to the expected version after install command
+        elif ! LATEST_VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
+            echo "ERROR: Failed to get latest version information for integration {{ PACKAGE }} after upgrade attempt"
+            PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
+            PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
+        elif INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then
+            echo "{{ PACKAGE }} integration upgraded to version $LATEST_VERSION."
+            PKG_UPGRADED=$((PKG_UPGRADED + 1))
+        else
+            echo "ERROR: {{ PACKAGE }} integration still at ${INSTALLED_VERSION:-unknown}; expected $LATEST_VERSION"
+            PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
+            PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
         fi
     fi
 else
@@ -30,6 +64,11 @@ else
 fi
 {%- endfor %}
 
+if [ $PKG_UPGRADED -gt 0 ]; then
+    echo "Elasticsearch template statefiles cleared after $PKG_UPGRADED package upgrade(s), so templates can reload."
+    rm -f /opt/so/state/estemplates.txt /opt/so/state/addon_estemplates.txt
+fi
+
 if [ $PKG_LOAD_FAILURES -gt 0 ]; then
     echo "ERROR: Failed to upgrade $PKG_LOAD_FAILURES package(s):"
     for PKG in "${PKG_LOAD_FAILURES_NAMES[@]}"; do

salt/elasticsearch/cluster.sls

@@ -98,6 +98,13 @@ so-es-cluster-settings:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
       - http: wait_for_so-elasticsearch
+
+so-elasticsearch-system-indices-patch:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-system-indices-patch
+    - require:
+      - http: wait_for_so-elasticsearch
+      - file: so-elasticsearch-system-indices-patch-script
 {%  endif %}
 
 # heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)

salt/elasticsearch/config.sls

@@ -42,6 +42,16 @@ elasticsearch_sbin:
     - file_mode: 755
     - exclude_pat:
       - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state
+      - so-elasticsearch-system-indices-patch
+    - show_changes: False
+
+so-elasticsearch-system-indices-patch-script:
+  file.managed:
+    - name: /usr/sbin/so-elasticsearch-system-indices-patch
+    - source: salt://elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch
+    - user: 930
+    - group: 939
+    - mode: 755
     - show_changes: False
 
 elasticsearch_sbin_jinja:

salt/elasticsearch/defaults.yaml

@@ -1,7 +1,7 @@
 elasticsearch:
   enabled: false
   esheap: '600m'
-  version: 9.3.7
+  version: 9.4.5
   index_clean: true
   data_retention_method: DLM
   vm:

salt/elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch

@@ -0,0 +1,199 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+set -eo pipefail
+
+SETTINGS='{"index":{"auto_expand_replicas":"0-1"}}'
+KIBANA_PASSWORD=
+INDEX_PATTERNS=(
+  '.entity_analytics.risk_score.lookup-*'
+  '.entity_analytics.watchlists.*'
+  '.entity_analytics.monitoring.users-*'
+  '.entity_analytics.entity-leads-*'
+  '.asset-criticality.asset-criticality-*'
+  '.workflows-executions'
+  '.workflows-step-executions'
+  '.entities.v2.latest.security_*'
+  '.entities.v2.history.security_*'
+  'risk-score.risk-score-latest-*'
+)
+DATA_STREAM_PATTERNS=(
+  '.entities.v2.updates.security_*'
+  'risk-score.risk-score-*'
+  '.rule-events'
+  '.alert-actions'
+)
+TEMPLATE_PATTERNS=(
+  'entities_v2_latest_security_default_index_template'
+  'entities_v2_history_security_default_index_template'
+  '.entities_v2_updates_security_default_index_template'
+  '.risk-score.risk-score-default-index-template'
+  '.rule-events'
+  '.alert-actions'
+)
+
+query_es() {
+  if so-elasticsearch-query "$@" --fail --retry 3 --retry-delay 5; then
+    return 0
+  fi
+
+  # retry failed attempts with so_kibana user (system managed indices reject so_elastic user)
+  local query_path="$1"
+  shift
+
+  if [[ -z "$KIBANA_PASSWORD" ]]; then
+    KIBANA_PASSWORD=$(salt-call pillar.get elasticsearch:auth:users:so_kibana_user:pass --out=newline_values_only)
+  fi
+  [[ -n "$KIBANA_PASSWORD" ]] || return 1
+
+  echo "Retrying ${query_path} as so_kibana." >&2
+  curl -K /opt/so/conf/elasticsearch/curl.config --user "so_kibana:${KIBANA_PASSWORD}" \
+    -s -k -L --fail --retry 3 --retry-delay 5 -H 'Content-Type: application/json' "https://localhost:9200/${query_path}" "$@"
+}
+
+# add auto_expand_replicas=0-1 to given index
+set_auto_expand_replicas() {
+  local index="$1"
+
+  echo "Setting auto_expand_replicas to 0-1 on ${index}."
+  query_es "${index}/_settings" -XPUT -d "$SETTINGS" >/dev/null
+}
+
+# resolve index patterns and find each index with an unassigned replica
+unassigned_replicas() {
+  local pattern="$1"
+  local resolved_indices response index
+
+  if ! resolved_indices=$(query_es "_resolve/index/${pattern}?expand_wildcards=all" 2>/dev/null); then
+    return 0
+  fi
+
+  while read -r index; do
+    if ! response=$(query_es "_cat/shards/${index}?format=json&h=index,prirep,state" 2>/dev/null); then
+      continue
+    fi
+    jq -r '.[]? | objects | select(.prirep == "r" and .state == "UNASSIGNED") | .index' <<<"$response"
+  done < <(jq -r '.indices[]?.name' <<<"$resolved_indices")
+}
+
+data_stream_indices() {
+  local pattern="$1"
+  local response
+
+  if ! response=$(query_es "_data_stream/${pattern}?expand_wildcards=all" 2>/dev/null); then
+    return 0
+  fi
+  jq -r '.data_streams[]?.indices[]?.index_name' <<<"$response"
+}
+
+update_system_indices() {
+  local pattern="$1"
+  local index
+
+  while read -r index; do
+    [[ -n "$index" ]] && set_auto_expand_replicas "$index"
+  done < <(unassigned_replicas "$pattern")
+}
+
+# update data stream backing indices with unassigned replicas
+update_system_ds() {
+  local pattern="$1"
+  local index
+
+  while read -r index; do
+    while read -r unassigned_index; do
+      [[ -n "$unassigned_index" ]] && set_auto_expand_replicas "$unassigned_index"
+    done < <(unassigned_replicas "$index")
+  done < <(data_stream_indices "$pattern")
+}
+
+has_unassigned_replicas() {
+  local pattern="$1"
+  local index
+
+  index=$(unassigned_replicas "$pattern" | sed -n '1p')
+  [[ -n "$index" ]]
+}
+
+data_stream_has_unassigned_replicas() {
+  local pattern="$1"
+  local index
+  while read -r index; do
+    has_unassigned_replicas "$index" && return 0
+  done < <(data_stream_indices "$pattern")
+
+  return 1
+}
+
+needs_patch() {
+  local pattern
+  for pattern in "${INDEX_PATTERNS[@]}"; do
+    has_unassigned_replicas "$pattern" && return 0
+  done
+  for pattern in "${DATA_STREAM_PATTERNS[@]}"; do
+    data_stream_has_unassigned_replicas "$pattern" && return 0
+  done
+
+  return 1
+}
+
+# get index templates, update with auto_expand_replicas=0-1, and PUT back. Keeping mappings/settings/aliases in-place
+update_system_templates() {
+  local pattern="$1"
+  local templates name response template auto_expand_replicas
+
+  if ! templates=$(query_es "_index_template/${pattern}" 2>/dev/null); then
+    return 0
+  fi
+  while read -r name; do
+    response=$(query_es "_index_template/${name}")
+    template=$(jq -c '.index_templates[0].index_template' <<<"$response")
+    auto_expand_replicas=$(jq -r '.template.settings["index.auto_expand_replicas"] // .template.settings.index.auto_expand_replicas // empty' <<<"$template")
+    [[ "$auto_expand_replicas" == "0-1" ]] && continue
+
+    template=$(jq '
+            if (.template.settings.index | type) == "object" then
+                .template.settings.index.auto_expand_replicas = "0-1"
+            else
+                .template.settings["index.auto_expand_replicas"] = "0-1"
+            end
+            | del(.created_date_millis, .modified_date_millis)
+        ' <<<"$template")
+    echo "Setting auto_expand_replicas to 0-1 on index template ${name}."
+    query_es "_index_template/${name}" -XPUT -d "$template" >/dev/null
+  done < <(jq -r '.index_templates[]?.name' <<<"$templates")
+}
+
+if [[ "${1:-}" == "--check" ]]; then
+  needs_patch
+  exit $?
+fi
+
+if [[ $# -ne 0 ]]; then
+  echo "Usage: $0 [--check]" >&2
+  exit 1
+fi
+
+patched=false
+for pattern in "${INDEX_PATTERNS[@]}"; do
+  if has_unassigned_replicas "$pattern"; then
+    update_system_indices "$pattern"
+    patched=true
+  fi
+done
+
+for pattern in "${DATA_STREAM_PATTERNS[@]}"; do
+  if data_stream_has_unassigned_replicas "$pattern"; then
+    update_system_ds "$pattern"
+    patched=true
+  fi
+done
+
+if [[ "$patched" == true ]]; then
+  for pattern in "${TEMPLATE_PATTERNS[@]}"; do
+    update_system_templates "$pattern"
+  done
+fi

salt/kibana/defaults.yaml

@@ -22,7 +22,7 @@ kibana:
           - default
           - file
     migrations:
-      discardCorruptObjects: "9.3.7"
+      discardCorruptObjects: "9.4.5"
     telemetry:
       enabled: False
     xpack:

salt/kibana/tools/sbin_jinja/so-kibana-space-defaults

@@ -9,5 +9,5 @@ SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http:
 # Disable certain Features from showing up in the Kibana UI
 echo
 echo "Setting up default Kibana Space:"
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV1","entityManager","streams","cloudConnect","slo"]} ' >> /opt/so/log/kibana/misc.log
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","securitySolutionCasesV3","inventory","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV4","securitySolutionAlertsV1","entityManager","slo","streams","anonymization","searchInferenceEndpoints","cloudConnect","queryActivity","automatic_import","stackAlerts","monitoring","dataQuality","actions"]} ' >> /opt/so/log/kibana/misc.log
 echo

salt/logrotate/defaults.yaml

@@ -220,6 +220,26 @@ logrotate:
       - extension .log
       - dateext
       - dateyesterday
+    /opt/so/log/salt/virtual_node_manager:
+      - daily
+      - rotate 14
+      - missingok
+      - copytruncate
+      - compress
+      - create
+      - extension .log
+      - dateext
+      - dateyesterday
+    /opt/so/log/salt/so-salt-cloud:
+      - daily
+      - rotate 14
+      - missingok
+      - copytruncate
+      - compress
+      - create
+      - extension .log
+      - dateext
+      - dateyesterday
     /opt/so/log/salt/so-soup-grid-highstate:
       - daily
       - rotate 14

salt/logrotate/soc_logrotate.yaml

@@ -140,6 +140,20 @@ logrotate:
       multiline: True
       global: True
       forcedType: "[]string"
+    "/opt/so/log/salt/virtual_node_manager":
+      description: List of logrotate options for this file.
+      title: /opt/so/log/salt/virtual_node_manager
+      advanced: True
+      multiline: True
+      global: True
+      forcedType: "[]string"
+    "/opt/so/log/salt/so-salt-cloud":
+      description: List of logrotate options for this file.
+      title: /opt/so/log/salt/so-salt-cloud
+      advanced: True
+      multiline: True
+      global: True
+      forcedType: "[]string"
     "/opt/so/log/salt/so-soup-grid-highstate":
       description: List of logrotate options for this file.
       title: /opt/so/log/salt/so-soup-grid-highstate

salt/manager/tools/sbin/soup

@@ -344,16 +344,6 @@ check_cluster_health() {
 
 check_fleet_server() {
   echo "Checking that Elastic Fleet Server is responding."
-  # Before checking fleet health, check for and fix known issue with elastic-agent container and fs.protected_symlinks
-  local protected_symlinks=$(sysctl -b fs.protected_symlinks)
-  if [[ "$protected_symlinks" == "1" ]]; then
-    # disable fs.protected_symlinks and restart elasticfleet
-    sysctl -w fs.protected_symlinks=0
-    docker stop so-elastic-fleet; docker rm -f so-elastic-fleet
-    printf "\nUpdated sysctl fs.protected_symlinks. Restarting fleet before running health check and continuing with soup.\n"
-    salt-call state.apply elasticfleet queue=True
-  fi
-  
   # Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls,
   # which waits for HTTP 200 from the Fleet Server status API.
   if curl -sk --fail --retry 3 --retry-delay 10 --max-time 30 "https://localhost:8220/api/status" > /dev/null 2>&1; then
@@ -465,6 +455,19 @@ highstate() {
   salt-call state.highstate -l info queue=True
 }
 
+upgrade_searchnode_elasticsearch() {
+  # Run the elasticsearch state across the true elastic cluster (non-heavy) with a retry attempt
+  #   Excludes the manager, so that kibana & elasticfleet are not upgraded until searchnodes are upgraded.
+  echo "Getting ready to upgrade Elasticsearch across the grid. This may take a while..."
+  if salt -b 10% -C "I@elasticsearch:enabled and not G@role:so-heavynode and not G@id:${MINIONID}" state.apply elasticsearch queue=True; then
+    return 0
+  fi
+  
+  echo "Initial elasticsearch state attempt had a problem; retrying in 30 seconds."
+  sleep 30
+  salt -b 10% -C "I@elasticsearch:enabled and not G@role:so-heavynode and not G@id:${MINIONID}" state.apply elasticsearch queue=True
+}
+
 push_grid_highstate() {
   # Drive a batched, role-tiered highstate across the rest of the grid so remote minions
   # pick up this upgrade now instead of waiting up to ~2.5 hours for their own scheduled
@@ -493,11 +496,10 @@ push_grid_highstate() {
 masterlock() {
   echo "Locking Salt Master"
   mv -v $TOPFILE $BACKUPTOPFILE
-  # Render the real top file only for the host running soup; every other
-  # minion gets an empty top (no states) while the master is upgrading.
-  echo "{% if grains['id'] == '$MINIONID' %}" > $TOPFILE
-  cat $BACKUPTOPFILE >> $TOPFILE
-  echo "{% endif %}" >> $TOPFILE
+  echo "base:" > $TOPFILE
+  echo "  $MINIONID:" >> $TOPFILE
+  echo "    - ca" >> $TOPFILE
+  echo "    - elasticsearch" >> $TOPFILE
 }
 
 masterunlock() {
@@ -1046,20 +1048,8 @@ post_to_3.2.0() {
 }
 ### 3.2.0 End ###
 
-### 3.3.0 Scripts ###
-
- # Sets fs.protected_symlinks=0
- #
- # Elastic Agent docker image chowns its directory to the running UID 
- # but does not chown the elastic-agent launcher symlink. 
- # Preventing non-root users from following that launcher symlink.
-disable_sysctl_fs_protected_symlink() {
-  salt -C 'I@stig:enabled' state.single sysctl.present name=fs.protected_symlinks value=0 config=/etc/sysctl.conf || true
-}
-    
+### 3.2.0 Scripts ###
 up_to_3.3.0() {
-  disable_sysctl_fs_protected_symlink
-  
   INSTALLEDVERSION=3.3.0
 }
 
@@ -1359,11 +1349,12 @@ verify_es_version_compatibility() {
     local is_active_intermediate_upgrade=1
     # supported upgrade paths for SO-ES versions
     declare -A es_upgrade_map=(
-        ["8.18.4"]="8.18.6 8.18.8 9.0.8"
+      ["8.18.4"]="8.18.6 8.18.8 9.0.8"
 	    ["8.18.6"]="8.18.8 9.0.8"
 	    ["8.18.8"]="9.0.8"
-        ["9.0.8"]="9.3.3 9.3.7"
-        ["9.3.3"]="9.3.7"
+      ["9.0.8"]="9.3.3 9.3.7 9.4.5"
+      ["9.3.3"]="9.3.7 9.4.5"
+      ["9.3.7"]="9.4.5"
     )
 
     # Elasticsearch MUST upgrade through these versions
@@ -2139,12 +2130,28 @@ main() {
 
     # ensure the mine is updated and populated before highstates run, following the salt-master restart
     update_salt_mine
-
+    
+    # kick off a searchnode elasticsearch upgrade
+    set +e
+    if [[ "$es_version" != "$target_es_version" ]]; then
+      if salt-key -L accepted | grep -q "_searchnode$" 2>/dev/null; then
+        # only run if there is atleast 1 searchnode
+        upgrade_searchnode_elasticsearch
+      fi
+    fi
+    set -e
+    
     highstate
     check_saltmaster_status
     postupgrade_changes
     [[ $is_airgap -eq 0 ]] && unmount_update
-
+    
+    if [[ "$es_version" != "$target_es_version" ]]; then
+      # Run final elasticsearch / fleet state on manager to ensure addon index templates are created/regenerated and loaded
+      echo "Running final Elastic states at $(date +"%T.%6N"), after upgrade to $NEWVERSION"
+      salt-call state.apply elasticsearch,elasticfleet queue=True
+    fi
+    
     echo ""
     echo "Upgrade to $NEWVERSION complete."
 

salt/nginx/etc/nginx.conf

@@ -260,7 +260,7 @@ http {
 		}
 
 {%  if 'api' in salt['pillar.get']('features', []) %}
-		location ~* (^/oauth2/token.*|^.well-known/jwks.json|^.well-known/openid-configuration) {
+		location ~* (^/oauth2/token.*|^/\.well-known/jwks.json|^/\.well-known/openid-configuration) {
 			limit_req             zone=auth_throttle burst={{ NGINXMERGED.config.throttle_login_burst }} nodelay;
 			limit_req_status      429;
 			proxy_pass            http://{{ GLOBALS.manager }}:4444;

salt/soc/defaults.yaml

@@ -1549,6 +1549,8 @@ soc:
           memoryModel: gemma@SOAI
           embedModel: amazon.titan-embed-text-v2@SOAI
           reconcileModel: gemma@SOAI
+          memoryPersona: ""
+          reconcilePersona: ""
         onionconfig:
           saltstackDir: /opt/so/saltstack
           bypassEnabled: false

salt/soc/soc_soc.yaml

@@ -884,6 +884,16 @@ soc:
           reconcileModel:
             description: The model to use when reconciling memories that contain nearly the same content.
             global: True
+          memoryPersona:
+            description: Text appended to the built-in prompt of the memory extraction agent, managed from the Agent Studio. Use it to steer what is worth remembering.
+            global: True
+            readonlyUi: True
+            multiline: True
+          reconcilePersona:
+            description: Text appended to the built-in prompt of the memory reconciliation agent, managed from the Agent Studio. Use it to steer how new memories are merged with existing ones.
+            global: True
+            readonlyUi: True
+            multiline: True
       client:
         assistant:
           enabled:

salt/stig/enabled.sls

@@ -65,15 +65,6 @@ run_remediate:
     - success_retcodes:
       - 2
 
-# Elastic Agent docker image chowns its directory to the running UID but does not
-#  chown the elastic-agent launcher symlink. fs.protected_symlinks=1 then prevents
-#  non-root users from following that launcher symlink.
-{# OSCAP rule id: xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks #}
-fs.protected_symlinks:
-  sysctl.present:
-    - value: 0
-    - config: /etc/sysctl.conf
-
 {# OSCAP rule id: xccdf_org.ssgproject.content_rule_disable_ctrlaltdel_burstaction #}
 disable_ctrl_alt_del_action:
   file.replace:

salt/stig/files/sos-oscap.xml

@@ -1601,7 +1601,7 @@ DISA STIG for Oracle Linux 9 V1R3.</xccdf-1.2:description>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sudoers_validate_passwd" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_crypto_fips_enabled" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_hardlinks" selected="true"/>
-        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="false"/>
+        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_core_pattern" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_dmesg_restrict" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_exec_shield" selected="true"/>
@@ -2202,7 +2202,7 @@ standard DISA STIG for Oracle Linux 9 profile.</xccdf-1.2:description>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sudoers_validate_passwd" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_crypto_fips_enabled" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_hardlinks" selected="true"/>
-        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="false"/>
+        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_core_pattern" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_dmesg_restrict" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_exec_shield" selected="true"/>

salt/telegraf/etc/telegraf.conf

@@ -335,7 +335,7 @@
 {%-   do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('sostatus.sh') %}
 [[inputs.exec]]
    commands = [
-      "/scripts/sostatus.sh"
+      ["/scripts/sostatus.sh"]
    ]
    data_format = "influx"
    timeout = "15s"
@@ -346,7 +346,7 @@
 [[inputs.exec]]
    commands = [
 {%-   for script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %}
-      "/scripts/{{script}}"{% if not loop.last %},{% endif %}
+      ["/scripts/{{script}}"]{% if not loop.last %},{% endif %}
 {%-   endfor %}
    ]
   data_format = "influx"
@@ -375,7 +375,7 @@
 {%- if GLOBALS.is_manager or GLOBALS.role == 'so-heavynode' %}
 [[ inputs.exec ]]
   commands = [
-    "/scripts/esindexsize.sh"
+    ["/scripts/esindexsize.sh"]
   ]
   data_format = "influx"
   interval = "1h"