Specify Default Embed Model

Added `memoryScanIntervalSeconds` with a default of 5 mins.

Opted to set `useMemoryScanner` to false so by default our user's sessions are not sent to the cloud before they have a chance to configure the new setting.

VERSION

@@ -1 +1 @@
-3.0.0-foxtrot
+3.3.0

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -5,7 +5,7 @@
   "package": {
     "name": "endpoint",
     "title": "Elastic Defend",
-    "version": "9.4.1",
+    "version": "9.3.1",
     "requires_root": true
   },
   "enabled": true,

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -29,7 +29,7 @@
               "\\.gz$"
             ],
             "include_files": [],
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.22.3\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.9.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.22.3\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.22.3\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.9.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.20.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.3\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.20.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.20.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.3\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
             ],

salt/elasticfleet/integration-defaults.map.jinja

@@ -16,6 +16,7 @@
     'awsfirehose.metrics': 'aws.cloudwatch',
     'cribl.logs': 'cribl',
     'cribl.metrics': 'cribl',
+    'sentinel_one_cloud_funnel.logins': 'sentinel_one_cloud_funnel.login',
     'azure_application_insights.app_insights': 'azure.app_insights',
     'azure_application_insights.app_state': 'azure.app_state',
     'azure_billing.billing': 'azure.billing',

salt/elasticsearch/cluster.sls

@@ -98,13 +98,6 @@ so-es-cluster-settings:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
       - http: wait_for_so-elasticsearch
-
-so-elasticsearch-system-indices-patch:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-system-indices-patch
-    - require:
-      - http: wait_for_so-elasticsearch
-      - file: so-elasticsearch-system-indices-patch-script
 {%  endif %}
 
 # heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)

salt/elasticsearch/config.sls

@@ -42,16 +42,6 @@ elasticsearch_sbin:
     - file_mode: 755
     - exclude_pat:
       - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state
-      - so-elasticsearch-system-indices-patch
-    - show_changes: False
-
-so-elasticsearch-system-indices-patch-script:
-  file.managed:
-    - name: /usr/sbin/so-elasticsearch-system-indices-patch
-    - source: salt://elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch
-    - user: 930
-    - group: 939
-    - mode: 755
     - show_changes: False
 
 elasticsearch_sbin_jinja:

salt/elasticsearch/defaults.yaml

@@ -1,7 +1,7 @@
 elasticsearch:
   enabled: false
   esheap: '600m'
-  version: 9.4.5
+  version: 9.3.7
   index_clean: true
   data_retention_method: DLM
   vm:

salt/elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch

@@ -1,199 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-set -eo pipefail
-
-SETTINGS='{"index":{"auto_expand_replicas":"0-1"}}'
-KIBANA_PASSWORD=
-INDEX_PATTERNS=(
-  '.entity_analytics.risk_score.lookup-*'
-  '.entity_analytics.watchlists.*'
-  '.entity_analytics.monitoring.users-*'
-  '.entity_analytics.entity-leads-*'
-  '.asset-criticality.asset-criticality-*'
-  '.workflows-executions'
-  '.workflows-step-executions'
-  '.entities.v2.latest.security_*'
-  '.entities.v2.history.security_*'
-  'risk-score.risk-score-latest-*'
-)
-DATA_STREAM_PATTERNS=(
-  '.entities.v2.updates.security_*'
-  'risk-score.risk-score-*'
-  '.rule-events'
-  '.alert-actions'
-)
-TEMPLATE_PATTERNS=(
-  'entities_v2_latest_security_default_index_template'
-  'entities_v2_history_security_default_index_template'
-  '.entities_v2_updates_security_default_index_template'
-  '.risk-score.risk-score-default-index-template'
-  '.rule-events'
-  '.alert-actions'
-)
-
-query_es() {
-  if so-elasticsearch-query "$@" --fail --retry 3 --retry-delay 5; then
-    return 0
-  fi
-
-  # retry failed attempts with so_kibana user (system managed indices reject so_elastic user)
-  local query_path="$1"
-  shift
-
-  if [[ -z "$KIBANA_PASSWORD" ]]; then
-    KIBANA_PASSWORD=$(salt-call pillar.get elasticsearch:auth:users:so_kibana_user:pass --out=newline_values_only)
-  fi
-  [[ -n "$KIBANA_PASSWORD" ]] || return 1
-
-  echo "Retrying ${query_path} as so_kibana." >&2
-  curl -K /opt/so/conf/elasticsearch/curl.config --user "so_kibana:${KIBANA_PASSWORD}" \
-    -s -k -L --fail --retry 3 --retry-delay 5 -H 'Content-Type: application/json' "https://localhost:9200/${query_path}" "$@"
-}
-
-# add auto_expand_replicas=0-1 to given index
-set_auto_expand_replicas() {
-  local index="$1"
-
-  echo "Setting auto_expand_replicas to 0-1 on ${index}."
-  query_es "${index}/_settings" -XPUT -d "$SETTINGS" >/dev/null
-}
-
-# resolve index patterns and find each index with an unassigned replica
-unassigned_replicas() {
-  local pattern="$1"
-  local resolved_indices response index
-
-  if ! resolved_indices=$(query_es "_resolve/index/${pattern}?expand_wildcards=all" 2>/dev/null); then
-    return 0
-  fi
-
-  while read -r index; do
-    if ! response=$(query_es "_cat/shards/${index}?format=json&h=index,prirep,state" 2>/dev/null); then
-      continue
-    fi
-    jq -r '.[]? | objects | select(.prirep == "r" and .state == "UNASSIGNED") | .index' <<<"$response"
-  done < <(jq -r '.indices[]?.name' <<<"$resolved_indices")
-}
-
-data_stream_indices() {
-  local pattern="$1"
-  local response
-
-  if ! response=$(query_es "_data_stream/${pattern}?expand_wildcards=all" 2>/dev/null); then
-    return 0
-  fi
-  jq -r '.data_streams[]?.indices[]?.index_name' <<<"$response"
-}
-
-update_system_indices() {
-  local pattern="$1"
-  local index
-
-  while read -r index; do
-    [[ -n "$index" ]] && set_auto_expand_replicas "$index"
-  done < <(unassigned_replicas "$pattern")
-}
-
-# update data stream backing indices with unassigned replicas
-update_system_ds() {
-  local pattern="$1"
-  local index
-
-  while read -r index; do
-    while read -r unassigned_index; do
-      [[ -n "$unassigned_index" ]] && set_auto_expand_replicas "$unassigned_index"
-    done < <(unassigned_replicas "$index")
-  done < <(data_stream_indices "$pattern")
-}
-
-has_unassigned_replicas() {
-  local pattern="$1"
-  local index
-
-  index=$(unassigned_replicas "$pattern" | sed -n '1p')
-  [[ -n "$index" ]]
-}
-
-data_stream_has_unassigned_replicas() {
-  local pattern="$1"
-  local index
-  while read -r index; do
-    has_unassigned_replicas "$index" && return 0
-  done < <(data_stream_indices "$pattern")
-
-  return 1
-}
-
-needs_patch() {
-  local pattern
-  for pattern in "${INDEX_PATTERNS[@]}"; do
-    has_unassigned_replicas "$pattern" && return 0
-  done
-  for pattern in "${DATA_STREAM_PATTERNS[@]}"; do
-    data_stream_has_unassigned_replicas "$pattern" && return 0
-  done
-
-  return 1
-}
-
-# get index templates, update with auto_expand_replicas=0-1, and PUT back. Keeping mappings/settings/aliases in-place
-update_system_templates() {
-  local pattern="$1"
-  local templates name response template auto_expand_replicas
-
-  if ! templates=$(query_es "_index_template/${pattern}" 2>/dev/null); then
-    return 0
-  fi
-  while read -r name; do
-    response=$(query_es "_index_template/${name}")
-    template=$(jq -c '.index_templates[0].index_template' <<<"$response")
-    auto_expand_replicas=$(jq -r '.template.settings["index.auto_expand_replicas"] // .template.settings.index.auto_expand_replicas // empty' <<<"$template")
-    [[ "$auto_expand_replicas" == "0-1" ]] && continue
-
-    template=$(jq '
-            if (.template.settings.index | type) == "object" then
-                .template.settings.index.auto_expand_replicas = "0-1"
-            else
-                .template.settings["index.auto_expand_replicas"] = "0-1"
-            end
-            | del(.created_date_millis, .modified_date_millis)
-        ' <<<"$template")
-    echo "Setting auto_expand_replicas to 0-1 on index template ${name}."
-    query_es "_index_template/${name}" -XPUT -d "$template" >/dev/null
-  done < <(jq -r '.index_templates[]?.name' <<<"$templates")
-}
-
-if [[ "${1:-}" == "--check" ]]; then
-  needs_patch
-  exit $?
-fi
-
-if [[ $# -ne 0 ]]; then
-  echo "Usage: $0 [--check]" >&2
-  exit 1
-fi
-
-patched=false
-for pattern in "${INDEX_PATTERNS[@]}"; do
-  if has_unassigned_replicas "$pattern"; then
-    update_system_indices "$pattern"
-    patched=true
-  fi
-done
-
-for pattern in "${DATA_STREAM_PATTERNS[@]}"; do
-  if data_stream_has_unassigned_replicas "$pattern"; then
-    update_system_ds "$pattern"
-    patched=true
-  fi
-done
-
-if [[ "$patched" == true ]]; then
-  for pattern in "${TEMPLATE_PATTERNS[@]}"; do
-    update_system_templates "$pattern"
-  done
-fi

salt/kibana/config.sls

@@ -63,12 +63,6 @@ kibanaconfig:
         KIBANACONFIG: {{ KIBANAMERGED.config }}
     - show_changes: False
 
-so-kibana-es-upgrade-ready:
-  file.managed:
-    - name: /usr/sbin/so-kibana-es-upgrade-ready
-    - source: salt://kibana/files/so-kibana-es-upgrade-ready
-    - mode: '0755'
-
 kibanalogdir:
   file.directory:
     - name: /opt/so/log/kibana

salt/kibana/defaults.yaml

@@ -22,7 +22,7 @@ kibana:
           - default
           - file
     migrations:
-      discardCorruptObjects: "9.4.5"
+      discardCorruptObjects: "9.3.7"
     telemetry:
       enabled: False
     xpack:

salt/kibana/enabled.sls

@@ -8,7 +8,6 @@
 {%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   set ELASTICSEARCH_UPGRADED = salt['pillar.get']('elasticsearch:upgraded', True) %}
 
 include:
   - kibana.config
@@ -62,11 +61,6 @@ so-kibana:
     {% endif %}
     - watch:
       - file: kibanaconfig
-    {% if not ELASTICSEARCH_UPGRADED %}
-    - onlyif: /usr/sbin/so-kibana-es-upgrade-ready {{ ELASTICSEARCHMERGED.version }}
-    {% endif %}
-    - require:
-      - file: so-kibana-es-upgrade-ready
 
 wait_for_so-kibana:
   http.wait_for_successful_query:
@@ -78,9 +72,6 @@ wait_for_so-kibana:
     - status: 200
     - wait_for: 600
     - request_interval: 15
-    {% if not ELASTICSEARCH_UPGRADED %}
-    - onlyif: /usr/sbin/so-kibana-es-upgrade-ready {{ ELASTICSEARCHMERGED.version }}
-    {% endif %}
     - require:
       - docker_container: so-kibana
 

salt/kibana/files/so-kibana-es-upgrade-ready

@@ -1,35 +0,0 @@
-#!/bin/bash
-#
-# Start Kibana only when all active Elasticsearch nodes match its configured
-# Elasticsearch version.
-
-PILLAR_FILE=/opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls
-
-set -o pipefail
-
-EXPECTED_VERSION=${1:-}
-if [[ ! "$EXPECTED_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-  echo "Usage: so-kibana-es-upgrade-ready <Elasticsearch version>" >&2
-  exit 1
-fi
-
-# Require every node active in Kibana's Elasticsearch cluster to be at the
-# target version. This matches Kibana's own nodes.info compatibility check.
-if ! cluster_versions=$(timeout 90 so-elasticsearch-query _nodes/_all/version --fail 2>/dev/null); then
-  echo "Kibana is held: unable to query Elasticsearch cluster node versions." >&2
-  exit 1
-fi
-
-if ! jq -e --arg expected "$EXPECTED_VERSION" \
-    '(.nodes | length) > 0 and all(.nodes[]; .version == $expected)' \
-    <<< "$cluster_versions" > /dev/null; then
-  echo "Kibana is held: one or more Elasticsearch cluster nodes are not running $EXPECTED_VERSION." >&2
-  exit 1
-fi
-
-if ! /usr/sbin/so-yaml.py replace "$PILLAR_FILE" elasticsearch.upgraded true; then
-  echo "Kibana is held: unable to mark the Elasticsearch upgrade complete." >&2
-  exit 1
-fi
-
-echo "All active Elasticsearch nodes are running $EXPECTED_VERSION; releasing Kibana."

salt/kibana/tools/sbin_jinja/so-kibana-space-defaults

@@ -9,5 +9,5 @@ SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http:
 # Disable certain Features from showing up in the Kibana UI
 echo
 echo "Setting up default Kibana Space:"
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","securitySolutionCasesV3","inventory","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV4","securitySolutionAlertsV1","entityManager","slo","streams","anonymization","searchInferenceEndpoints","cloudConnect","queryActivity","automatic_import","stackAlerts","monitoring","dataQuality","actions"]} ' >> /opt/so/log/kibana/misc.log
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV1","entityManager","streams","cloudConnect","slo"]} ' >> /opt/so/log/kibana/misc.log
 echo

salt/manager/tools/sbin/soup

@@ -1359,12 +1359,11 @@ verify_es_version_compatibility() {
     local is_active_intermediate_upgrade=1
     # supported upgrade paths for SO-ES versions
     declare -A es_upgrade_map=(
-      ["8.18.4"]="8.18.6 8.18.8 9.0.8"
+        ["8.18.4"]="8.18.6 8.18.8 9.0.8"
 	    ["8.18.6"]="8.18.8 9.0.8"
 	    ["8.18.8"]="9.0.8"
-      ["9.0.8"]="9.3.3 9.3.7 9.4.5"
+        ["9.0.8"]="9.3.3 9.3.7"
-      ["9.3.3"]="9.3.7 9.4.5"
+        ["9.3.3"]="9.3.7"
-      ["9.3.7"]="9.4.5"
     )
 
     # Elasticsearch MUST upgrade through these versions
@@ -1466,7 +1465,7 @@ verify_es_version_compatibility() {
             done <<< "$expected_es_nodes"
 
             if [[ "$all_searchnodes_compatible" == true ]]; then
-                echo -e "\t- All Searchnodes are upgradable to Elasticsearch $target_es_version."
+                echo "All Searchnodes are upgradable to Elasticsearch $target_es_version."
                 return 0
             fi
 
@@ -1536,11 +1535,11 @@ verify_es_version_compatibility() {
             done <<< "$heavynode_minions"
 
             if [[ "$all_heavynodes_compatible" == true ]]; then
-                echo -e "\t- All heavynodes are upgradable to Elasticsearch $target_es_version."
+                echo -e "\nAll heavynodes can upgrade to Elasticsearch $target_es_version."
                 return 0
             fi
 
-            echo -e "\t- One or more heavynodes cannot upgrade directly to Elasticsearch $target_es_version. Rechecking in $delay seconds. Attempt $((retry_count + 1)) of $retries."
+            echo "One or more heavynodes cannot upgrade directly to Elasticsearch $target_es_version. Rechecking in $delay seconds. Attempt $((retry_count + 1)) of $retries."
             ((retry_count++))
             sleep $delay
         done
@@ -1976,7 +1975,7 @@ main() {
   upgrade_check
   upgrade_space
 
-  echo -e "\nVerifying Elasticsearch version compatibility across the grid before upgrading."
+  echo "Verifying Elasticsearch version compatibility across the grid before upgrading."
   verify_es_version_compatibility
 
   # Pre-flight health checks: confirm the grid is in a good state before we change
@@ -2080,12 +2079,6 @@ main() {
     copy_new_files
     echo ""
     create_local_directories "/opt/so/saltstack/default"
-    if [[ "$es_version" != "$target_es_version" ]]; then
-      elasticsearch_pillar_file=/opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls
-      echo "Elasticsearch upgrade scheduled. Kibana will be held at $es_version until cluster has upgraded to $target_es_version."
-      so-yaml.py replace "$elasticsearch_pillar_file" elasticsearch.upgraded false \
-        || fail "Unable to mark the Elasticsearch upgrade as incomplete."
-    fi
     # Seed the resume marker before the highstate stamps /etc/soversion to the new
     # version, so an interrupted upgrade is detectable as "not finished" on re-run.
     # POSTVERSION still holds the pre-upgrade (or prior resume) version here.

salt/salt/minion/boot_highstate.sls

@@ -3,8 +3,6 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-
 # Manages /etc/systemd/system/so-boot-highstate.service, a Type=oneshot
 # RemainAfterExit=yes unit that runs `salt-call state.highstate` exactly once
 # per system boot. Replaces the legacy `startup_states: highstate` minion
@@ -21,25 +19,9 @@ so_boot_highstate_unit_file:
     - onchanges_in:
       - module: systemd_reload
 
-# Non-managers never apply salt.minion during setup, so reaching this state means
-# setup is finished and the marker is safe to write unconditionally. This also
-# heals nodes installed before this fix, which have no marker and no legacy
-# startup_states line to grep for. Managers do highstate mid-setup, so they only
-# get the marker from the legacy upgrade signal; fresh installs get it from
-# mark_setup_complete in setup/so-functions.
-mark_setup_complete:
-  file.managed:
-    - name: /opt/so/state/setup-complete
-    - replace: false
-    - makedirs: True
-{% if GLOBALS.is_manager %}
-    - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion"
-{% endif %}
-    - require_in:
-      - service: so_boot_highstate_service
-
 # Only enable once setup is complete. Until then the gate file is missing and
-# the unit's own ConditionPathExists would no-op it anyway.
+# the unit's own ConditionPathExists would no-op it anyway -- this just keeps
+# `systemctl is-enabled` honest for the sync_es_users gate.
 so_boot_highstate_service:
   service.enabled:
     - name: so-boot-highstate.service

salt/salt/minion/init.sls

@@ -87,15 +87,27 @@ set_log_levels:
 # so-boot-highstate.service (managed in salt.minion.boot_highstate), which
 # runs once per system boot only. Strip the line from /etc/salt/minion on
 # upgrade; both the commented and uncommented forms historically existed.
-# Ordered after mark_setup_complete (salt.minion.boot_highstate); the manager
-# gate there greps for this line, so it must run before we delete it.
 remove_startup_states:
   file.line:
     - name: /etc/salt/minion
     - match: 'startup_states: highstate'
     - mode: delete
-    - require:
+
-      - file: mark_setup_complete
+# Upgrade-path bridge: systems that already passed setup under the old gate
+# (`grep -x 'startup_states: highstate' /etc/salt/minion`) get a /opt/so/state/setup-complete
+# marker so so-boot-highstate.service can be enabled and the so-user_sync cron
+# in sync_es_users.sls keeps installing. Setup-in-progress systems instead get
+# the marker from `mark_setup_complete` in setup/so-functions at the right
+# moment. `replace: false` means we never overwrite a marker once written.
+mark_setup_complete_for_upgrades:
+  file.managed:
+    - name: /opt/so/state/setup-complete
+    - replace: false
+    - makedirs: True
+    - onlyif: "grep -qx 'startup_states: highstate' /etc/salt/minion"
+    - require_in:
+      - file: remove_startup_states
+      - service: so_boot_highstate_service
 
 {% endif %}
 

salt/setup/virt/setSalt.sls

@@ -8,15 +8,6 @@ set_role_grain:
     - name: role
     - value: so-{{ grains.id.split("_") | last }}
 
-# salt-cloud guests never run so-setup, so nothing else marks them setup-complete.
-# Replaces the 'startup_states: highstate' line this state used to append. No
-# GLOBALS import -- this runs before the guest's pillars exist.
-mark_setup_complete_vm_guest:
-  file.managed:
-    - name: /opt/so/state/setup-complete
-    - replace: false
-    - makedirs: True
-
 enable_salt_minion:
   service.enabled:
     - name: salt-minion

salt/soc/defaults.yaml

@@ -1537,6 +1537,18 @@ soc:
             Orchestrator: sonnet@SOAI
             Investigator: gemma@SOAI
             DetectionEngineer: gemma@SOAI
+          useMemory: true
+          useMemoryScanner: false
+          memoryScanIntervalSeconds: 300
+          memoryProximityThreshold: 0.8
+          messageProximityThreshold: 0.5
+          maxUserMemoriesToInclude: 5
+          maxGlobalMemoriesToInclude: 5
+          maxUserMemoriesToReconcile: 20
+          maxGlobalMemoriesToReconcile: 20
+          memoryModel: gemma@SOAI
+          embedModel: amazon.titan-embed-text-v2@SOAI
+          reconcileModel: gemma@SOAI
         onionconfig:
           saltstackDir: /opt/so/saltstack
           bypassEnabled: false
@@ -2671,7 +2683,7 @@ soc:
               # The id (UUIDv4) is pregenerated and can safely be used.
               # Click "Convert" to convert the Sigma rule to use Security Onion field mappings within an EQL query
               #
-              # Rule Creation Guide: https://github.com/SigmaHQ/sigma/wiki/Rule-Creation-High%E2%80%90Level-Guide
+              # Rule Creation Guide: https://github.com/SigmaHQ/sigma/wiki/Rule-Creation-Guide
               # Logsources: https://sigmahq.io/docs/basics/log-sources.html
 
               title: 'A Short Capitalized Title With Less Than 50 Characters'
@@ -2683,7 +2695,7 @@ soc:
               references:
                 - 'https://local.invalid'
               author: '@SecurityOnion'
-              date: '[today]'
+              date: 'YYYY/MM/DD'
               tags:
                 - detection.threat_hunting
                 - attack.technique_id
@@ -2727,5 +2739,14 @@ soc:
               enabled: true
               adapter: SOAI
               charsPerTokenEstimate: 4
+            - id: amazon.titan-embed-text-v2
+              displayName: amazon.titan-embed-text-v2
+              origin: USA
+              contextLimitSmall: 8192
+              contextLimitLarge: 8192
+              lowBalanceColorAlert: 500000
+              enabled: true
+              adapter: SOAI
+              charsPerTokenEstimate: 4
               
             

salt/soc/soc_soc.yaml

@@ -845,6 +845,42 @@ soc:
             DetectionEngineer:
               description: This agent manages detections and their overrides, including tuning noisy rules and authoring rule content.
               global: True
+          useMemory:
+            description: Enables the Memory system for OnionAI
+            global: True
+          useMemoryScanner:
+            description: Enables the memory scanner for automatic memory extraction from historical sessions.
+            global: True
+          memoryScanIntervalSeconds:
+            description: How long to wait in seconds between attempts to scan sessions for new memories.
+            global: True
+          memoryProximityThreshold:
+            description: Describes how close memories need to be on a floating point scale from 0.0 to 1.0 to be considered when reconciling new memories with old ones. This value is usually higher than messageProximityThreshold.
+            global: True
+          messageProximityThreshold:
+            description: Describes how close a memory needs to be to a user's message on a floating point scale from 0.0 to 1.0 to be included in the context. This value is usually lower than memoryProximityThreshold.
+            global: True
+          maxUserMemoriesToInclude:
+            description: Specify the max number of user-specific memories to include in the prompt when a user sends a message.
+            global: True
+          maxGlobalMemoriesToInclude:
+            description: Specify the max number of global memories to include in the prompt when a user sends a message.
+            global: True
+          maxUserMemoriesToReconcile:
+            description: When reconciling new user-specific memories with existing user-specific memories, this determines how many old memories may be considered.
+            global: True
+          maxGlobalMemoriesToReconcile:
+            description: When reconciling new global memories with existing global memories, this determines how many old memories may be considered.
+            global: True
+          memoryModel:
+            description: The model to use when extracting memories from sessions.
+            global: True
+          embedModel: 
+            description: The model to use when embedding a memory as a vector. Note that only memories embedded using the same model may be compared and only memories created with the model specified here will be considered when informing an agent of existing memories.
+            global: True
+          reconcileModel:
+            description: The model to use when reconciling memories that contain nearly the same content.
+            global: True
       client:
         assistant:
           enabled:

setup/so-setup

@@ -833,7 +833,6 @@ if ! [[ -f $install_opt_file ]]; then
 		check_sos_appliance
 		drop_install_options
 		hypervisor_local_states
-		mark_setup_complete
 		verify_setup
 	fi