Merge pull request #16210 from Security-Onion-Solutions/soup-fixes

Fix soup

salt/elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch

@@ -19,6 +19,8 @@ INDEX_PATTERNS=(
   '.entities.v2.latest.security_*'
   '.entities.v2.history.security_*'
   'risk-score.risk-score-latest-*'
+  '.metrics-endpoint.metadata_united_*'
+  '.metrics-endpoint.metadata_current_*'
 )
 DATA_STREAM_PATTERNS=(
   '.entities.v2.updates.security_*'
@@ -33,6 +35,8 @@ TEMPLATE_PATTERNS=(
   '.risk-score.risk-score-default-index-template'
   '.rule-events'
   '.alert-actions'
+  '.metrics-endpoint.metadata_united_default-template'
+  '.metrics-endpoint.metadata_current_default-template'
 )
 
 query_es() {

salt/manager/tools/sbin/soup

@@ -1020,9 +1020,6 @@ update_kafka_metadata() {
 up_to_3.2.0() {
   ensure_salt_local_pillar
 
-  # download 9.3.7 elastic agent packages
-  determine_elastic_agent_upgrade
-
   fix_logstash_0013_lumberjack_pipeline_name
 
   pin_elasticsearch_data_retention_method
@@ -1041,10 +1038,6 @@ post_to_3.2.0() {
   # SOC database bootstrap is handled by the postgres.enabled highstate.
   scrub_postgres_log_passwords
 
-  # Generate 9.3.7 elastic agent installers
-  echo "Regenerating Elastic Agent Installers"
-  /sbin/so-elastic-agent-gen-installers
-
   kibana_backport_streams_index_template
 
   update_kafka_metadata "4.3"
@@ -1053,8 +1046,11 @@ post_to_3.2.0() {
 }
 ### 3.2.0 End ###
 
-### 3.2.0 Scripts ###
+### 3.3.0 Scripts ###
 up_to_3.3.0() {
+  # download 9.4.5 elastic agent packages
+  determine_elastic_agent_upgrade
+
   INSTALLEDVERSION=3.3.0
 }
 
@@ -1081,7 +1077,13 @@ post_to_3.3.0() {
   # Recollate again since some internal DBs were excluded during 3.2.0 soup
   recollate_postgres
 
+  # Generate 9.4.5 elastic agent installers
+  echo "Regenerating Elastic Agent Installers"
+  /sbin/so-elastic-agent-gen-installers
+
   telegraf_repair
+
+  set_postversion 3.3.0
 }
 ### 3.3.0 End ###
 

salt/soc/defaults.yaml

@@ -1547,6 +1547,8 @@ soc:
           maxGlobalMemoriesToInclude: 5
           maxUserMemoriesToReconcile: 20
           maxGlobalMemoriesToReconcile: 20
+          memoryExtractBatchSize: 5
+          maxMemoryRetries: 2
           memoryModel: gemma@SOAI
           embedModel: amazon.titan-embed-text-v2@SOAI
           reconcileModel: gemma@SOAI

salt/soc/soc_soc.yaml

@@ -908,6 +908,14 @@ soc:
             description: Milliseconds to wait between attempts to look up a tool request that has finished streaming but has not yet been saved to the database. The API retries up to toolUseTurnAttempts times, so this value times the attempt count is the maximum wait before auto-approval fails. Increase it if tool auto-approval fails intermittently.
             global: True
             advanced: True
+          memoryExtractBatchSize:
+            description: Max number of a session's messages that will be processed for memory extractions at a time. Larger values are more efficient but risk exceeding the memory extraction LLM's size limits, due to the variable and unpredictable size of each message.
+            global: True
+            advanced: True
+          maxMemoryRetries:
+            description: The number of times to retry extracting memories from a session if errors occur.
+            global: True
+            advanced: True
       client:
         assistant:
           enabled: