Memory Defaults and Annotations

patch issue with fs.protected_symlinks

salt/common/tools/sbin/so-log-check

@@ -154,6 +154,8 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|id.orig_h"                    # false positive (zeek test data)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|emerging-all.rules"           # false positive (error in rulename)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|invalid query input"          # false positive (Invalid user input in hunt query)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no data available for the requested dates" # false positive (pcap cypress test submits a job with an empty time frame)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no job processor"             # false positive (same empty-time-frame job on import nodes, where no pcap processor runs)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|example"                      # false positive (example test data)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200"                   # false positive (request successful, contained error string in content)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|app_layer.error"              # false positive (suricata 7) in stats.log e.g. app_layer.error.imap.parser | Total | 0

salt/manager/tools/sbin/soup

@@ -344,6 +344,16 @@ check_cluster_health() {
 
 check_fleet_server() {
   echo "Checking that Elastic Fleet Server is responding."
+  # Before checking fleet health, check for and fix known issue with elastic-agent container and fs.protected_symlinks
+  local protected_symlinks=$(sysctl -b fs.protected_symlinks)
+  if [[ "$protected_symlinks" == "1" ]]; then
+    # disable fs.protected_symlinks and restart elasticfleet
+    sysctl -w fs.protected_symlinks=0
+    docker stop so-elastic-fleet; docker rm -f so-elastic-fleet
+    printf "\nUpdated sysctl fs.protected_symlinks. Restarting fleet before running health check and continuing with soup.\n"
+    salt-call state.apply elasticfleet queue=True
+  fi
+  
   # Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls,
   # which waits for HTTP 200 from the Fleet Server status API.
   if curl -sk --fail --retry 3 --retry-delay 10 --max-time 30 "https://localhost:8220/api/status" > /dev/null 2>&1; then
@@ -1036,8 +1046,20 @@ post_to_3.2.0() {
 }
 ### 3.2.0 End ###
 
-### 3.2.0 Scripts ###
+### 3.3.0 Scripts ###
+
+ # Sets fs.protected_symlinks=0
+ #
+ # Elastic Agent docker image chowns its directory to the running UID 
+ # but does not chown the elastic-agent launcher symlink. 
+ # Preventing non-root users from following that launcher symlink.
+disable_sysctl_fs_protected_symlink() {
+  salt -C 'I@stig:enabled' state.single sysctl.present name=fs.protected_symlinks value=0 config=/etc/sysctl.conf || true
+}
+    
 up_to_3.3.0() {
+  disable_sysctl_fs_protected_symlink
+  
   INSTALLEDVERSION=3.3.0
 }
 

salt/soc/defaults.yaml

@@ -1537,6 +1537,17 @@ soc:
             Orchestrator: sonnet@SOAI
             Investigator: gemma@SOAI
             DetectionEngineer: gemma@SOAI
+          useMemory: true
+          useMemoryScanner: true
+          memoryProximityThreshold: 0.8
+          messageProximityThreshold: 0.5
+          maxUserMemoriesToInclude: 5
+          maxGlobalMemoriesToInclude: 5
+          maxUserMemoriesToReconcile: 20
+          maxGlobalMemoriesToReconcile: 20
+          memoryModel: gemma@SOAI
+          embedModel: gemma@SOAI
+          reconcileModel: gemma@SOAI
         onionconfig:
           saltstackDir: /opt/so/saltstack
           bypassEnabled: false

salt/soc/soc_soc.yaml

@@ -845,6 +845,39 @@ soc:
             DetectionEngineer:
               description: This agent manages detections and their overrides, including tuning noisy rules and authoring rule content.
               global: True
+          useMemory:
+            description: Enables the Memory system for OnionAI
+            global: True
+          useMemoryScanner:
+            description: Enables the memory scanner for automatic memory extraction from historical sessions.
+            global: True
+          memoryProximityThreshold:
+            description: Describes how close memories need to be on a floating point scale from 0.0 to 1.0 to be considered when reconciling new memories with old ones. This value is usually higher than messageProximityThreshold.
+            global: True
+          messageProximityThreshold:
+            description: Describes how close a memory needs to be to a user's message on a floating point scale from 0.0 to 1.0 to be included in the context. This value is usually lower than memoryProximityThreshold.
+            global: True
+          maxUserMemoriesToInclude:
+            description: Specify the max number of user-specific memories to include in the prompt when a user sends a message.
+            global: True
+          maxGlobalMemoriesToInclude:
+            description: Specify the max number of global memories to include in the prompt when a user sends a message.
+            global: True
+          maxUserMemoriesToReconcile:
+            description: When reconciling new user-specific memories with existing user-specific memories, this determines how many old memories may be considered.
+            global: True
+          maxGlobalMemoriesToReconcile:
+            description: When reconciling new global memories with existing global memories, this determines how many old memories may be considered.
+            global: True
+          memoryModel:
+            description: The model to use when extracting memories from sessions.
+            global: True
+          embedModel: 
+            description: The model to use when embedding a memory as a vector. Note that only memories embedded using the same model may be compared and only memories created with the model specified here will be considered when informing an agent of existing memories.
+            global: True
+          reconcileModel:
+            description: The model to use when reconciling memories that contain nearly the same content.
+            global: True
       client:
         assistant:
           enabled:

salt/stig/enabled.sls

@@ -65,6 +65,15 @@ run_remediate:
     - success_retcodes:
       - 2
 
+# Elastic Agent docker image chowns its directory to the running UID but does not
+#  chown the elastic-agent launcher symlink. fs.protected_symlinks=1 then prevents
+#  non-root users from following that launcher symlink.
+{# OSCAP rule id: xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks #}
+fs.protected_symlinks:
+  sysctl.present:
+    - value: 0
+    - config: /etc/sysctl.conf
+
 {# OSCAP rule id: xccdf_org.ssgproject.content_rule_disable_ctrlaltdel_burstaction #}
 disable_ctrl_alt_del_action:
   file.replace:

salt/stig/files/sos-oscap.xml

@@ -1601,7 +1601,7 @@ DISA STIG for Oracle Linux 9 V1R3.</xccdf-1.2:description>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sudoers_validate_passwd" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_crypto_fips_enabled" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_hardlinks" selected="true"/>
-        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="true"/>
+        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="false"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_core_pattern" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_dmesg_restrict" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_exec_shield" selected="true"/>
@@ -2202,7 +2202,7 @@ standard DISA STIG for Oracle Linux 9 profile.</xccdf-1.2:description>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sudoers_validate_passwd" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_crypto_fips_enabled" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_hardlinks" selected="true"/>
-        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="true"/>
+        <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_fs_protected_symlinks" selected="false"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_core_pattern" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_dmesg_restrict" selected="true"/>
         <xccdf-1.2:select idref="xccdf_org.ssgproject.content_rule_sysctl_kernel_exec_shield" selected="true"/>

salt/zeek/defaults.yaml

@@ -15,7 +15,7 @@ zeek:
       MailHostUpDown: 0
       LogRotationInterval: 3600
       LogExpireInterval: 0
-      StatsLogEnable: 1
+      StatsLogEnable: 0
       StatsLogExpireInterval: 0
       StatusCmdShowAll: 0
       CrashExpireInterval: 0

salt/zeek/disabled.sls

@@ -23,6 +23,11 @@ zeekpacketlosscron:
     - identifier: zeekpacketlosscron
     - user: root
 
+zeekctlcron:
+  cron.absent:
+    - identifier: zeekctlcron
+    - user: root
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/zeek/enabled.sls

@@ -87,6 +87,21 @@ zeekpacketlosscron:
     - month: '*'
     - dayweek: '*'
 
+# LogExpireInterval, StatsLogExpireInterval and CrashExpireInterval are only acted on by
+# 'zeekctl cron', so run it on the interval upstream recommends. This also restarts any
+# node that died unexpectedly. Runs as root because the script needs the docker socket;
+# it drops to the zeek user inside the container.
+zeekctlcron:
+  cron.present:
+    - name: /usr/sbin/so-zeek-cron > /dev/null 2>&1
+    - identifier: zeekctlcron
+    - user: root
+    - minute: '*/5'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/zeek/soc_zeek.yaml

@@ -58,6 +58,86 @@ zeek:
       CompressLogs:
         description: This setting enables compression of Zeek logs. If you are seeing packet loss at the top of the hour in Zeek or PCAP you might need to disable this by seting it to 0. This will use more disk space but save IO and CPU.
         helpLink: zeek
+      LogExpireInterval:
+        description: >-
+          How long to keep rotated Zeek logs in /nsm/zeek/logs. A bare number means DAYS, so 7 means 7 days.
+          You may also give an explicit unit, such as "7 days" or "12 hr". Use 0 to keep logs forever.
+          This value must not be shorter than LogRotationInterval (3600 seconds by default), so the smallest
+          usable value is 1 hr - Zeek will fail to start if it is shorter. Expiry is applied by "zeekctl cron",
+          which runs every 5 minutes, and removes log files older than this based on their modification time.
+        regex: ^(0|[1-9][0-9]*( ?(day|hr)s?)?)$
+        regexFailureMessage: Enter 0, or a positive number optionally followed by "day" or "hr" (for example 7, "7 days", or "12 hr"). Minutes are not accepted because a log expire interval shorter than the log rotation interval prevents Zeek from starting.
+        helpLink: zeek
+        advanced: True
+      StatsLogEnable:
+        description: >-
+          Set to 1 to have "zeekctl cron" write node statistics to /nsm/zeek/logs/stats. This is
+          disabled because the CPU and memory portion depends on the "top" command, which the Zeek
+          container does not include, so every run records an error for each node instead. The
+          interface packet counters it also collects are not used anywhere in Security Onion, which
+          tracks Zeek packet loss separately through packetloss.log and Telegraf. It is read only
+          for that reason.
+        regex: ^[01]$
+        regexFailureMessage: You must enter 0 or 1.
+        helpLink: zeek
+        advanced: True
+        readonly: True
+      StatsLogExpireInterval:
+        description: >-
+          Number of days to keep entries in the Zeek stats log, or 0 to keep them forever.
+          Applied by "zeekctl cron", which runs every 5 minutes. This has no effect unless
+          StatsLogEnable is turned on, which it is not by default.
+        regex: ^[0-9]+$
+        regexFailureMessage: You must enter a whole number of days, or 0 to keep entries forever.
+        helpLink: zeek
+        advanced: True
+      CrashExpireInterval:
+        description: >-
+          Number of days to keep Zeek crash directories, or 0 to keep them forever.
+          Applied by "zeekctl cron", which runs every 5 minutes.
+        regex: ^[0-9]+$
+        regexFailureMessage: You must enter a whole number of days, or 0 to keep crash directories forever.
+        helpLink: zeek
+        advanced: True
+      MinDiskSpace:
+        description: >-
+          Percentage of free disk space below which ZeekControl reports a warning, or 0 to disable the check
+          entirely. The Zeek container does not include a mail program, so the warning is not emailed - it
+          appears in the output of "zeekctl cron" instead. This setting never deletes anything - cleanup based
+          on disk usage is handled separately by so-sensor-clean.
+        regex: ^([0-9]|[1-9][0-9]|100)$
+        regexFailureMessage: You must enter a percentage between 0 and 100.
+        helpLink: zeek
+        advanced: True
+      MailTo:
+        description: >-
+          Address that ZeekControl would send mail to, covering cron output and crash reports, and the address
+          Zeek's notice framework would use. The Zeek container does not include a mail program, and Security
+          Onion never enables the notice email action, so no mail is sent and this address is unused. It is
+          read only for that reason.
+        helpLink: zeek
+        advanced: True
+        readonly: True
+      MailConnectionSummary:
+        description: >-
+          Set to 1 to email the hourly connection summary. This only controls the emailed copy - the summary is
+          generated and archived with the other Zeek logs either way. The Zeek container does not include a mail
+          program, so no mail is sent and this setting has no effect. It is read only for that reason.
+        regex: ^[01]$
+        regexFailureMessage: You must enter 0 or 1.
+        helpLink: zeek
+        advanced: True
+        readonly: True
+      MailHostUpDown:
+        description: >-
+          Set to 1 to report when a Zeek node changes between the up and down states. The Zeek container does
+          not include a mail program, so this notification cannot be emailed. It is read only for that reason.
+          Host status detection still runs regardless of this setting - only the notification is affected.
+        regex: ^[01]$
+        regexFailureMessage: You must enter 0 or 1.
+        helpLink: zeek
+        advanced: True
+        readonly: True
   policy:
     custom:
       filters: 

salt/zeek/tools/sbin/so-zeek-cron

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+# Run zeekctl's periodic maintenance tasks. This is what actually enforces
+# LogExpireInterval, StatsLogExpireInterval and CrashExpireInterval - without a periodic
+# 'zeekctl cron' those settings are inert no matter what they are set to.
+
+# This also restarts any node that died unexpectedly, and marks it crashed so a crash report
+# is produced. That is upstream's default cron behavior and it recovers a single node in
+# place. The beacon in salt/_beacons/zeek.py is the only other recovery path, it is disabled
+# by default (healthcheck:enabled), and it removes and recreates the whole container, so
+# letting zeekctl handle a single dead worker avoids the heavier restart.
+
+if ! docker ps --filter name=so-zeek --format '{{.Names}}' | grep -q '^so-zeek$'; then
+  exit 0
+fi
+
+# Run as the zeek user so the stats logs and zeekctl-config.sh this writes stay owned by
+# uid 937 rather than root.
+docker exec so-zeek runuser -l zeek -c '/opt/zeek/bin/zeekctl cron'