Merge remote-tracking branch 'origin/3/dev' into zeekrestart

Merge pull request #16236 from Security-Onion-Solutions/cogburn/move-fields
filterEventFields
2026-09-30 11:37:16 +02:00 · 2026-09-11 15:06:28 -04:00 · 2026-09-11 11:23:03 -06:00 · 2026-09-11 10:54:20 -06:00 · 2026-09-10 13:20:20 -04:00 · 2026-09-10 07:36:32 -04:00
10 changed files with 100 additions and 4 deletions
@@ -13,6 +13,7 @@ body:
        - 3.1.0
        - 3.2.0
        - 3.3.0
+        - 3.4.0
        - Other (please provide detail below)
    validations:
      required: true
@@ -1 +1 @@
-20260911
+
@@ -1 +1 @@
-3.3.0
+3.4.0
@@ -183,7 +183,7 @@ http {
 		ssl_prefer_server_ciphers on;
 		ssl_protocols TLSv1.2 TLSv1.3;

-		location ~* (^/login/.*|^/js/.*|^/css/.*|^/images/.*|^/pages/.*|^/docs/.*) {
+		location ~* (^/login|^/login/.*|^/js/.*|^/css/.*|^/images/.*|^/pages/.*|^/docs/.*) {
 			proxy_pass            http://{{ GLOBALS.manager }}:9822;
 			proxy_read_timeout    90;
 			proxy_connect_timeout 90;
@@ -1537,7 +1537,7 @@ soc:
            Orchestrator: sonnet@SOAI
            Investigator: gemma@SOAI
            DetectionEngineer: gemma@SOAI
-          useMemory: false
+          useMemory: true
          useMemoryScanner: false
          dontScanBefore: ""
          memoryScanIntervalSeconds: 300
@@ -1556,6 +1556,69 @@ soc:
          reconcilePersona: ""
          toolUseTurnAttempts: 12
          toolUseTurnDelayMs: 175
+          tools:
+            filterEventFields:
+              - "@timestamp"
+              - "client.name"
+              - "destination.ip"
+              - "destination.port"
+              - "destination.geo.country_name"
+              - "dns.query.name"
+              - "dns.query_name"
+              - "event.action"
+              - "event.category"
+              - "event.module"
+              - "event.dataset"
+              - "event.outcome"
+              - "event.severity"
+              - "event.severity_label"
+              - "event.type"
+              - "event_data.agent.name"
+              - "event_data.host.os.name"
+              - "file.mime_type"
+              - "file.name"
+              - "hash.md5"
+              - "hash.sha1"
+              - "host.mac"
+              - "host.name"
+              - "host.os.name"
+              - "http.method"
+              - "http.useragent"
+              - "http.virtual_host"
+              - "log.id.uid"
+              - "network.community_id"
+              - "network.protocol"
+              - "network.transport"
+              - "notice.message"
+              - "observer.name"
+              - "process.name"
+              - "process.executable"
+              - "process.entity_id"
+              - "process.command_line"
+              - "process.Ext.ancestry"
+              - "process.parent.entity_id"
+              - "process.parent.command_line"
+              - "rule.category"
+              - "rule.name"
+              - "rule.uuid"
+              - "software.name"
+              - "software.type"
+              - "software.version.unparsed"
+              - "source.ip"
+              - "source.port"
+              - "source.geo.country_name"
+              - "ssh.cypher_algorithm"
+              - "ssh.client"
+              - "ssh.server"
+              - "ssl.cipher"
+              - "ssl.server_name"
+              - "ssl.version"
+              - "system.auth.sudo.command"
+              - "user.name"
+              - "user.domain"
+              - "user.effective.name"
+              - "weird.name"
+              - "tags"
        onionconfig:
          saltstackDir: /opt/so/saltstack
          bypassEnabled: false
@@ -916,6 +916,11 @@ soc:
            description: The number of times to retry extracting memories from a session if errors occur.
            global: True
            advanced: True
+          tools:
+            filterEventFields:
+              description: A whitelist of fields to return when OnionAI uses the query_events tool. All other fields are removed. One field per line.
+              global: True
+              multiline: True
      client:
        assistant:
          enabled:
@@ -18,6 +18,7 @@ zeek:
      StatsLogEnable: 0
      StatsLogExpireInterval: 0
      StatusCmdShowAll: 0
+      StopWait: 1
      CrashExpireInterval: 0
      SitePolicyScripts: local.zeek
      LogDir: /nsm/zeek/logs
@@ -9,9 +9,19 @@
 include:
  - zeek.sostatus
  
+# Stop first so the entrypoint's SIGTERM trap can archive the final logs; docker_container.absent
+# with force is a 'docker rm -f', which never delivers SIGTERM. force stays so the state still
+# converges if the stop overruns.
+so-zeek_stopped:
+  docker_container.stopped:
+    - name: so-zeek
+    - error_on_absent: False
+
 so-zeek:
  docker_container.absent:
    - force: True
+    - require:
+      - docker_container: so-zeek_stopped

 so-zeek_so-status.disabled:
  file.comment:
@@ -19,6 +19,10 @@ so-zeek:
    - restart_policy: unless-stopped
    - start: True
    - privileged: True
+    # Docker's default 10s grace is not enough for the entrypoint's SIGTERM trap to run
+    # 'zeekctl stop' and let StopWait archive the final logs. Overrunning it means SIGKILL,
+    # which strands those logs in spool/tmp and marks every node crashed on the next start.
+    - stop_timeout: 180
    {% if DOCKERMERGED.containers['so-zeek'].ulimits %}
    - ulimits:
    {%   for ULIMIT in DOCKERMERGED.containers['so-zeek'].ulimits %}
@@ -99,6 +99,18 @@ zeek:
        regexFailureMessage: You must enter a whole number of days, or 0 to keep crash directories forever.
        helpLink: zeek
        advanced: True
+      StopWait:
+        description: >-
+          Set to 1 to make "zeekctl stop" wait for the final logs to be archived instead of
+          letting that finish in the background. Security Onion stops Zeek by stopping its
+          container, so anything still running in the background is killed when the container
+          exits - without this, the last logs of each run are stranded unarchived in
+          /nsm/zeek/spool/tmp and never reach Elasticsearch. It is read only for that reason.
+        regex: ^[01]$
+        regexFailureMessage: You must enter 0 or 1.
+        helpLink: zeek
+        advanced: True
+        readonly: True
      MinDiskSpace:
        description: >-
          Percentage of free disk space below which ZeekControl reports a warning, or 0 to disable the check
Author	SHA1	Message	Date
Josh Patterson	56e3e44d04	Merge remote-tracking branch 'origin/3/dev' into zeekrestart	2026-09-11 15:06:28 -04:00
coreyogburn	32d1274b80	Merge pull request #16236 from Security-Onion-Solutions/cogburn/move-fields filterEventFields	2026-09-11 11:23:03 -06:00
Corey Ogburn	1624e8c094	filterEventFields Previously hard coded array of fields is now a config option with the hard coded value as the default value.	2026-09-11 10:54:20 -06:00
Josh Patterson	9652a2053b	Stop the Zeek container gracefully zeekctl's post-terminate archives the final logs in the background and returns immediately unless StopWait is set, so the container exits and takes the archiving with it, stranding unarchived logs in /nsm/zeek/spool/tmp on every restart. Docker's default 10s grace is also too tight for the entrypoint's SIGTERM trap; overrunning it means SIGKILL and crash directories on the next start. Both are needed. StopWait alone gives the stop more work to do inside the same 10s window, which was measured ending in SIGKILL with logs stranded in the spool. disabled.sls used docker rm -f, which never delivers SIGTERM, so stop the container before removing it. Reported in discussion #16174.	2026-09-10 13:20:20 -04:00
Jason Ertel	191ee159ef	Merge pull request #16229 from Security-Onion-Solutions/jertel/wip fix location typo	2026-09-10 07:36:32 -04:00
Jason Ertel	a8bfe955a5	fix location typo	2026-09-10 07:27:25 -04:00
Jason Ertel	bd354abe83	Merge pull request #16225 from Security-Onion-Solutions/jertel/wip /login is showing an nginx failure	2026-09-09 16:54:44 -04:00
Jason Ertel	37782fb45c	/login is showing an nginx failure	2026-09-09 16:44:27 -04:00
Mike Reeves	c49008a413	Merge pull request #16216 from Security-Onion-Solutions/TOoSmOotH-patch-1 Bump version from 3.3.0 to 3.4.0	2026-09-08 15:15:12 -04:00
Mike Reeves	fcbea1a1c4	Add 3.4.0 option to discussion template	2026-09-08 15:14:26 -04:00
Mike Reeves	6736f9c3a0	Bump version from 3.3.0 to 3.4.0	2026-09-08 15:13:40 -04:00
@@ -1 +1 @@
 .3.0
 .4.0