From ec27517bdd7a29f2f0f5532b1c9ba84df0d1ac45 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Fri, 11 Jul 2025 10:37:50 -0600 Subject: [PATCH 01/14] New Config Values New config values with annotations and defaults. Updated Nginx config to allow streaming requests to not be buffered on the way to the client. --- salt/nginx/etc/nginx.conf | 30 +++++++++++++++++------------- salt/soc/defaults.yaml | 4 ++++ salt/soc/soc_soc.yaml | 11 +++++++++++ 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index 742f5d08d..caa05bbff 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -196,19 +196,23 @@ http { } location / { - auth_request /auth/sessions/whoami; - auth_request_set $userid $upstream_http_x_kratos_authenticated_identity_id; - proxy_set_header x-user-id $userid; - proxy_pass http://{{ GLOBALS.manager }}:9822/; - proxy_read_timeout 300; - proxy_connect_timeout 300; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Proxy ""; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header X-Forwarded-Proto $scheme; + auth_request /auth/sessions/whoami; + auth_request_set $userid $upstream_http_x_kratos_authenticated_identity_id; + proxy_set_header x-user-id $userid; + proxy_pass http://{{ GLOBALS.manager }}:9822/; + proxy_read_timeout 300; + proxy_connect_timeout 300; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_buffering off; + proxy_cache off; + proxy_request_buffering off; } location ~ ^/auth/.*?(login|oidc/callback) { diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 7bb2c1f03..c86889be7 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1491,6 +1491,10 @@ soc: - repo: file:///nsm/airgap-resources/playbooks/securityonion-resources-playbooks branch: main folder: securityonion-normalized + assistant: + apiKey: + apiUrl: https://onionai-dev.securityonion.net + model: claude-sonnet salt: queueDir: /opt/sensoroni/queue timeoutMs: 45000 diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 2d0eb3792..b8133999f 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -580,6 +580,17 @@ soc: - field: folder label: Folder airgap: *pbRepos + assistant: + apiKey: + description: The auth token to be used when reaching out to the AI Assistant. + global: True + apiUrl: + description: The URL of the AI gateway. + advanced: True + global: True + model: + description: The model to use as the AI Assistant + global: True client: apiTimeoutMs: description: Duration (in milliseconds) to wait for a response from the SOC server API before giving up and showing an error on the SOC UI. From ba601c39b37e8d2aa3158849d903753d4af3653f Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 29 Jul 2025 11:23:28 -0600 Subject: [PATCH 02/14] Rough Go at New Mappings/Settings --- .../component/so/assistant-mappings.json | 48 +++++++++++++++++++ .../component/so/assistant-settings.json | 7 +++ 2 files changed, 55 insertions(+) create mode 100644 salt/elasticsearch/templates/component/so/assistant-mappings.json create mode 100644 salt/elasticsearch/templates/component/so/assistant-settings.json diff --git a/salt/elasticsearch/templates/component/so/assistant-mappings.json b/salt/elasticsearch/templates/component/so/assistant-mappings.json new file mode 100644 index 000000000..ad17ebd66 --- /dev/null +++ b/salt/elasticsearch/templates/component/so/assistant-mappings.json @@ -0,0 +1,48 @@ +{ + "template": { + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "so_kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "so_operation": { + "ignore_above": 1024, + "type": "keyword" + }, + "so_chat": { + "properties": { + "role": { + "ignore_above": 1024, + "type": "keyword" + }, + "content": { + "type": "text" + }, + "conversation_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "createTime": { + "type": "date" + }, + "tool_use_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "userId": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + }, + "_meta": { + "ecs_version": "1.12.2" + } +} diff --git a/salt/elasticsearch/templates/component/so/assistant-settings.json b/salt/elasticsearch/templates/component/so/assistant-settings.json new file mode 100644 index 000000000..0281fa0e1 --- /dev/null +++ b/salt/elasticsearch/templates/component/so/assistant-settings.json @@ -0,0 +1,7 @@ +{ + "template": {}, + "version": 1, + "_meta": { + "description": "default settings for common Security Onion Assistant indices" + } +} From 6323fbf46b3b27bfc6bc8bb0caf9152a04892892 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Wed, 30 Jul 2025 11:48:27 -0600 Subject: [PATCH 03/14] Content Object --- .../templates/component/so/assistant-mappings.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/component/so/assistant-mappings.json b/salt/elasticsearch/templates/component/so/assistant-mappings.json index ad17ebd66..f6f552465 100644 --- a/salt/elasticsearch/templates/component/so/assistant-mappings.json +++ b/salt/elasticsearch/templates/component/so/assistant-mappings.json @@ -20,7 +20,8 @@ "type": "keyword" }, "content": { - "type": "text" + "type": "object", + "enabled": false }, "conversation_id": { "ignore_above": 1024, From b1753f86f91b8345151f16de7eb9a06d2145fbe9 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Wed, 30 Jul 2025 13:14:09 -0600 Subject: [PATCH 04/14] New Message Structure --- .../component/so/assistant-mappings.json | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/component/so/assistant-mappings.json b/salt/elasticsearch/templates/component/so/assistant-mappings.json index f6f552465..89a907165 100644 --- a/salt/elasticsearch/templates/component/so/assistant-mappings.json +++ b/salt/elasticsearch/templates/component/so/assistant-mappings.json @@ -23,7 +23,7 @@ "type": "object", "enabled": false }, - "conversation_id": { + "sessionId": { "ignore_above": 1024, "type": "keyword" }, @@ -37,6 +37,54 @@ "userId": { "ignore_above": 1024, "type": "keyword" + }, + "message": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "role": { + "ignore_above": 1024, + "type": "keyword" + }, + "model": { + "ignore_above": 1024, + "type": "keyword" + }, + "contentStr": { + "type": "text" + }, + "contentBlocks": { + "type": "nested", + "enabled": false + }, + "stopReason": { + "ignore_above": 1024, + "type": "keyword" + }, + "stopSequence": { + "ignore_above": 1024, + "type": "keyword" + }, + "usage": { + "properties": { + "input_tokens": { + "type": "long" + }, + "output_tokens": { + "type": "long" + }, + "credits": { + "type": "long" + } + } + } + } } } } From cea4eaf0819ba87e7edc2c16622d5ad8ccc891f6 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Wed, 6 Aug 2025 09:02:43 -0600 Subject: [PATCH 05/14] Updated Assistant Mapping --- .../templates/component/so/assistant-mappings.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/elasticsearch/templates/component/so/assistant-mappings.json b/salt/elasticsearch/templates/component/so/assistant-mappings.json index 89a907165..3433acbd6 100644 --- a/salt/elasticsearch/templates/component/so/assistant-mappings.json +++ b/salt/elasticsearch/templates/component/so/assistant-mappings.json @@ -30,6 +30,13 @@ "createTime": { "type": "date" }, + "deletedAt": { + "type": "date" + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, "tool_use_id": { "ignore_above": 1024, "type": "keyword" From fc2d450de04c688a1ade7724aad4aa31fe637c5a Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 26 Aug 2025 09:16:04 -0600 Subject: [PATCH 06/14] Update Settings The apiKey will be built off of the license rather than a new setting. The model is hardcoded for now at the AI Gateway level. We're going to use the investigationPrompt as a trigger for the feature being visible in the UI but by default will be blank for now. --- salt/soc/defaults.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index c86889be7..f89d9e99f 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1492,9 +1492,8 @@ soc: branch: main folder: securityonion-normalized assistant: - apiKey: apiUrl: https://onionai-dev.securityonion.net - model: claude-sonnet + investigationPrompt: salt: queueDir: /opt/sensoroni/queue timeoutMs: 45000 From 120e61e45cce0bd6796f819d4b96b5bbb2b7e9a3 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 26 Aug 2025 16:06:14 -0600 Subject: [PATCH 07/14] ClientParams Removed investigation prompt from module settings and moved to client settings, added enabledInSoc. --- salt/soc/defaults.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index f89d9e99f..d847d1d1b 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1493,7 +1493,6 @@ soc: folder: securityonion-normalized assistant: apiUrl: https://onionai-dev.securityonion.net - investigationPrompt: salt: queueDir: /opt/sensoroni/queue timeoutMs: 45000 @@ -2544,3 +2543,6 @@ soc: - ' -priv' condition: all of selection_* level: 'high' # info | low | medium | high | critical + assistant: + enabledInSoc: false + investigationPrompt: Investigate Alert ID {socid} \ No newline at end of file From 73776f8d11ac14269bf531cc0c6cefbe548413f3 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Wed, 27 Aug 2025 12:46:19 -0600 Subject: [PATCH 08/14] Cleaning up New ES Indexes --- salt/elasticsearch/defaults.yaml | 74 +++++++++++++++++++ ...ings.json => assistant-chat-mappings.json} | 0 ...ings.json => assistant-chat-settings.json} | 0 .../so/assistant-session-mappings.json | 44 +++++++++++ .../so/assistant-session-settings.json | 7 ++ 5 files changed, 125 insertions(+) rename salt/elasticsearch/templates/component/so/{assistant-mappings.json => assistant-chat-mappings.json} (100%) rename salt/elasticsearch/templates/component/so/{assistant-settings.json => assistant-chat-settings.json} (100%) create mode 100644 salt/elasticsearch/templates/component/so/assistant-session-mappings.json create mode 100644 salt/elasticsearch/templates/component/so/assistant-session-settings.json diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 1200701c9..b5031b9b2 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -284,6 +284,80 @@ elasticsearch: hot: actions: {} min_age: 0ms + so-assistant-chat: + index_sorting: false + index_template: + composed_of: + - assistant-chat-mappings + - assistant-chat-settings + ignore_missing_component_templates: [] + index_patterns: + - so-assistant-chat* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + lifecycle: + name: so-assistant-chat-logs + mapping: + total_fields: + limit: 1500 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 1s + sort: + field: '@timestamp' + order: desc + policy: + phases: + hot: + actions: {} + min_age: 0ms + so-assistant-session: + index_sorting: false + index_template: + composed_of: + - assistant-session-mappings + - assistant-session-settings + ignore_missing_component_templates: [] + index_patterns: + - so-assistant-session* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + lifecycle: + name: so-assistant-session-logs + mapping: + total_fields: + limit: 1500 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 1s + sort: + field: '@timestamp' + order: desc + policy: + phases: + hot: + actions: {} + min_age: 0ms so-endgame: index_sorting: false index_template: diff --git a/salt/elasticsearch/templates/component/so/assistant-mappings.json b/salt/elasticsearch/templates/component/so/assistant-chat-mappings.json similarity index 100% rename from salt/elasticsearch/templates/component/so/assistant-mappings.json rename to salt/elasticsearch/templates/component/so/assistant-chat-mappings.json diff --git a/salt/elasticsearch/templates/component/so/assistant-settings.json b/salt/elasticsearch/templates/component/so/assistant-chat-settings.json similarity index 100% rename from salt/elasticsearch/templates/component/so/assistant-settings.json rename to salt/elasticsearch/templates/component/so/assistant-chat-settings.json diff --git a/salt/elasticsearch/templates/component/so/assistant-session-mappings.json b/salt/elasticsearch/templates/component/so/assistant-session-mappings.json new file mode 100644 index 000000000..b72bbb389 --- /dev/null +++ b/salt/elasticsearch/templates/component/so/assistant-session-mappings.json @@ -0,0 +1,44 @@ +{ + "template": { + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "so_kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "so_session": { + "properties": { + "title": { + "ignore_above": 1024, + "type": "keyword" + }, + "sessionId": { + "ignore_above": 1024, + "type": "keyword" + }, + "createTime": { + "type": "date" + }, + "deleteTime": { + "type": "date" + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "userId": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + }, + "_meta": { + "ecs_version": "1.12.2" + } +} diff --git a/salt/elasticsearch/templates/component/so/assistant-session-settings.json b/salt/elasticsearch/templates/component/so/assistant-session-settings.json new file mode 100644 index 000000000..0281fa0e1 --- /dev/null +++ b/salt/elasticsearch/templates/component/so/assistant-session-settings.json @@ -0,0 +1,7 @@ +{ + "template": {}, + "version": 1, + "_meta": { + "description": "default settings for common Security Onion Assistant indices" + } +} From 834e34128d5b3f8d5e0e89176968e89d8464f0d1 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Thu, 28 Aug 2025 16:03:35 -0600 Subject: [PATCH 09/14] Non-dev URL --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index d847d1d1b..0e7d8ac99 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1492,7 +1492,7 @@ soc: branch: main folder: securityonion-normalized assistant: - apiUrl: https://onionai-dev.securityonion.net + apiUrl: https://onionai.securityonion.net salt: queueDir: /opt/sensoroni/queue timeoutMs: 45000 From 0a3ff47008029b369381abf0e4ecc5d7629a7505 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Wed, 3 Sep 2025 12:12:27 -0600 Subject: [PATCH 10/14] Cleanup Annotations Removed fields no longer need annotations. --- salt/soc/soc_soc.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index b8133999f..251a3c037 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -581,16 +581,10 @@ soc: label: Folder airgap: *pbRepos assistant: - apiKey: - description: The auth token to be used when reaching out to the AI Assistant. - global: True apiUrl: description: The URL of the AI gateway. advanced: True global: True - model: - description: The model to use as the AI Assistant - global: True client: apiTimeoutMs: description: Duration (in milliseconds) to wait for a response from the SOC server API before giving up and showing an error on the SOC UI. From 673f9cb544682efe5129727e1c490fce03f69748 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Thu, 4 Sep 2025 09:20:50 -0600 Subject: [PATCH 11/14] Responding to Feedback --- salt/elasticsearch/defaults.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index b5031b9b2..9eb4b4901 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -292,8 +292,8 @@ elasticsearch: - assistant-chat-settings ignore_missing_component_templates: [] index_patterns: - - so-assistant-chat* - priority: 500 + - so-assistant-chat-* + priority: 501 template: mappings: date_detection: false @@ -329,8 +329,8 @@ elasticsearch: - assistant-session-settings ignore_missing_component_templates: [] index_patterns: - - so-assistant-session* - priority: 500 + - so-assistant-session-* + priority: 501 template: mappings: date_detection: false From 855b489c4b4ead1ca5273136723ba881ed567750 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 4 Sep 2025 10:39:57 -0500 Subject: [PATCH 12/14] datastream --- salt/elasticsearch/defaults.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 9eb4b4901..e51f5ac4e 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -290,6 +290,9 @@ elasticsearch: composed_of: - assistant-chat-mappings - assistant-chat-settings + data_stream: + allow_custom_routing: false + hidden: false ignore_missing_component_templates: [] index_patterns: - so-assistant-chat-* @@ -327,6 +330,9 @@ elasticsearch: composed_of: - assistant-session-mappings - assistant-session-settings + data_stream: + allow_custom_routing: false + hidden: false ignore_missing_component_templates: [] index_patterns: - so-assistant-session-* From 12959d114c103de5050f33081ea94211f869baad Mon Sep 17 00:00:00 2001 From: Matthew Wright Date: Thu, 4 Sep 2025 16:36:51 -0400 Subject: [PATCH 13/14] added threshold config fields for assistant --- salt/soc/defaults.yaml | 8 +++++++- salt/soc/soc_soc.yaml | 30 ++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 0e7d8ac99..fe50fced7 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -2545,4 +2545,10 @@ soc: level: 'high' # info | low | medium | high | critical assistant: enabledInSoc: false - investigationPrompt: Investigate Alert ID {socid} \ No newline at end of file + investigationPrompt: Investigate Alert ID {socid} + contextLimitSmall: 200000 + contextLimitLarge: 1000000 + thresholdColorRatioLow: 0.5 + thresholdColorRatioMed: 0.75 + thresholdColorRatioMax: 1 + lowBalanceColorAlert: 500000 \ No newline at end of file diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 251a3c037..cde5996ee 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -586,6 +586,36 @@ soc: advanced: True global: True client: + assistant: + enabledInSoc: + description: Set to true to enable the Onion AI assistant in SOC. + global: True + investigationPrompt: + description: Prompt given to Onion AI when beginning an investigation. + global: True + contextLimitSmall: + description: Smaller context limit for Onion AI. + global: True + advanced: True + contextLimitLarge: + description: Larger context limit for Onion AI. + global: True + advanced: True + thresholdColorRatioLow: + description: Lower visual context color change threshold. + global: True + advanced: True + thresholdColorRatioMed: + description: Middle visual context color change threshold. + global: True + advanced: True + thresholdColorRatioMax: + description: Max visual context color change threshold. + global: True + advanced: True + lowBalanceColorAlert: + description: Onion AI credit amount at which balance turns red. + advanced: True apiTimeoutMs: description: Duration (in milliseconds) to wait for a response from the SOC server API before giving up and showing an error on the SOC UI. global: True From aa43177d8c48af6c4d6170d087689b6dfdfcb24f Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Fri, 5 Sep 2025 11:31:08 -0600 Subject: [PATCH 14/14] Fix Setting Name enabledInSoc => enabled --- salt/soc/defaults.yaml | 2 +- salt/soc/soc_soc.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index fe50fced7..58b3a3827 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -2544,7 +2544,7 @@ soc: condition: all of selection_* level: 'high' # info | low | medium | high | critical assistant: - enabledInSoc: false + enabled: false investigationPrompt: Investigate Alert ID {socid} contextLimitSmall: 200000 contextLimitLarge: 1000000 diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index cde5996ee..4af20d444 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -587,7 +587,7 @@ soc: global: True client: assistant: - enabledInSoc: + enabled: description: Set to true to enable the Onion AI assistant in SOC. global: True investigationPrompt: