Merge pull request #15319 from Security-Onion-Solutions/2.4/dev

2.4.200

.github/DISCUSSION_TEMPLATE/2-4.yml

@@ -33,7 +33,6 @@ body:
         - 2.4.180
         - 2.4.190
         - 2.4.200
-        - 2.4.210
         - Other (please provide detail below)
     validations:
       required: true

VERSION

@@ -1 +1 @@
-2.4.210
+2.4.200

salt/common/tools/sbin/so-log-check

@@ -129,7 +129,6 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|responded with status-code 503" # telegraf getting 503 from ES during startup
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|process_cluster_event_timeout_exception" # logstash waiting for elasticsearch to start
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|not configured for GeoIP"     # SO does not bundle the maxminddb with Zeek
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|HTTP 404: Not Found"          # Salt loops until Kratos returns 200, during startup Kratos may not be ready
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then

salt/elasticsearch/defaults.yaml

@@ -857,11 +857,53 @@ elasticsearch:
         composed_of:
         - agent-mappings
         - dtc-agent-mappings
+        - base-mappings
+        - dtc-base-mappings
+        - client-mappings
+        - dtc-client-mappings
+        - container-mappings
+        - destination-mappings
+        - dtc-destination-mappings
+        - pb-override-destination-mappings
+        - dll-mappings
+        - dns-mappings
+        - dtc-dns-mappings
+        - ecs-mappings
+        - dtc-ecs-mappings
+        - error-mappings
+        - event-mappings
+        - dtc-event-mappings
+        - file-mappings
+        - dtc-file-mappings
+        - group-mappings
         - host-mappings
         - dtc-host-mappings
         - http-mappings
         - dtc-http-mappings
+        - log-mappings
         - metadata-mappings
+        - network-mappings
+        - dtc-network-mappings
+        - observer-mappings
+        - dtc-observer-mappings
+        - organization-mappings
+        - package-mappings
+        - process-mappings
+        - dtc-process-mappings
+        - related-mappings
+        - rule-mappings
+        - dtc-rule-mappings
+        - server-mappings
+        - service-mappings
+        - dtc-service-mappings
+        - source-mappings
+        - dtc-source-mappings
+        - pb-override-source-mappings
+        - threat-mappings
+        - tls-mappings
+        - url-mappings
+        - user_agent-mappings
+        - dtc-user_agent-mappings
         - common-settings
         - common-dynamic-mappings
         data_stream:

salt/elasticsearch/files/ingest/kratos

@@ -1,90 +1,9 @@
 {
-    "description": "kratos",
+  "description" : "kratos",
-    "processors": [
+  "processors" : [
-        {
+    {"set":{"field":"audience","value":"access","override":false,"ignore_failure":true}},
-            "set": {
+    {"set":{"field":"event.dataset","ignore_empty_value":true,"ignore_failure":true,"value":"kratos.{{{audience}}}","media_type":"text/plain"}},
-                "field": "audience",
+    {"set":{"field":"event.action","ignore_failure":true,"copy_from":"msg"    }},
-                "value": "access",
+    { "pipeline":    { "name": "common" } }
-                "override": false,
+  ]
-                "ignore_failure": true
-            }
-        },
-        {
-            "set": {
-                "field": "event.dataset",
-                "ignore_empty_value": true,
-                "ignore_failure": true,
-                "value": "kratos.{{{audience}}}",
-                "media_type": "text/plain"
-            }
-        },
-        {
-            "set": {
-                "field": "event.action",
-                "ignore_failure": true,
-                "copy_from": "msg"
-            }
-        },
-        {
-            "rename": {
-                "field": "http_request",
-                "target_field": "http.request",
-                "ignore_failure": true,
-                "ignore_missing": true
-            }
-        },
-        {
-            "rename": {
-                "field": "http_response",
-                "target_field": "http.response",
-                "ignore_failure": true,
-                "ignore_missing": true
-            }
-        },
-        {
-            "rename": {
-                "field": "http.request.path",
-                "target_field": "http.uri",
-                "ignore_failure": true,
-                "ignore_missing": true
-            }
-        },
-        {
-            "rename": {
-                "field": "http.request.method",
-                "target_field": "http.method",
-                "ignore_failure": true,
-                "ignore_missing": true
-            }
-        },
-        {
-            "rename": {
-                "field": "http.request.method",
-                "target_field": "http.method",
-                "ignore_failure": true,
-                "ignore_missing": true
-            }
-        },
-        {
-            "rename": {
-                "field": "http.request.query",
-                "target_field": "http.query",
-                "ignore_failure": true,
-                "ignore_missing": true
-            }
-        },
-        {
-            "rename": {
-                "field": "http.request.headers.user-agent",
-                "target_field": "http.useragent",
-                "ignore_failure": true,
-                "ignore_missing": true
-            }
-        },
-        {
-            "pipeline": {
-                "name": "common"
-            }
-        }
-    ]
 }

salt/kratos/config.sls

@@ -75,7 +75,6 @@ kratosconfig:
     - group: 928
     - mode: 600
     - template: jinja
-    - show_changes: False
     - defaults:
         KRATOSMERGED: {{ KRATOSMERGED }}
 

salt/kratos/defaults.yaml

@@ -46,7 +46,6 @@ kratos:
           ui_url: https://URL_BASE/
         login:
           ui_url: https://URL_BASE/login/
-          lifespan: 60m
         error:
           ui_url: https://URL_BASE/login/
         registration:

salt/kratos/soc_kratos.yaml

@@ -182,10 +182,6 @@ kratos:
             global: True
             advanced: True
             helpLink: kratos.html
-          lifespan: 
-            description: Defines the duration that a login form will remain valid.
-            global: True
-            helpLink: kratos.html
         error:
           ui_url: 
             description: User accessible URL containing the Security Onion login page. Leave as default to ensure proper operation.

salt/manager/tools/sbin/soup

@@ -427,7 +427,6 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.4.170 ]] && up_to_2.4.180
     [[ "$INSTALLEDVERSION" == 2.4.180 ]] && up_to_2.4.190
     [[ "$INSTALLEDVERSION" == 2.4.190 ]] && up_to_2.4.200
-    [[ "$INSTALLEDVERSION" == 2.4.200 ]] && up_to_2.4.210
     true
 }
 
@@ -460,7 +459,6 @@ postupgrade_changes() {
     [[ "$POSTVERSION"  == 2.4.170 ]] && post_to_2.4.180
     [[ "$POSTVERSION"  == 2.4.180 ]] && post_to_2.4.190
     [[ "$POSTVERSION"  == 2.4.190 ]] && post_to_2.4.200
-    [[ "$POSTVERSION"  == 2.4.200 ]] && post_to_2.4.210
     true
 }
 
@@ -647,14 +645,6 @@ post_to_2.4.200() {
   POSTVERSION=2.4.200
 }
 
-post_to_2.4.210() {
-  echo "Rolling over Kratos index to apply new index template"
-
-  rollover_index "logs-kratos-so"
-
-  POSTVERSION=2.4.210
-}
-
 repo_sync() {
   echo "Sync the local repo."
   su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
@@ -931,12 +921,6 @@ up_to_2.4.200() {
   INSTALLEDVERSION=2.4.200
 }
 
-up_to_2.4.210() {
-  echo "Nothing to do for 2.4.210"
-
-  INSTALLEDVERSION=2.4.210
-}
-
 add_hydra_pillars() {
   mkdir -p /opt/so/saltstack/local/pillar/hydra
   touch /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls

salt/sensoroni/files/templates/reports/standard/assistant_session_report.md

@@ -1,91 +0,0 @@
-Onion AI Session Report
-==========================
-
-## Session Details
-
-**Session ID:** {{.Session.SessionId}}
-
-**Title:** {{.Session.Title}}
-
-**Created:** {{formatDateTime "Mon Jan 02 15:04:05 -0700 2006" .Session.CreateTime}}
-
-**Updated:** {{formatDateTime "Mon Jan 02 15:04:05 -0700 2006" .Session.UpdateTime}}
-
-{{ if .Session.DeleteTime }}
-**Deleted:** {{ formatDateTime "Mon Jan 02 15:04:05 -0700 2006" .Session.DeleteTime}}
-{{ end }}
-
-**User ID:** {{getUserDetail "email" .Session.UserId}}
-
-## Session Usage
-
-**Total Input Tokens** {{.Session.Usage.TotalInputTokens}}
-
-**Total Output Tokens** {{.Session.Usage.TotalOutputTokens}}
-
-**Total Credits:** {{.Session.Usage.TotalCredits}}
-
-**Total Messages:** {{.Session.Usage.TotalMessages}}
-
-## Messages
-
-{{ range $index, $msg := sortAssistantMessages "CreateTime" "asc" .History }}
-#### Message {{ add $index 1 }}
-
-**Created:** {{formatDateTime "Mon Jan 02 15:04:05 -0700 2006" $msg.CreateTime}}
-
-**User ID:** {{getUserDetail "email" $msg.UserId}}
-
-**Role:** {{$msg.Message.Role}}
-
-{{ range $i, $block := $msg.Message.ContentBlocks }}
-
----
-
-{{ if eq $block.Type "text" }}
-**Text:** {{ stripEmoji $block.Text }}
-{{ else if eq $block.Type "tool_use" }}
-**Tool:** {{ $block.Name }}
-{{ if $block.Input }}
-**Parameters:**
-{{ range $key, $value := parseJSON $block.Input }}
-{{ if eq $key "limit" }}- {{ $key }}: {{ $value }}
-{{ else }}- {{ $key }}: "{{ $value }}"
-{{ end }}{{ end }}{{ end }}
-{{ else if $block.ToolResult }}
-**Tool Result:**
-{{ if $block.ToolResult.Content }}
-{{ range $j, $contentBlock := $block.ToolResult.Content }}
-{{ if gt $j 0 }}
-
----
-
-{{ end }}
-{{ if $contentBlock.Text }}
-{{ if $block.ToolResult.IsError }}
-**Error:** {{ $contentBlock.Text }}
-{{ else }}
-{{ $contentBlock.Text }}
-{{ end }}
-{{ else if $contentBlock.Json }}
-```json
-{{ toJSON $contentBlock.Json }}
-```
-{{ end }}{{ end }}
-{{ end }}{{ end }}{{ end }}
-
-{{ if eq $msg.Message.Role "assistant" }}{{ if $msg.Message.Usage }}
-
----
-
-**Message Usage:**
-
-- Input Tokens: {{$msg.Message.Usage.InputTokens}}
-- Output Tokens: {{$msg.Message.Usage.OutputTokens}}
-- Credits: {{$msg.Message.Usage.Credits}}
-
-{{end}}{{end}}
-
----
-
-{{end}}

salt/sensoroni/files/templates/reports/standard/case_report.md

@@ -130,42 +130,4 @@ Security Onion Case Report
 | ---- | ---- | ------ | --------- |
 {{ range sortHistory "CreateTime" "asc" .History -}}
 | {{formatDateTime "Mon Jan 02 15:04:05 -0700 2006" .CreateTime}} | {{getUserDetail "email" .UserId}} | {{.Kind}} | {{.Operation}} |
-{{end}}
-
-## Attached Onion AI Sessions
-
-{{ range $idx, $session := sortAssistantSessionDetails "CreateTime" "desc" .AssistantSessions }}
-
-#### Session {{ add $idx 1 }}
-
-**Session ID:** {{$session.Session.SessionId}}
-
-**Title:** {{$session.Session.Title}}
-
-**User ID:** {{getUserDetail "email" $session.Session.UserId}}
-
-**Created:** {{formatDateTime "Mon Jan 02 15:04:05 -0700 2006" $session.Session.CreateTime}}
-
-**Updated:** {{formatDateTime "Mon Jan 02 15:04:05 -0700 2006" $session.Session.UpdateTime}}
-
-{{ if $session.Session.DeleteTime }}
-**Deleted:** {{ formatDateTime "Mon Jan 02 15:04:05 -0700 2006" $session.Session.DeleteTime}}
-{{ end }}
-
-#### Messages
-
-{{ range $index, $msg := sortAssistantMessages "CreateTime" "asc" $session.History }}
-{{ range $i, $block := $msg.Message.ContentBlocks }}
-
-{{ if eq $block.Type "text" }}
-
-**Role:** {{$msg.Message.Role}}
-
-{{ stripEmoji $block.Text }}
-
----
-
-{{ end }}{{ end }}
-
-{{end}}
 {{end}}

salt/sensoroni/soc_sensoroni.yaml

@@ -357,7 +357,7 @@ sensoroni:
       reports:
         standard:
           case_report__md:
-            title: Case Report Template
+            title: Case report Template
             description: The template used when generating a case report. Supports markdown format.
             file: True
             global: True
@@ -370,13 +370,6 @@ sensoroni:
             global: True
             syntax: md
             helpLink: reports.html
-          assistant_session_report__md:
-            title: Assistant Session Report Template
-            description: The template used when generating an assistant session report. Supports markdown format.
-            file: True
-            global: True
-            syntax: md
-            helplink: reports.html
         custom:
           generic_report1__md:
             title: Custom Report 1

salt/soc/defaults.yaml

@@ -115,16 +115,16 @@ soc:
       ':kratos:':
         - soc_timestamp
         - event.dataset
-        - http.request.headers.x-real-ip
+        - http_request.headers.x-real-ip
         - user.name
-        - http.useragent
+        - http_request.headers.user-agent
         - msg
       ':hydra:':
         - soc_timestamp
         - event.dataset
-        - http.request.headers.x-real-ip
+        - http_request.headers.x-real-ip
         - user.name
-        - http.useragent
+        - http_request.headers.user-agent
         - msg
       '::conn':
         - soc_timestamp
@@ -1747,7 +1747,7 @@ soc:
               showSubtitle: true
             - name: SOC - Auth
               description: Users authenticated to SOC grouped by IP address and identity
-              query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby http.request.headers.x-real-ip user.name'
+              query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby http_request.headers.x-real-ip user.name'
               showSubtitle: true
             - name: SOC - App
               description: Logs generated by the Security Onion Console (SOC) server and modules
@@ -2027,10 +2027,10 @@ soc:
               query: '* | groupby event.category | groupby -sankey event.category event.module | groupby event.module | groupby -sankey event.module event.dataset | groupby event.dataset | groupby observer.name | groupby host.name | groupby source.ip | groupby destination.ip | groupby destination.port'
             - name: SOC Logins
               description: SOC (Security Onion Console) logins
-              query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby http.request.headers.x-real-ip | groupby -sankey http.request.headers.x-real-ip user.name | groupby user.name | groupby http.useragent'
+              query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby http_request.headers.x-real-ip | groupby -sankey http_request.headers.x-real-ip user.name | groupby user.name | groupby http_request.headers.user-agent'
             - name: SOC Login Failures
               description: SOC (Security Onion Console) login failures
-              query: 'event.dataset:kratos.audit AND msg:*Encountered*self-service*login*error* | groupby user.name | groupby http.request.headers.x-real-ip | groupby -sankey http.request.headers.x-real-ip http.useragent | groupby http.useragent'
+              query: 'event.dataset:kratos.audit AND msg:*Encountered*self-service*login*error* | groupby user.name | groupby http_request.headers.x-real-ip | groupby -sankey http_request.headers.x-real-ip http_request.headers.user-agent | groupby http_request.headers.user-agent'
             - name: Alerts
               description: Overview of all alerts
               query: 'tags:alert | groupby event.module* | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby rule.name | groupby event.severity | groupby destination.as.organization.name'

salt/soc/soc_soc.yaml

@@ -652,6 +652,7 @@ soc:
         assistant:
           apiUrl:
             description: The URL of the AI gateway.
+            advanced: True
             global: True
           healthTimeoutSeconds:
             description: Timeout in seconds for the Onion AI health check.