Merge pull request #15353 from Security-Onion-Solutions/reyesj2/kratos

update kratos index template

salt/elasticsearch/defaults.yaml

@@ -857,53 +857,11 @@ elasticsearch:
         composed_of:
         - agent-mappings
         - dtc-agent-mappings
-        - base-mappings
-        - dtc-base-mappings
-        - client-mappings
-        - dtc-client-mappings
-        - container-mappings
-        - destination-mappings
-        - dtc-destination-mappings
-        - pb-override-destination-mappings
-        - dll-mappings
-        - dns-mappings
-        - dtc-dns-mappings
-        - ecs-mappings
-        - dtc-ecs-mappings
-        - error-mappings
-        - event-mappings
-        - dtc-event-mappings
-        - file-mappings
-        - dtc-file-mappings
-        - group-mappings
         - host-mappings
         - dtc-host-mappings
         - http-mappings
         - dtc-http-mappings
-        - log-mappings
         - metadata-mappings
-        - network-mappings
-        - dtc-network-mappings
-        - observer-mappings
-        - dtc-observer-mappings
-        - organization-mappings
-        - package-mappings
-        - process-mappings
-        - dtc-process-mappings
-        - related-mappings
-        - rule-mappings
-        - dtc-rule-mappings
-        - server-mappings
-        - service-mappings
-        - dtc-service-mappings
-        - source-mappings
-        - dtc-source-mappings
-        - pb-override-source-mappings
-        - threat-mappings
-        - tls-mappings
-        - url-mappings
-        - user_agent-mappings
-        - dtc-user_agent-mappings
         - common-settings
         - common-dynamic-mappings
         data_stream:

salt/elasticsearch/files/ingest/kratos

@@ -1,9 +1,90 @@
 {
-  "description" : "kratos",
-  "processors" : [
-    {"set":{"field":"audience","value":"access","override":false,"ignore_failure":true}},
-    {"set":{"field":"event.dataset","ignore_empty_value":true,"ignore_failure":true,"value":"kratos.{{{audience}}}","media_type":"text/plain"}},
-    {"set":{"field":"event.action","ignore_failure":true,"copy_from":"msg"    }},
-    { "pipeline":    { "name": "common" } }
-  ]
+    "description": "kratos",
+    "processors": [
+        {
+            "set": {
+                "field": "audience",
+                "value": "access",
+                "override": false,
+                "ignore_failure": true
+            }
+        },
+        {
+            "set": {
+                "field": "event.dataset",
+                "ignore_empty_value": true,
+                "ignore_failure": true,
+                "value": "kratos.{{{audience}}}",
+                "media_type": "text/plain"
+            }
+        },
+        {
+            "set": {
+                "field": "event.action",
+                "ignore_failure": true,
+                "copy_from": "msg"
+            }
+        },
+        {
+            "rename": {
+                "field": "http_request",
+                "target_field": "http.request",
+                "ignore_failure": true,
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "http_response",
+                "target_field": "http.response",
+                "ignore_failure": true,
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "http.request.path",
+                "target_field": "http.uri",
+                "ignore_failure": true,
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "http.request.method",
+                "target_field": "http.method",
+                "ignore_failure": true,
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "http.request.method",
+                "target_field": "http.method",
+                "ignore_failure": true,
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "http.request.query",
+                "target_field": "http.query",
+                "ignore_failure": true,
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "http.request.headers.user-agent",
+                "target_field": "http.useragent",
+                "ignore_failure": true,
+                "ignore_missing": true
+            }
+        },
+        {
+            "pipeline": {
+                "name": "common"
+            }
+        }
+    ]
 }

salt/manager/tools/sbin/soup

@@ -427,6 +427,7 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.4.170 ]] && up_to_2.4.180
     [[ "$INSTALLEDVERSION" == 2.4.180 ]] && up_to_2.4.190
     [[ "$INSTALLEDVERSION" == 2.4.190 ]] && up_to_2.4.200
+    [[ "$INSTALLEDVERSION" == 2.4.200 ]] && up_to_2.4.210
     true
 }
 
@@ -459,6 +460,7 @@ postupgrade_changes() {
     [[ "$POSTVERSION"  == 2.4.170 ]] && post_to_2.4.180
     [[ "$POSTVERSION"  == 2.4.180 ]] && post_to_2.4.190
     [[ "$POSTVERSION"  == 2.4.190 ]] && post_to_2.4.200
+    [[ "$POSTVERSION"  == 2.4.200 ]] && post_to_2.4.210
     true
 }
 
@@ -645,6 +647,14 @@ post_to_2.4.200() {
   POSTVERSION=2.4.200
 }
 
+post_to_2.4.210() {
+  echo "Rolling over Kratos index to apply new index template"
+
+  rollover_index "logs-kratos-so"
+
+  POSTVERSION=2.4.210
+}
+
 repo_sync() {
   echo "Sync the local repo."
   su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
@@ -921,6 +931,12 @@ up_to_2.4.200() {
   INSTALLEDVERSION=2.4.200
 }
 
+up_to_2.4.210() {
+  echo "Nothing to do for 2.4.210"
+
+  INSTALLEDVERSION=2.4.210
+}
+
 add_hydra_pillars() {
   mkdir -p /opt/so/saltstack/local/pillar/hydra
   touch /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls

salt/soc/defaults.yaml

@@ -115,16 +115,16 @@ soc:
       ':kratos:':
         - soc_timestamp
         - event.dataset
-        - http_request.headers.x-real-ip
+        - http.request.headers.x-real-ip
         - user.name
-        - http_request.headers.user-agent
+        - http.useragent
         - msg
       ':hydra:':
         - soc_timestamp
         - event.dataset
-        - http_request.headers.x-real-ip
+        - http.request.headers.x-real-ip
         - user.name
-        - http_request.headers.user-agent
+        - http.useragent
         - msg
       '::conn':
         - soc_timestamp
@@ -1747,7 +1747,7 @@ soc:
               showSubtitle: true
             - name: SOC - Auth
               description: Users authenticated to SOC grouped by IP address and identity
-              query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby http_request.headers.x-real-ip user.name'
+              query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby http.request.headers.x-real-ip user.name'
               showSubtitle: true
             - name: SOC - App
               description: Logs generated by the Security Onion Console (SOC) server and modules
@@ -2027,10 +2027,10 @@ soc:
               query: '* | groupby event.category | groupby -sankey event.category event.module | groupby event.module | groupby -sankey event.module event.dataset | groupby event.dataset | groupby observer.name | groupby host.name | groupby source.ip | groupby destination.ip | groupby destination.port'
             - name: SOC Logins
               description: SOC (Security Onion Console) logins
-              query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby http_request.headers.x-real-ip | groupby -sankey http_request.headers.x-real-ip user.name | groupby user.name | groupby http_request.headers.user-agent'
+              query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby http.request.headers.x-real-ip | groupby -sankey http.request.headers.x-real-ip user.name | groupby user.name | groupby http.useragent'
             - name: SOC Login Failures
               description: SOC (Security Onion Console) login failures
-              query: 'event.dataset:kratos.audit AND msg:*Encountered*self-service*login*error* | groupby user.name | groupby http_request.headers.x-real-ip | groupby -sankey http_request.headers.x-real-ip http_request.headers.user-agent | groupby http_request.headers.user-agent'
+              query: 'event.dataset:kratos.audit AND msg:*Encountered*self-service*login*error* | groupby user.name | groupby http.request.headers.x-real-ip | groupby -sankey http.request.headers.x-real-ip http.useragent | groupby http.useragent'
             - name: Alerts
               description: Overview of all alerts
               query: 'tags:alert | groupby event.module* | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby rule.name | groupby event.severity | groupby destination.as.organization.name'