Merge pull request #10819 from Security-Onion-Solutions/fix/elasticsearch_endpoint

Add endpoint to defaults

salt/elasticfleet/defaults.yaml

@@ -28,6 +28,7 @@ elasticfleet:
     - aws
     - azure
     - cloudflare
+    - endpoint
     - fim
     - github
     - google_workspace

salt/elasticsearch/defaults.yaml

@@ -81,6 +81,8 @@ elasticsearch:
               managed: true
         composed_of:
           - "so-data-streams-mappings"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
           - "so-logs-mappings"
           - "so-logs-settings"
         priority: 225
@@ -1312,6 +1314,398 @@ elasticsearch:
             name: elastic_agent
           managed_by: security_onion
           managed: true
+    so-logs-endpoint.alerts:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-endpoint.alerts-*"
+        template:
+          settings:
+            index:
+              number_of_replicas: 0
+              mapping:
+                total_fields:
+                  limit: 5000
+              sort:
+                field: "@timestamp"
+                order: desc
+        composed_of:
+          - "event-mappings"
+          - "logs-endpoint.alerts@custom"
+          - "logs-endpoint.alerts@package"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
+        _meta:
+          package:
+            name: elastic_agent
+          managed_by: security_onion
+          managed: true
+    so-logs-endpoint.events.api:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-endpoint.events.api-*"
+        template:
+          settings:
+            index:
+              number_of_replicas: 0
+              mapping:
+                total_fields:
+                  limit: 5000
+              sort:
+                field: "@timestamp"
+                order: desc
+        composed_of:
+          - "event-mappings"
+          - "logs-endpoint.events.api@custom"
+          - "logs-endpoint.events.api@package"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
+        _meta:
+          package:
+            name: elastic_agent
+          managed_by: security_onion
+          managed: true
+    so-logs-endpoint.events.file:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-endpoint.events.file-*"
+        template:
+          settings:
+            index:
+              number_of_replicas: 0
+              mapping:
+                total_fields:
+                  limit: 5000
+              sort:
+                field: "@timestamp"
+                order: desc
+        composed_of:
+          - "event-mappings"
+          - "logs-endpoint.events.file@custom"
+          - "logs-endpoint.events.file@package"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
+        _meta:
+          package:
+            name: elastic_agent
+          managed_by: security_onion
+          managed: true
+    so-logs-endpoint.events.library:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-endpoint.events.library-*"
+        template:
+          settings:
+            index:
+              number_of_replicas: 0
+              mapping:
+                total_fields:
+                  limit: 5000
+              sort:
+                field: "@timestamp"
+                order: desc
+        composed_of:
+          - "event-mappings"
+          - "logs-endpoint.events.library@custom"
+          - "logs-endpoint.events.library@package"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
+        _meta:
+          package:
+            name: elastic_agent
+          managed_by: security_onion
+          managed: true
+    so-logs-endpoint.events.network:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-endpoint.events.network-*"
+        template:
+          settings:
+            index:
+              number_of_replicas: 0
+              mapping:
+                total_fields:
+                  limit: 5000
+              sort:
+                field: "@timestamp"
+                order: desc
+        composed_of:
+          - "event-mappings"
+          - "logs-endpoint.events.network@custom"
+          - "logs-endpoint.events.network@package"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
+        _meta:
+          package:
+            name: elastic_agent
+          managed_by: security_onion
+          managed: true
+    so-logs-endpoint.events.process:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-endpoint.events.process-*"
+        template:
+          settings:
+            index:
+              number_of_replicas: 0
+              mapping:
+                total_fields:
+                  limit: 5000
+              sort:
+                field: "@timestamp"
+                order: desc
+        composed_of:
+          - "event-mappings"
+          - "logs-endpoint.events.process@custom"
+          - "logs-endpoint.events.process@package"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
+        _meta:
+          package:
+            name: elastic_agent
+          managed_by: security_onion
+          managed: true
+    so-logs-endpoint.events.registry:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-endpoint.events.registry-*"
+        template:
+          settings:
+            index:
+              number_of_replicas: 0
+              mapping:
+                total_fields:
+                  limit: 5000
+              sort:
+                field: "@timestamp"
+                order: desc
+        composed_of:
+          - "event-mappings"
+          - "logs-endpoint.events.registry@custom"
+          - "logs-endpoint.events.registry@package"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
+        _meta:
+          package:
+            name: elastic_agent
+          managed_by: security_onion
+          managed: true
+    so-logs-endpoint.events.security:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-endpoint.events.security-*"
+        template:
+          settings:
+            index:
+              number_of_replicas: 0
+              mapping:
+                total_fields:
+                  limit: 5000
+              sort:
+                field: "@timestamp"
+                order: desc
+        composed_of:
+          - "event-mappings"
+          - "logs-endpoint.events.security@custom"
+          - "logs-endpoint.events.security@package"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              set_priority:
+                priority: 100
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+          cold:
+            min_age: 30d
+            actions:
+              set_priority:
+                priority: 0
+          delete:
+            min_age: 365d
+            actions:
+              delete: {}
+        _meta:
+          package:
+            name: elastic_agent
+          managed_by: security_onion
+          managed: true
     so-logs-elastic_agent.filebeat:
       index_sorting: False
       index_template:

salt/elasticsearch/files/ingest/.fleet_final_pipeline-1

@@ -72,8 +72,8 @@
     { "set":        { "ignore_failure": true,  "field": "event.module", "value": "elastic_agent" } },
     { "split":      { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp"  } },
     { "set":        { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } },
-    { "split":      { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "dataset_tag_temp" } },
+    { "gsub":       { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } },
-    { "append":        { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}"  } },
+    { "append":     { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}"  } },
     { "set":        { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } },
     { "set":        { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } },
     { "set":        { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } },

salt/elasticsearch/templates/component/elastic-agent/so-fleet_agent_id_verification-1.json

@@ -9,6 +9,10 @@
       "properties": {
         "event": {
           "properties": {
+            "agent_id": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
             "agent_id_status": {
               "ignore_above": 1024,
               "type": "keyword"
@@ -18,6 +22,10 @@
               "type": "date"
             }
           }
+        },
+        "tags": {
+          "ignore_above": 1024,
+          "type": "keyword"
         }
       }
     }