From de7b7ff98909ca1964a85d94d695bf8c6917642e Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 24 Jul 2023 18:35:02 +0000 Subject: [PATCH 1/4] Add endpoint --- salt/elasticfleet/defaults.yaml | 1 + salt/elasticsearch/defaults.yaml | 392 +++++++++++++++++++++++++++++++ 2 files changed, 393 insertions(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 93b5eba9a..46d496955 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -28,6 +28,7 @@ elasticfleet: - aws - azure - cloudflare + - endpoint - fim - github - google_workspace diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 3f29483e0..06e51cb1d 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1312,6 +1312,398 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true + so-logs-endpoint.alerts: + index_sorting: False + index_template: + index_patterns: + - "logs-endpoint.alerts-*" + template: + settings: + index: + number_of_replicas: 0 + mapping: + total_fields: + limit: 5000 + sort: + field: "@timestamp" + order: desc + composed_of: + - "event-mappings" + - "logs-endpoint.alerts@custom" + - "logs-endpoint.alerts@package" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} + _meta: + package: + name: elastic_agent + managed_by: security_onion + managed: true + so-logs-endpoint.events.api: + index_sorting: False + index_template: + index_patterns: + - "logs-endpoint.events.api-*" + template: + settings: + index: + number_of_replicas: 0 + mapping: + total_fields: + limit: 5000 + sort: + field: "@timestamp" + order: desc + composed_of: + - "event-mappings" + - "logs-endpoint.events.api@custom" + - "logs-endpoint.events.api@package" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} + _meta: + package: + name: elastic_agent + managed_by: security_onion + managed: true + so-logs-endpoint.events.file: + index_sorting: False + index_template: + index_patterns: + - "logs-endpoint.events.file-*" + template: + settings: + index: + number_of_replicas: 0 + mapping: + total_fields: + limit: 5000 + sort: + field: "@timestamp" + order: desc + composed_of: + - "event-mappings" + - "logs-endpoint.events.file@custom" + - "logs-endpoint.events.file@package" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} + _meta: + package: + name: elastic_agent + managed_by: security_onion + managed: true + so-logs-endpoint.events.library: + index_sorting: False + index_template: + index_patterns: + - "logs-endpoint.events.library-*" + template: + settings: + index: + number_of_replicas: 0 + mapping: + total_fields: + limit: 5000 + sort: + field: "@timestamp" + order: desc + composed_of: + - "event-mappings" + - "logs-endpoint.events.library@custom" + - "logs-endpoint.events.library@package" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} + _meta: + package: + name: elastic_agent + managed_by: security_onion + managed: true + so-logs-endpoint.events.network: + index_sorting: False + index_template: + index_patterns: + - "logs-endpoint.events.network-*" + template: + settings: + index: + number_of_replicas: 0 + mapping: + total_fields: + limit: 5000 + sort: + field: "@timestamp" + order: desc + composed_of: + - "event-mappings" + - "logs-endpoint.events.network@custom" + - "logs-endpoint.events.network@package" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} + _meta: + package: + name: elastic_agent + managed_by: security_onion + managed: true + so-logs-endpoint.events.process: + index_sorting: False + index_template: + index_patterns: + - "logs-endpoint.events.process-*" + template: + settings: + index: + number_of_replicas: 0 + mapping: + total_fields: + limit: 5000 + sort: + field: "@timestamp" + order: desc + composed_of: + - "event-mappings" + - "logs-endpoint.events.process@custom" + - "logs-endpoint.events.process@package" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} + _meta: + package: + name: elastic_agent + managed_by: security_onion + managed: true + so-logs-endpoint.events.registry: + index_sorting: False + index_template: + index_patterns: + - "logs-endpoint.events.registry-*" + template: + settings: + index: + number_of_replicas: 0 + mapping: + total_fields: + limit: 5000 + sort: + field: "@timestamp" + order: desc + composed_of: + - "event-mappings" + - "logs-endpoint.events.registry@custom" + - "logs-endpoint.events.registry@package" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} + _meta: + package: + name: elastic_agent + managed_by: security_onion + managed: true + so-logs-endpoint.events.security: + index_sorting: False + index_template: + index_patterns: + - "logs-endpoint.events.security-*" + template: + settings: + index: + number_of_replicas: 0 + mapping: + total_fields: + limit: 5000 + sort: + field: "@timestamp" + order: desc + composed_of: + - "event-mappings" + - "logs-endpoint.events.security@custom" + - "logs-endpoint.events.security@package" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} + _meta: + package: + name: elastic_agent + managed_by: security_onion + managed: true so-logs-elastic_agent.filebeat: index_sorting: False index_template: From d84dbf9535c6bb8a1149ecb58d7bc512bfad0a97 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 24 Jul 2023 18:53:52 +0000 Subject: [PATCH 2/4] Add fleet --- salt/elasticsearch/defaults.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 06e51cb1d..035079f54 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -81,6 +81,8 @@ elasticsearch: managed: true composed_of: - "so-data-streams-mappings" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" - "so-logs-mappings" - "so-logs-settings" priority: 225 From 4efc951eafad4999dd9d33892a0edd6e3a63f289 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 24 Jul 2023 20:57:39 +0000 Subject: [PATCH 3/4] Add tags --- .../elastic-agent/so-fleet_agent_id_verification-1.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_agent_id_verification-1.json b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_agent_id_verification-1.json index 18c6f1d6d..55e3e921c 100644 --- a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_agent_id_verification-1.json +++ b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_agent_id_verification-1.json @@ -9,6 +9,10 @@ "properties": { "event": { "properties": { + "agent_id": { + "ignore_above": 1024, + "type": "keyword" + }, "agent_id_status": { "ignore_above": 1024, "type": "keyword" @@ -18,6 +22,10 @@ "type": "date" } } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" } } } From 5553be02ac570890219c6f09159136f22480db66 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 24 Jul 2023 21:31:28 +0000 Subject: [PATCH 4/4] Change how tags are added --- salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 index d6f5b65aa..743e4181a 100644 --- a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 +++ b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 @@ -72,8 +72,8 @@ { "set": { "ignore_failure": true, "field": "event.module", "value": "elastic_agent" } }, { "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp" } }, { "set": { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } }, - { "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "dataset_tag_temp" } }, - { "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}" } }, + { "gsub": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } }, + { "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}" } }, { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } ], "on_failure": [