From 5d0c187497a0cb24fdb1f6e3d99fea0a1482280c Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 23 Jan 2026 14:45:31 -0600 Subject: [PATCH] format json --- salt/elasticsearch/files/ingest/global@custom | 228 +++++++++++++++--- 1 file changed, 198 insertions(+), 30 deletions(-) diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom index 8e48eb0b9..6bf36d1a3 100644 --- a/salt/elasticsearch/files/ingest/global@custom +++ b/salt/elasticsearch/files/ingest/global@custom @@ -1,31 +1,199 @@ { - "version": 3, - "_meta": { - "managed_by": "securityonion", - "managed": true - }, - "description": "Custom pipeline for processing all incoming Fleet Agent documents. \n", - "processors": [ - { "set": { "ignore_failure": true, "field": "event.module", "value": "elastic_agent" } }, - { "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp" } }, - { "split": { "if": "ctx.data_stream?.dataset != null && ctx.data_stream?.dataset.contains('.')", "field":"data_stream.dataset", "separator":"\\.", "target_field":"datastream_dataset_temp", "ignore_missing":true } }, - { "set": { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } }, - { "set": { "if": "ctx.datastream_dataset_temp != null && ctx.datastream_dataset_temp[0] == 'network_traffic'", "field":"event.module", "value":"{{ datastream_dataset_temp.0 }}", "ignore_failure":true, "ignore_empty_value":true, "description":"Fix EA network packet capture" } }, - { "gsub": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } }, - { "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}", "allow_duplicates": false } }, - { "set": { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } }, - { "set": { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } }, - { "set": { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } }, - { "set": { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } }, - { "set": { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.dataset", "value": "import" } }, - { "set": { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.namespace", "value": "so" } }, - { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } }, - { "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } }, - { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, - { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, - { "set": { "if": "ctx.event?.dataset != null && ctx.event?.dataset == 'elasticsearch.server'", "field": "event.module", "value":"elasticsearch" }}, - {"append": {"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"if":"ctx?.event?.dataset == 'endpoint.events.network' && ctx?.source?.ip != null","ignore_failure":true}}, - {"foreach": {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"if":"ctx?.event?.module == 'endpoint' && ctx?.host?.ip != null","ignore_missing":true, "description":"Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"}}, - { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp", "datastream_dataset_temp" ], "ignore_missing": true, "ignore_failure": true } } - ] -} + "version": 3, + "_meta": { + "managed_by": "securityonion", + "managed": true + }, + "description": "Custom pipeline for processing all incoming Fleet Agent documents. \n", + "processors": [ + { + "set": { + "ignore_failure": true, + "field": "event.module", + "value": "elastic_agent" + } + }, + { + "split": { + "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", + "field": "event.dataset", + "separator": "\\.", + "target_field": "module_temp" + } + }, + { + "split": { + "if": "ctx.data_stream?.dataset != null && ctx.data_stream?.dataset.contains('.')", + "field": "data_stream.dataset", + "separator": "\\.", + "target_field": "datastream_dataset_temp", + "ignore_missing": true + } + }, + { + "set": { + "if": "ctx.module_temp != null", + "override": true, + "field": "event.module", + "value": "{{module_temp.0}}" + } + }, + { + "set": { + "if": "ctx.datastream_dataset_temp != null && ctx.datastream_dataset_temp[0] == 'network_traffic'", + "field": "event.module", + "value": "{{ datastream_dataset_temp.0 }}", + "ignore_failure": true, + "ignore_empty_value": true, + "description": "Fix EA network packet capture" + } + }, + { + "gsub": { + "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", + "field": "event.dataset", + "pattern": "^[^.]*.", + "replacement": "", + "target_field": "dataset_tag_temp" + } + }, + { + "append": { + "if": "ctx.dataset_tag_temp != null", + "field": "tags", + "value": "{{dataset_tag_temp}}", + "allow_duplicates": false + } + }, + { + "set": { + "if": "ctx.network?.direction == 'egress'", + "override": true, + "field": "network.initiated", + "value": "true" + } + }, + { + "set": { + "if": "ctx.network?.direction == 'ingress'", + "override": true, + "field": "network.initiated", + "value": "false" + } + }, + { + "set": { + "if": "ctx.network?.type == 'ipv4'", + "override": true, + "field": "destination.ipv6", + "value": "false" + } + }, + { + "set": { + "if": "ctx.network?.type == 'ipv6'", + "override": true, + "field": "destination.ipv6", + "value": "true" + } + }, + { + "set": { + "if": "ctx.tags != null && ctx.tags.contains('import')", + "override": true, + "field": "data_stream.dataset", + "value": "import" + } + }, + { + "set": { + "if": "ctx.tags != null && ctx.tags.contains('import')", + "override": true, + "field": "data_stream.namespace", + "value": "so" + } + }, + { + "community_id": { + "if": "ctx.event?.dataset == 'endpoint.events.network'", + "ignore_failure": true + } + }, + { + "set": { + "if": "ctx.event?.module == 'fim'", + "override": true, + "field": "event.module", + "value": "file_integrity" + } + }, + { + "rename": { + "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", + "ignore_missing": true, + "field": "winlog.event_data.Threat Name", + "target_field": "winlog.event_data.threat_name" + } + }, + { + "set": { + "if": "ctx?.metadata?.kafka != null", + "field": "kafka.id", + "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", + "ignore_failure": true + } + }, + { + "set": { + "if": "ctx.event?.dataset != null && ctx.event?.dataset == 'elasticsearch.server'", + "field": "event.module", + "value": "elasticsearch" + } + }, + { + "append": { + "field": "related.ip", + "value": [ + "{{source.ip}}", + "{{destination.ip}}" + ], + "allow_duplicates": false, + "if": "ctx?.event?.dataset == 'endpoint.events.network' && ctx?.source?.ip != null", + "ignore_failure": true + } + }, + { + "foreach": { + "field": "host.ip", + "processor": { + "append": { + "field": "related.ip", + "value": "{{_ingest._value}}", + "allow_duplicates": false + } + }, + "if": "ctx?.event?.module == 'endpoint' && ctx?.host?.ip != null", + "ignore_missing": true, + "description": "Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip" + } + }, + { + "remove": { + "field": [ + "message2", + "type", + "fields", + "category", + "module", + "dataset", + "event.dataset_temp", + "dataset_tag_temp", + "module_temp", + "datastream_dataset_temp" + ], + "ignore_missing": true, + "ignore_failure": true + } + } + ] +} \ No newline at end of file