From 9022dc24fb11bd061fc345d328af04f08768d043 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 1 May 2025 08:19:08 -0400 Subject: [PATCH] Add Parsing for Playbooks --- salt/elasticsearch/files/ingest/global@custom | 4 +- .../elasticsearch/files/ingest/suricata.alert | 3 +- .../files/ingest/suricata.common | 7 ++ salt/soc/files/soc/sigma_so_pipeline.yaml | 97 ++++++++++++++++++- 4 files changed, 106 insertions(+), 5 deletions(-) diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom index 993c08373..54b96be18 100644 --- a/salt/elasticsearch/files/ingest/global@custom +++ b/salt/elasticsearch/files/ingest/global@custom @@ -22,8 +22,10 @@ { "date": { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } }, { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } }, { "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } }, - { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, + { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, + {"append": {"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"if":"ctx?.event?.dataset == 'endpoint.events.network'","ignore_failure":true}}, + {"foreach": {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"ignore_failure":true}}, { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp", "datastream_dataset_temp" ], "ignore_missing": true, "ignore_failure": true } } ] } diff --git a/salt/elasticsearch/files/ingest/suricata.alert b/salt/elasticsearch/files/ingest/suricata.alert index 9dd97e190..afdc890dd 100644 --- a/salt/elasticsearch/files/ingest/suricata.alert +++ b/salt/elasticsearch/files/ingest/suricata.alert @@ -9,6 +9,7 @@ { "rename":{ "field": "rule.signature_id", "target_field": "rule.uuid", "ignore_failure": true } }, { "rename":{ "field": "rule.signature_id", "target_field": "rule.signature", "ignore_failure": true } }, { "rename":{ "field": "message2.payload_printable", "target_field": "network.data.decoded", "ignore_failure": true } }, + { "dissect": { "field": "rule.rule", "pattern": "%{?prefix}content:\"%{dns.query_name}\"%{?remainder}", "ignore_missing": true, "tag": "suricata" } }, { "pipeline": { "name": "common.nids" } } ] -} +} \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/suricata.common b/salt/elasticsearch/files/ingest/suricata.common index 8143882c7..578fad17e 100644 --- a/salt/elasticsearch/files/ingest/suricata.common +++ b/salt/elasticsearch/files/ingest/suricata.common @@ -18,6 +18,13 @@ { "set": { "field": "event.ingested", "value": "{{@timestamp}}" } }, { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } }, { "remove":{ "field": "agent", "ignore_failure": true } }, + {"append":{"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"ignore_failure":true}}, + { + "script": { + "source": "boolean isPrivate(def ip) { if (ip == null) return false; if (ip.startsWith('10.')) return true; if (ip.startsWith('192.168.')) return true; if (ip.startsWith('172.')) { String[] parts = ip.split('\\\\.'); if (parts.length > 1) { int second = Integer.parseInt(parts[1]); if (second >= 16 && second <= 31) return true; } } return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) { def field = fields[i]; def ip = ctx.containsKey(field) && ctx[field].containsKey('ip') ? ctx[field].ip : null; if (ip != null) { if (ctx.network == null) ctx.network = new HashMap(); String netField = isPrivate(ip) ? \"private\" : \"public\"; if (!ctx.network.containsKey(netField)) { ctx.network[netField] = new HashMap(); } if (!ctx.network[netField].containsKey(\"ip\")) { ctx.network[netField].ip = new ArrayList(); } if (!ctx.network[netField].ip.contains(ip)) { ctx.network[netField].ip.add(ip); } } }", + "ignore_failure": true + } + }, { "pipeline": { "if": "ctx?.event?.dataset != null", "name": "suricata.{{event.dataset}}" } } ] } diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index 48e9e1215..88abcc200 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -1,6 +1,45 @@ name: Security Onion Baseline Pipeline priority: 90 transformations: +vars: + document_id: + - '{soc_id}' + hostname: + - '{event_data.host.name}' + ProcessGuid: + - '{event_data.process.entity_id}' + User: + - '{user.name}' + private_ip: + - '{network.private.ip}' + public_ip: + - '{network.public.ip}' + related_ip: + - '{event_data.related.ip}' + related.hosts: + - '{event_data.related.hosts' + CurrentDirectory: + - '{event_data.process.working_directory}' + ParentProcessGuid: + - '{ParentProcessGuid}' + Image: + - '{process.executable}' + community_id: + - '{network.community_id}' +transformations: + - type: value_placeholders + include: + - 'community_id' + - 'document_id' + - 'ProcessGuid' + - 'hostname' + - 'User' + - 'CurrentDirectory' + - 'ParentProcessGuid' + - 'Image' + - 'related_ip' + - 'private_ip' + - 'public_ip' - id: baseline_field_name_mapping type: field_name_mapping mapping: @@ -12,12 +51,27 @@ transformations: sid: rule.uuid answer: answers query: dns.query.name - src_ip: source.ip.keyword + src_ip: source.ip src_port: source.port - dst_ip: destination.ip.keyword + dst_ip: destination.ip dst_port: destination.port winlog.event_data.User: user.name logtype: event.code # OpenCanary + ## Start Temp Linux Mappings ## + ProcessGuid: process.entity_id + ProcessId: process.pid + Image: process.executable + CommandLine: process.command_line + CurrentDirectory: process.working_directory + ParentProcessGuid: process.parent.entity_id + ParentProcessId: process.parent.pid" + ParentImage: process.parent.executable + ParentCommandLine: process.parent.command_line + ## End Temp Linux Mappings ## + rule.type: event.module + related_ip: related.ip + community_id: network.community_id + event_dataset: event.dataset # Maps "opencanary" product to SO IDH logs - id: opencanary_idh_add-fields type: add_condition @@ -126,4 +180,41 @@ transformations: event.type: 'creation' rule_conditions: - type: logsource - category: file_event \ No newline at end of file + category: file_event + category: file_event + # Maps network_connection rules to endpoint network creation logs + # This is an OS-agnostic mapping, to account for logs that don't specify source OS + - id: endpoint_network_connection_add-fields + type: add_condition + conditions: + event.category: 'network' + event.type: 'start' + rule_conditions: + - type: logsource + category: network_connection + # Maps "alert" category to SO Alert events + - id: alert_so_add-fields + type: add_condition + conditions: + tags: 'alert' + rule_conditions: + - type: logsource + category: alert + # Maps "network + connection" to SO connection logs + - id: network_connection_so_add-fields + type: add_condition + conditions: + tags: 'conn' + rule_conditions: + - type: logsource + category: network + service: connection + # Maps "network + dns" to SO DNS logs + - id: network_dns_so_add-fields + type: add_condition + conditions: + tags: 'dns' + rule_conditions: + - type: logsource + category: network + service: dns