From 433dab7376dc02d79f628102ecd5c010a5632153 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 20 Nov 2025 14:16:10 -0600 Subject: [PATCH 1/3] format json --- .../files/ingest/suricata.common | 176 +++++++++++++++--- 1 file changed, 147 insertions(+), 29 deletions(-) diff --git a/salt/elasticsearch/files/ingest/suricata.common b/salt/elasticsearch/files/ingest/suricata.common index 102b5dac8..5af35dc37 100644 --- a/salt/elasticsearch/files/ingest/suricata.common +++ b/salt/elasticsearch/files/ingest/suricata.common @@ -1,30 +1,148 @@ { - "description" : "suricata.common", - "processors" : [ - { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, - { "rename": { "field": "message2.pkt_src", "target_field": "network.packet_source","ignore_failure": true } }, - { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_failure": true } }, - { "rename": { "field": "message2.in_iface", "target_field": "observer.ingress.interface.name", "ignore_failure": true } }, - { "rename": { "field": "message2.flow_id", "target_field": "log.id.uid", "ignore_failure": true } }, - { "rename": { "field": "message2.src_ip", "target_field": "source.ip", "ignore_failure": true } }, - { "rename": { "field": "message2.src_port", "target_field": "source.port", "ignore_failure": true } }, - { "rename": { "field": "message2.dest_ip", "target_field": "destination.ip", "ignore_failure": true } }, - { "rename": { "field": "message2.dest_port", "target_field": "destination.port", "ignore_failure": true } }, - { "rename": { "field": "message2.vlan", "target_field": "network.vlan.id", "ignore_failure": true } }, - { "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_missing": true } }, - { "rename": { "field": "message2.xff", "target_field": "xff.ip", "ignore_missing": true } }, - { "set": { "field": "event.dataset", "value": "{{ message2.event_type }}" } }, - { "set": { "field": "observer.name", "value": "{{agent.name}}" } }, - { "set": { "field": "event.ingested", "value": "{{@timestamp}}" } }, - { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } }, - { "remove":{ "field": "agent", "ignore_failure": true } }, - {"append":{"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"ignore_failure":true}}, - { - "script": { - "source": "boolean isPrivate(def ip) { if (ip == null) return false; int dot1 = ip.indexOf('.'); if (dot1 == -1) return false; int dot2 = ip.indexOf('.', dot1 + 1); if (dot2 == -1) return false; int first = Integer.parseInt(ip.substring(0, dot1)); if (first == 10) return true; if (first == 192 && ip.startsWith('168.', dot1 + 1)) return true; if (first == 172) { int second = Integer.parseInt(ip.substring(dot1 + 1, dot2)); return second >= 16 && second <= 31; } return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) { def field = fields[i]; def ip = ctx[field]?.ip; if (ip != null) { if (ctx.network == null) ctx.network = new HashMap(); if (isPrivate(ip)) { if (ctx.network.private_ip == null) ctx.network.private_ip = new ArrayList(); if (!ctx.network.private_ip.contains(ip)) ctx.network.private_ip.add(ip); } else { if (ctx.network.public_ip == null) ctx.network.public_ip = new ArrayList(); if (!ctx.network.public_ip.contains(ip)) ctx.network.public_ip.add(ip); } } }", - "ignore_failure": false - } - }, - { "pipeline": { "if": "ctx?.event?.dataset != null", "name": "suricata.{{event.dataset}}" } } - ] -} + "description": "suricata.common", + "processors": [ + { + "json": { + "field": "message", + "target_field": "message2", + "ignore_failure": true + } + }, + { + "rename": { + "field": "message2.pkt_src", + "target_field": "network.packet_source", + "ignore_failure": true + } + }, + { + "rename": { + "field": "message2.proto", + "target_field": "network.transport", + "ignore_failure": true + } + }, + { + "rename": { + "field": "message2.in_iface", + "target_field": "observer.ingress.interface.name", + "ignore_failure": true + } + }, + { + "rename": { + "field": "message2.flow_id", + "target_field": "log.id.uid", + "ignore_failure": true + } + }, + { + "rename": { + "field": "message2.src_ip", + "target_field": "source.ip", + "ignore_failure": true + } + }, + { + "rename": { + "field": "message2.src_port", + "target_field": "source.port", + "ignore_failure": true + } + }, + { + "rename": { + "field": "message2.dest_ip", + "target_field": "destination.ip", + "ignore_failure": true + } + }, + { + "rename": { + "field": "message2.dest_port", + "target_field": "destination.port", + "ignore_failure": true + } + }, + { + "rename": { + "field": "message2.vlan", + "target_field": "network.vlan.id", + "ignore_failure": true + } + }, + { + "rename": { + "field": "message2.community_id", + "target_field": "network.community_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "message2.xff", + "target_field": "xff.ip", + "ignore_missing": true + } + }, + { + "set": { + "field": "event.dataset", + "value": "{{ message2.event_type }}" + } + }, + { + "set": { + "field": "observer.name", + "value": "{{agent.name}}" + } + }, + { + "set": { + "field": "event.ingested", + "value": "{{@timestamp}}" + } + }, + { + "date": { + "field": "message2.timestamp", + "target_field": "@timestamp", + "formats": [ + "ISO8601", + "UNIX" + ], + "timezone": "UTC", + "ignore_failure": true + } + }, + { + "remove": { + "field": "agent", + "ignore_failure": true + } + }, + { + "append": { + "field": "related.ip", + "value": [ + "{{source.ip}}", + "{{destination.ip}}" + ], + "allow_duplicates": false, + "ignore_failure": true + } + }, + { + "script": { + "source": "boolean isPrivate(def ip) { if (ip == null) return false; int dot1 = ip.indexOf('.'); if (dot1 == -1) return false; int dot2 = ip.indexOf('.', dot1 + 1); if (dot2 == -1) return false; int first = Integer.parseInt(ip.substring(0, dot1)); if (first == 10) return true; if (first == 192 && ip.startsWith('168.', dot1 + 1)) return true; if (first == 172) { int second = Integer.parseInt(ip.substring(dot1 + 1, dot2)); return second >= 16 && second <= 31; } return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) { def field = fields[i]; def ip = ctx[field]?.ip; if (ip != null) { if (ctx.network == null) ctx.network = new HashMap(); if (isPrivate(ip)) { if (ctx.network.private_ip == null) ctx.network.private_ip = new ArrayList(); if (!ctx.network.private_ip.contains(ip)) ctx.network.private_ip.add(ip); } else { if (ctx.network.public_ip == null) ctx.network.public_ip = new ArrayList(); if (!ctx.network.public_ip.contains(ip)) ctx.network.public_ip.add(ip); } } }", + "ignore_failure": false + } + }, + { + "pipeline": { + "if": "ctx?.event?.dataset != null", + "name": "suricata.{{event.dataset}}" + } + } + ] +} \ No newline at end of file From 6f42ff34422ba1bf7e8c560a1cd7fb7da1fe63c9 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 20 Nov 2025 14:16:49 -0600 Subject: [PATCH 2/3] suricata capture_file --- salt/elasticsearch/files/ingest/suricata.common | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/elasticsearch/files/ingest/suricata.common b/salt/elasticsearch/files/ingest/suricata.common index 5af35dc37..7b2dc7eeb 100644 --- a/salt/elasticsearch/files/ingest/suricata.common +++ b/salt/elasticsearch/files/ingest/suricata.common @@ -138,6 +138,13 @@ "ignore_failure": false } }, + { + "rename": { + "field": "message2.capture_file", + "target_field": "suricata.capture_file", + "ignore_missing": true + } + }, { "pipeline": { "if": "ctx?.event?.dataset != null", From c5db7c87525c1670a109e2e8a91bdc7587f72cb8 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 20 Nov 2025 14:26:12 -0600 Subject: [PATCH 3/3] suricata.capture_file keyword --- salt/elasticsearch/templates/component/ecs/suricata.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/elasticsearch/templates/component/ecs/suricata.json b/salt/elasticsearch/templates/component/ecs/suricata.json index 1eb06d266..3f393ff6a 100644 --- a/salt/elasticsearch/templates/component/ecs/suricata.json +++ b/salt/elasticsearch/templates/component/ecs/suricata.json @@ -841,6 +841,10 @@ "type": "long" } } + }, + "capture_file": { + "type": "keyword", + "ignore_above": 1024 } } }