From cf0d56eee7cd772b0b0a954ef548494b5bd8c041 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 17 Sep 2024 19:24:31 +0000 Subject: [PATCH] Fix suricata alerts for opnsense and pfsense --- .../files/ingest/logs-pfsense.log-1.16.0 | 2 +- .../ingest/logs-pfsense.log-1.16.0-suricata | 7 +- .../files/ingest/logs-pfsense.log-1.19.1 | 414 ++++++++++++++++++ .../files/ingest/suricata.alert_pfsense | 16 + .../files/ingest/suricata.common_pfsense | 23 + 5 files changed, 460 insertions(+), 2 deletions(-) create mode 100644 salt/elasticsearch/files/ingest/logs-pfsense.log-1.19.1 create mode 100644 salt/elasticsearch/files/ingest/suricata.alert_pfsense create mode 100644 salt/elasticsearch/files/ingest/suricata.common_pfsense diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.16.0 b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.16.0 index af31e1518..f53abb0e3 100644 --- a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.16.0 +++ b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.16.0 @@ -1,5 +1,5 @@ { - "description": "Pipeline for pfSense", + "description": "Pipeline for PFsense", "processors": [ { "set": { diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.16.0-suricata b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.16.0-suricata index f3a14af44..9cb10c2bb 100644 --- a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.16.0-suricata +++ b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.16.0-suricata @@ -1,9 +1,14 @@ { "description": "Pipeline for parsing pfSense Suricata logs.", "processors": [ + { "set": { + "field": "event.module", + "value": "suricata" + } + }, { "pipeline": { - "name": "suricata.common" + "name": "suricata.common_pfsense" } } ], diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.19.1 b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.19.1 new file mode 100644 index 000000000..6166f6b55 --- /dev/null +++ b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.19.1 @@ -0,0 +1,414 @@ +{ + "description": "Pipeline for PFsense", + "processors": [ + { + "set": { + "field": "ecs.version", + "value": "8.11.0" + } + }, + { + "set": { + "field": "observer.vendor", + "value": "netgate" + } + }, + { + "set": { + "field": "observer.type", + "value": "firewall" + } + }, + { + "rename": { + "field": "message", + "target_field": "event.original", + "ignore_missing": true, + "if": "ctx.event?.original == null" + } + }, + { + "set": { + "field": "event.kind", + "value": "event" + } + }, + { + "set": { + "field": "event.timezone", + "value": "{{_tmp.tz_offset}}", + "if": "ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local'" + } + }, + { + "grok": { + "description": "Parse syslog header", + "field": "event.original", + "patterns": [ + "^(%{ECS_SYSLOG_PRI})?%{TIMESTAMP} %{GREEDYDATA:message}" + ], + "pattern_definitions": { + "ECS_SYSLOG_PRI": "<%{NONNEGINT:log.syslog.priority:long}>(\\d )?", + "TIMESTAMP": "(?:%{BSD_TIMESTAMP_FORMAT}|%{SYSLOG_TIMESTAMP_FORMAT})", + "BSD_TIMESTAMP_FORMAT": "%{SYSLOGTIMESTAMP:_tmp.timestamp}(%{SPACE}%{BSD_PROCNAME}|%{SPACE}%{OBSERVER}%{SPACE}%{BSD_PROCNAME})(\\[%{POSINT:process.pid:long}\\])?:", + "BSD_PROCNAME": "(?:\\b%{NAME:process.name}|\\(%{NAME:process.name}\\))", + "NAME": "[[[:alnum:]]_-]+", + "SYSLOG_TIMESTAMP_FORMAT": "%{TIMESTAMP_ISO8601:_tmp.timestamp8601}%{SPACE}%{OBSERVER}%{SPACE}%{PROCESS}%{SPACE}(%{POSINT:process.pid:long}|-) - (-|%{META})", + "TIMESTAMP_ISO8601": "%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE:event.timezone}?", + "OBSERVER": "(?:%{IP:observer.ip}|%{HOSTNAME:observer.name})", + "UNIXPATH": "(/([\\w_%!$@:.,+~-]+|\\\\.)*)*", + "PROCESS": "(\\(%{DATA:process.name}\\)|(?:%{UNIXPATH})%{BASEPATH:process.name})", + "BASEPATH": "[[[:alnum:]]_%!$@:.,+~-]+", + "META": "\\[[^\\]]*\\]" + } + } + }, + { + "date": { + "if": "ctx._tmp.timestamp8601 != null", + "field": "_tmp.timestamp8601", + "target_field": "@timestamp", + "formats": [ + "ISO8601" + ] + } + }, + { + "date": { + "if": "ctx.event?.timezone != null && ctx._tmp?.timestamp != null", + "field": "_tmp.timestamp", + "target_field": "@timestamp", + "formats": [ + "MMM d HH:mm:ss", + "MMM d HH:mm:ss", + "MMM dd HH:mm:ss" + ], + "timezone": "{{ event.timezone }}" + } + }, + { + "grok": { + "description": "Set Event Provider", + "field": "process.name", + "patterns": [ + "^%{HYPHENATED_WORDS:event.provider}" + ], + "pattern_definitions": { + "HYPHENATED_WORDS": "\\b[A-Za-z0-9_]+(-[A-Za-z_]+)*\\b" + } + } + }, + { + "pipeline": { + "name": "logs-pfsense.log-1.19.1-firewall", + "if": "ctx.event.provider == 'filterlog'" + } + }, + { + "pipeline": { + "name": "logs-pfsense.log-1.19.1-openvpn", + "if": "ctx.event.provider == 'openvpn'" + } + }, + { + "pipeline": { + "name": "logs-pfsense.log-1.19.1-ipsec", + "if": "ctx.event.provider == 'charon'" + } + }, + { + "pipeline": { + "name": "logs-pfsense.log-1.19.1-dhcp", + "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)" + } + }, + { + "pipeline": { + "name": "logs-pfsense.log-1.19.1-unbound", + "if": "ctx.event.provider == 'unbound'" + } + }, + { + "pipeline": { + "name": "logs-pfsense.log-1.19.1-haproxy", + "if": "ctx.event.provider == 'haproxy'" + } + }, + { + "pipeline": { + "name": "logs-pfsense.log-1.19.1-php-fpm", + "if": "ctx.event.provider == 'php-fpm'" + } + }, + { + "pipeline": { + "name": "logs-pfsense.log-1.19.1-squid", + "if": "ctx.event.provider == 'squid'" + } + }, + { + "pipeline": { + "name": "logs-pfsense.log-1.16.0-suricata", + "if": "ctx.event.provider == 'suricata'" + } + }, + { + "drop": { + "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"suricata\"].contains(ctx.event?.provider)" + } + }, + { + "append": { + "field": "event.category", + "value": "network", + "if": "ctx.network != null" + } + }, + { + "convert": { + "field": "source.address", + "target_field": "source.ip", + "type": "ip", + "ignore_failure": true, + "ignore_missing": true + } + }, + { + "convert": { + "field": "destination.address", + "target_field": "destination.ip", + "type": "ip", + "ignore_failure": true, + "ignore_missing": true + } + }, + { + "set": { + "field": "network.type", + "value": "ipv6", + "if": "ctx.source?.ip != null && ctx.source.ip.contains(\":\")" + } + }, + { + "set": { + "field": "network.type", + "value": "ipv4", + "if": "ctx.source?.ip != null && ctx.source.ip.contains(\".\")" + } + }, + { + "geoip": { + "field": "source.ip", + "target_field": "source.geo", + "ignore_missing": true + } + }, + { + "geoip": { + "field": "destination.ip", + "target_field": "destination.geo", + "ignore_missing": true + } + }, + { + "geoip": { + "ignore_missing": true, + "database_file": "GeoLite2-ASN.mmdb", + "field": "source.ip", + "target_field": "source.as", + "properties": [ + "asn", + "organization_name" + ] + } + }, + { + "geoip": { + "database_file": "GeoLite2-ASN.mmdb", + "field": "destination.ip", + "target_field": "destination.as", + "properties": [ + "asn", + "organization_name" + ], + "ignore_missing": true + } + }, + { + "rename": { + "field": "source.as.asn", + "target_field": "source.as.number", + "ignore_missing": true + } + }, + { + "rename": { + "field": "source.as.organization_name", + "target_field": "source.as.organization.name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "destination.as.asn", + "target_field": "destination.as.number", + "ignore_missing": true + } + }, + { + "rename": { + "field": "destination.as.organization_name", + "target_field": "destination.as.organization.name", + "ignore_missing": true + } + }, + { + "community_id": { + "target_field": "network.community_id", + "ignore_failure": true + } + }, + { + "grok": { + "field": "observer.ingress.interface.name", + "patterns": [ + "%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}" + ], + "ignore_missing": true, + "ignore_failure": true + } + }, + { + "set": { + "field": "network.vlan.id", + "copy_from": "observer.ingress.vlan.id", + "ignore_empty_value": true + } + }, + { + "append": { + "field": "related.ip", + "value": "{{destination.ip}}", + "allow_duplicates": false, + "if": "ctx.destination?.ip != null" + } + }, + { + "append": { + "field": "related.ip", + "value": "{{source.ip}}", + "allow_duplicates": false, + "if": "ctx.source?.ip != null" + } + }, + { + "append": { + "field": "related.ip", + "value": "{{source.nat.ip}}", + "allow_duplicates": false, + "if": "ctx.source?.nat?.ip != null" + } + }, + { + "append": { + "field": "related.hosts", + "value": "{{destination.domain}}", + "if": "ctx.destination?.domain != null" + } + }, + { + "append": { + "field": "related.user", + "value": "{{user.name}}", + "if": "ctx.user?.name != null" + } + }, + { + "set": { + "field": "network.direction", + "value": "{{network.direction}}bound", + "if": "ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/" + } + }, + { + "remove": { + "field": [ + "_tmp" + ], + "ignore_failure": true + } + }, + { + "script": { + "lang": "painless", + "description": "This script processor iterates over the whole document to remove fields with null values.", + "source": "void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n" + } + }, + { + "remove": { + "field": "event.original", + "if": "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))", + "ignore_failure": true, + "ignore_missing": true + } + }, + { + "pipeline": { + "name": "global@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Global pipeline for all data streams" + } + }, + { + "pipeline": { + "name": "logs@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for all data streams of type `logs`" + } + }, + { + "pipeline": { + "name": "logs-pfsense.integration@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for all data streams of type `logs` defined by the `pfsense` integration" + } + }, + { + "pipeline": { + "name": "logs-pfsense.log@custom", + "ignore_missing_pipeline": true, + "description": "[Fleet] Pipeline for the `pfsense.log` dataset" + } + } + ], + "on_failure": [ + { + "remove": { + "field": [ + "_tmp" + ], + "ignore_failure": true + } + }, + { + "set": { + "field": "event.kind", + "value": "pipeline_error" + } + }, + { + "append": { + "field": "error.message", + "value": "{{{ _ingest.on_failure_message }}}" + } + } + ], + "_meta": { + "managed_by": "fleet", + "managed": true, + "package": { + "name": "pfsense" + } + } +} diff --git a/salt/elasticsearch/files/ingest/suricata.alert_pfsense b/salt/elasticsearch/files/ingest/suricata.alert_pfsense new file mode 100644 index 000000000..32b0a285d --- /dev/null +++ b/salt/elasticsearch/files/ingest/suricata.alert_pfsense @@ -0,0 +1,16 @@ +{ + "description" : "suricata.alert", + "processors" : [ + { "set": { "field": "data_stream.dataset", "value": "suricata" } }, + { "set": { "field": "data_stream.namespace", "value": "so" } }, + { "set": { "field": "_index", "value": "logs-suricata.alerts-so" } }, + { "set": { "field": "tags","value": "alert" }}, + { "rename":{ "field": "message2.alert", "target_field": "rule", "ignore_failure": true } }, + { "rename":{ "field": "rule.signature", "target_field": "rule.name", "ignore_failure": true } }, + { "rename":{ "field": "rule.ref", "target_field": "rule.version", "ignore_failure": true } }, + { "rename":{ "field": "rule.signature_id", "target_field": "rule.uuid", "ignore_failure": true } }, + { "rename":{ "field": "rule.signature_id", "target_field": "rule.signature", "ignore_failure": true } }, + { "rename":{ "field": "message2.payload_printable", "target_field": "network.data.decoded", "ignore_failure": true } }, + { "pipeline": { "name": "common.nids" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/suricata.common_pfsense b/salt/elasticsearch/files/ingest/suricata.common_pfsense new file mode 100644 index 000000000..04da7b482 --- /dev/null +++ b/salt/elasticsearch/files/ingest/suricata.common_pfsense @@ -0,0 +1,23 @@ +{ + "description" : "suricata.common", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.pkt_src", "target_field": "network.packet_source","ignore_failure": true } }, + { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_failure": true } }, + { "rename": { "field": "message2.in_iface", "target_field": "observer.ingress.interface.name", "ignore_failure": true } }, + { "rename": { "field": "message2.flow_id", "target_field": "log.id.uid", "ignore_failure": true } }, + { "rename": { "field": "message2.src_ip", "target_field": "source.ip", "ignore_failure": true } }, + { "rename": { "field": "message2.src_port", "target_field": "source.port", "ignore_failure": true } }, + { "rename": { "field": "message2.dest_ip", "target_field": "destination.ip", "ignore_failure": true } }, + { "rename": { "field": "message2.dest_port", "target_field": "destination.port", "ignore_failure": true } }, + { "rename": { "field": "message2.vlan", "target_field": "network.vlan.id", "ignore_failure": true } }, + { "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_missing": true } }, + { "rename": { "field": "message2.xff", "target_field": "xff.ip", "ignore_missing": true } }, + { "set": { "field": "event.dataset", "value": "{{ message2.event_type }}" } }, + { "set": { "field": "observer.name", "value": "{{agent.name}}" } }, + { "set": { "field": "event.ingested", "value": "{{@timestamp}}" } }, + { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } }, + { "remove":{ "field": "agent", "ignore_failure": true } }, + { "pipeline": { "if": "ctx?.event?.dataset != null", "name": "suricata.{{event.dataset}}_pfsense" } } + ] +}