diff --git a/salt/logstash/pipelines/config/so/1033_preprocess_snort.conf b/salt/logstash/pipelines/config/so/1033_preprocess_snort.conf index 897a8ae4b..9b18bbc15 100644 --- a/salt/logstash/pipelines/config/so/1033_preprocess_snort.conf +++ b/salt/logstash/pipelines/config/so/1033_preprocess_snort.conf @@ -1,181 +1,125 @@ -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Updated by: Doug Burks -# Last Update: 3/15/2018 - filter { - if [type] == "ids" { - # This is the initial parsing of the log - if [engine] == "suricata" { - json { - source => "message" + if [engine] == "suricata" { + json { + source => "message" + } + mutate { + # Make this compatible with event.id as a string + convert => { "[flow_id]" => "string" } + rename => { + "proto" => "[network][transport]" + "event_type" => "[event][dataset]" + "flow_id" => "[event][id]" + "community_id" => "[network][community_id]" + } + lowercase => [ "[network][transport]" ] + merge => {"[event][id]" => "[related][id]" } + add_field => { + "[related][domain]" => [] + "[related][ip]" => [] + "[related][id]" => [] + "[event][module]" => "suricata" + "[event][created]" => "%{[@timestamp]}" + "[event][version]" => "1.0.0" + "[event][category]" => "network" + } + } + + # Set the timestamp from the event + date { + match => [ "timestamp", "ISO8601" ] + tag_on_failure => [ "_dateparsefailure", "_parsefailure", "_suricata_dateparsefailure" ] + remove_field => [ "timestamp" ] + } + + # Suricata uses top-level src/dest to track flow + if [src_ip] { + mutate { + rename => { + "[src_ip]" => "[source][ip]" + "[src_port]" => "[source][port]" } + merge => { "[related][ip]" => "[source][ip]" } + } + } + if [dest_ip] { + mutate { + rename => { + "[dest_ip]" => "[destination][ip]" + "[dest_port]" => "[destination][port]" + } + merge => { "[related][ip]" => "[destination][ip]" } + } + } + + if [vlan] { + mutate { + rename => { "[vlan]" => "[vlan][id]" } + } + } + if [app_proto] { + if [app_proto] == "failed" { + # delete failed detections to be consistent with zeek + mutate { rename => { "app_proto" => "[error][message]" } } + } + else { + mutate { rename => {"app_proto" => "[network][protocol]"}} + } + } + if [event_type] == "alert" { + if [alert][severity] == 1 { mutate { - rename => { "alert" => "orig_alert" } - rename => { "[orig_alert][gid]" => "gid" } - rename => { "[orig_alert][signature_id]" => "sid" } - rename => { "[orig_alert][rev]" => "rev" } - rename => { "[orig_alert][signature]" => "alert" } - rename => { "[orig_alert][category]" => "classification" } - rename => { "[orig_alert][severity]" => "priority" } - rename => { "[orig_alert][rule]" => "rule_signature" } - rename => { "app_proto" => "application_protocol" } - rename => { "dest_ip" => "destination_ip" } - rename => { "dest_port" => "destination_port" } - rename => { "in_iface" => "interface" } - rename => { "proto" => "protocol" } - rename => { "src_ip" => "source_ip" } - rename => { "src_port" => "source_port" } - #rename => { "[fileinfo][filename]" => "filename" } - #rename => { "[fileinfo][gaps]" => "gaps" } - #rename => { "[fileinfo][size]" => "size" } - #rename => { "[fileinfo][state]" => "state" } - #rename => { "[fileinfo][stored]" => "stored" } - #rename => { "[fileinfo][tx_id]" => "tx_id" } - #rename => { "[flow][age]" => "duration" } - #rename => { "[flow][alerted]" => "flow_alerted" } - #rename => { "[flow][bytes_toclient]" => "bytes_to_client" } - #rename => { "[flow][bytes_toserver]" => "bytes_to_server" } - #rename => { "[flow][end]" => "flow_end" } - #rename => { "[flow][pkts_toclient]" => "packets_to_client" } - #rename => { "[flow][pkts_toserver]" => "packets_to_server" } - #rename => { "[flow][reason]" => "reason" } - #rename => { "[flow][start]" => "flow_start" } - #rename => { "[flow][state]" => "state" } - #rename => { "[netflow][age]" => "duration" } - #rename => { "[netflow][bytes]" => "bytes" } - #rename => { "[netflow][end]" => "netflow_end" } - #rename => { "[netflow][start]" => "netflow_start" } - #rename => { "[netflow][pkts]" => "packets" } - rename => { "[alert][action]" => "action" } - rename => { "[alert][category]" => "category" } - rename => { "[alert][gid]" => "gid" } - rename => { "[alert][rev]" => "rev" } - rename => { "[alert][severity]" => "severity" } - rename => { "[alert][signature]" => "signature" } - rename => { "[alert][signature_id]" => "sid" } - #rename => { "[dns][aa]" => "aa" } - #rename => { "[dns][flags]" => "flags" } - #rename => { "[dns][id]" => "id" } - #rename => { "[dns][qr]" => "qr" } - #rename => { "[dns][rcode]" => "rcode_name" } - #rename => { "[dns][rrname]" => "rrname" } - #rename => { "[dns][rrtype]" => "rrtype" } - #rename => { "[dns][tx_id]" => "tx_id" } - #rename => { "[dns][type]" => "record_type" } - #rename => { "[dns][version]" => "version" } - rename => { "[http][hostname]" => "virtual_host" } - rename => { "[http][http_content_type]" => "content_type" } - rename => { "[http][http_port]" => "http_port" } - rename => { "[http][http_method]" => "method" } - rename => { "[http][http_user_agent]" => "useragent" } - #rename => { "[http][length]" => "payload_length" } - #rename => { "[http][protocol]" => "http_version" } - rename => { "[http][status]" => "status_message" } - rename => { "[http][url]" => "url" } - #rename => { "[metadata][flowbits]" => "flowbits" } - rename => { "[tls][fingerprint]" => "certificate_serial_number" } - rename => { "[tls][issuerdn]" => "issuer_distinguished_name" } - rename => { "[tls][notafter]" => "certificate_not_valid_after" } - rename => { "[tls][notbefore]" => "certificate_not_valid_before" } - rename => { "[tls][subject]" => "certificate_common_name" } - rename => { "[tls][version]" => "tls_version" } - rename => { "event_type" => "ids_event_type" } - remove_field => [ "offset", "orig_alert", "beat", "input", "prospector" ] - remove_tag => [ "beats_input_codec_plain_applied" ] - add_tag => [ "eve" ] - - } - } else { + add_field => { "severity" => "High" } + } + } + if [alert][severity] == 2 { + mutate { + add_field => { "severity" => "Medium" } + } + } + if [alert][severity] == 3 { + mutate { + add_field => { "severity" => "Low" } + } + } + # If the alert is a Snort GPL alert break it apart for easier reading and categorization + if [alert][signature] =~ "GPL " { + # This will parse out the category type from the alert grok { - match => ["message", "\[%{INT:gid}:%{INT:sid}:%{INT:rev}\]\s%{DATA:alert}\[Classification:\s+%{DATA:classification}\]\s+\[Priority:\s+%{INT:priority}\]:\s+<%{DATA:interface}>\s+{%{DATA:protocol}}\s+(?:%{IPV4:source_ip}|%{IPV6:source_ip}):%{INT:source_port}\s+->\s+(?:%{IPV4:destination_ip}|%{IPV6:destination_ip}):%{INT:destination_port}", - "message", "\[%{INT:gid}:%{INT:sid}:%{INT:rev}\]\s%{DATA:alert}\[Classification:\s+%{DATA:classification}\]\s+\[Priority:\s+%{INT:priority}\]:\s+<%{DATA:interface}>\s+{%{DATA:protocol}}\s(?:%{IPV4:source_ip}|%{IPV6:source_ip})\s+->\s+(?:%{IPV4:destination_ip}|%{IPV6:destination_ip})", - "message", "\[%{INT:gid}:%{INT:sid}:%{INT:rev}\]\s%{DATA:alert}\[Classification:\s+%{DATA:classification}\]\s+\[Priority:\s+%{INT:priority}\]:\s+{%{DATA:protocol}}\s+(?:%{IPV4:source_ip}|%{IPV6:source_ip}):%{INT:source_port}\s+->\s+%{IPV4:destination_ip}:%{INT:destination_port}", - "message", "\[%{INT:gid}:%{INT:sid}:%{INT:rev}\]\s%{DATA:alert}\[Classification:\s+%{DATA:classification}\]\s+\[Priority:\s+%{INT:priority}\]:\s+{%{DATA:protocol}}\s(?:%{IPV4:source_ip}|%{IPV6:source_ip})\s+->\s+(?:%{IPV4:destination_ip}|%{IPV6:destination_ip})", - "message", "\[%{INT:gid}:%{INT:sid}:%{INT:rev}\]\s%{DATA:alert}\[Classification:\s+%{DATA:classification}\]\s+\[Priority:\s+%{INT:priority}\]:\s+{%{DATA:protocol}}\s+(?:%{IPV4:source_ip}|%{IPV6:source_ip}):%{INT:source_port}\s+->\s+(?:%{IPV4:destination_ip}|%{IPV6:destination_ip}):%{INT:destination_port}", - "message", "\[%{INT:gid}:%{INT:sid}:%{INT:rev}\]\s%{DATA:alert}\[Classification:\s+%{DATA:classification}\]\s+\[Priority:\s+%{INT:priority}\]:\s+{%{DATA:protocol}}\s(?:%{IPV4:source_ip}|%{IPV6:source_ip})\s+->\s+(?:%{IPV4:source_ip}|%{IPV6:source_ip})", - "message", "\[%{INT:gid}:%{INT:sid}:%{INT:rev}\]\s%{DATA:alert}\[Classification:\s+%{DATA:classification}\]\s+\[Priority:\s+%{INT:priority}\]:\s+{%{DATA:protocol}}", - "message", "\A%{TIME} pid\(%{INT}\) Alert Received: %{INT} %{INT:priority} %{DATA:classification} %{DATA:interface} \{%{DATA:timestamp}} %{INT} %{INT} \{%{DATA:alert}} %{IP:source_ip} %{IP:destination_ip} %{INT:protocol} %{INT:source_port} %{INT:destination_port} %{INT:gid} %{INT:sid} %{INT:rev} %{INT} %{INT}\Z", - "message", "%{GREEDYDATA:alert}"] + match => { "[alert][signature]" => "GPL\s+%{DATA:category}\s" } } - } - if [timestamp] { + # This will store the category mutate { - add_field => { "logstash_timestamp" => "%{@timestamp}" } + add_field => { "rule_type" => "Snort GPL" } + lowercase => [ "category" ] } - mutate { - convert => { "logstash_timestamp" => "string" } - } - date { - match => [ "timestamp", "ISO8601" ] - } - mutate { - rename => { "logstash_timestamp" => "timestamp" } - } - } - - # If the alert is a Snort GPL alert break it apart for easier reading and categorization - if [alert] =~ "GPL " { - # This will parse out the category type from the alert - grok { - match => { "alert" => "GPL\s+%{DATA:category}\s" } } - # This will store the category - mutate { - add_field => { "rule_type" => "Snort GPL" } - lowercase => [ "category"] + # If the alert is an Emerging Threat alert break it apart for easier reading and categorization + if [alert][signature] =~ "ET " { + # This will parse out the category type from the alert + grok { + match => { "[alert][signature]" => "ET\s+%{DATA:category}\s" } + } + # This will store the category + mutate { + add_field => { "rule_type" => "Emerging Threats" } + lowercase => [ "category" ] } - } - # If the alert is an Emerging Threat alert break it apart for easier reading and categorization - if [alert] =~ "ET " { - # This will parse out the category type from the alert - grok { - match => { "alert" => "ET\s+%{DATA:category}\s" } } - # This will store the category - mutate { - add_field => { "rule_type" => "Emerging Threats" } - lowercase => [ "category"] + # This section adds URLs to lookup information about a rule online + if [rule_type] == "Snort GPL" { + mutate { + add_field => [ "signature_info", "https://www.snort.org/search?query=%{[alert][gid]}-%{[alert][signature_id]}" ] + } + } + if [rule_type] == "Emerging Threats" { + mutate { + add_field => [ "signature_info", "http://doc.emergingthreats.net/%{[alert][signature_id]}" ] + } } } - # I recommend changing the field types below to integer so searches can do greater than or less than - # and also so math functions can be ran against them mutate { - convert => [ "source_port", "integer" ] - convert => [ "destination_port", "integer" ] - convert => [ "gid", "integer" ] - convert => [ "sid", "integer" ] - # remove_field => [ "message"] + remove_field => [ "alert" ] } - # This will translate the priority field into a severity field of either High, Medium, or Low - if [priority] == 1 { - mutate { - add_field => { "severity" => "High" } - } - } - if [priority] == 2 { - mutate { - add_field => { "severity" => "Medium" } - } - } - if [priority] == 3 { - mutate { - add_field => { "severity" => "Low" } - } - } - # This section adds URLs to lookup information about a rule online - if [sid] and [sid] > 0 and [sid] < 1000000 { - mutate { - add_field => [ "signature_info", "https://www.snort.org/search?query=%{gid}-%{sid}" ] - } - } - if [sid] and [sid] > 1999999 and [sid] < 2999999 { - mutate { - add_field => [ "signature_info", "http://doc.emergingthreats.net/%{sid}" ] - } - } -# mutate { - #add_tag => [ "conf_file_1033"] -# } } }