diff --git a/salt/logstash/conf/conf.enabled.txt.so-helix b/salt/logstash/conf/conf.enabled.txt.so-helix index 5d10847cd..72bbc2ece 100644 --- a/salt/logstash/conf/conf.enabled.txt.so-helix +++ b/salt/logstash/conf/conf.enabled.txt.so-helix @@ -8,4 +8,39 @@ ## # All of the defaults are loaded. /usr/share/logstash/pipeline.dynamic/0010_input_hhbeats.conf +/usr/share/logstash/pipeline.so/1033_preprocess_snort.conf +/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf +/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf +/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf +/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf +/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf +/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf +/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf +/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf +/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf +/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf +/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf +/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf +/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf +/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf +/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf +/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf +/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf +/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf +/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf +/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf +/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf +/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf +/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf +/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf +/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf +/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf +/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf +/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf +/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf +/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf +/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf +/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf +/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf +/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf /usr/share/logstash/pipeline.dynamic/9997_output_helix.conf diff --git a/salt/logstash/files/dynamic/9997_output_helix.conf b/salt/logstash/files/dynamic/9997_output_helix.conf index 495c4ea9e..6168bfb07 100644 --- a/salt/logstash/files/dynamic/9997_output_helix.conf +++ b/salt/logstash/files/dynamic/9997_output_helix.conf @@ -1,7 +1,7 @@ {% set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %} filter { - if "fe_clone" in [type] { + if [type] =~ /^bro_conn|bro_dns|bro_http|bro_files|bro_ssl$/ { grok { match => [ "source_ip", "^%{IPV4:srcipv4}$", @@ -14,6 +14,11 @@ filter { "destination_ip", "^%{IPV4:dstipv4}$" ] } + + grok { + match => [ "syslog-tags", "^.source.s_%{DATA:class}$" ] + } + geoip { source => "[source_ip]" target => "source_geo" @@ -21,87 +26,108 @@ filter { geoip { source => "[destination_ip]" target => "destination_geo" - } mutate { #rename => { "%{[source_geo][country_code]}" => "srccountrycode" } #rename => { "%{[destination_geo][country_code]}" => "dstcountrycode" } rename => { "syslog-host_from" => "sensor" } rename => { "message" => "rawmsg" } - rename => { "event_type" => "program" } - copy => { "program" => "class" } + #rename => { "event_type" => "program" } + #copy => { "program" => "class" } rename => { "source_port" => "srcport" } rename => { "destination_port" => "dstport" } - remove_field => ["source_ip", "destination_ip"] remove_field => ["sensorname", "sensor_name", "service", "source", "tags", "syslog-host"] remove_field => ["sensor_name", "source_ips", "ips", "destination_ips", "syslog-priority", "syslog-file_name", "syslog-facility"] + } + if "bro_conn" in [class] { + mutate { + #add_field => { "metaclass" => "connection" } + rename => { "original_bytes" => "sentbytes" } + rename => { "respond_bytes" => "rcvdbytes" } + rename => { "connection_state" => "connstate" } + rename => { "uid" => "connectionid" } + rename => { "respond_packets" => "rcvdpackets" } + rename => { "original_packets" => "sentpackets" } + rename => { "respond_ip_bytes" => "rcvdipbytes" } + rename => { "original_ip_bytes" => "sentipbytes" } + rename => { "local_respond" => "local_resp" } + rename => { "local_orig" => "localorig" } + rename => { "missed_bytes" => "missingbytes" } } } - if "bro_conn" in [program] { - mutate { - #add_field => { "metaclass" => "connection" } - rename => { "original_bytes" => "sentbytes" } - rename => { "respond_bytes" => "rcvdbytes" } - rename => { "connection_state" => "connstate" } - rename => { "uid" => "connectionid" } - rename => { "respond_packets" => "rcvdpackets" } - rename => { "original_packets" => "sentpackets" } - rename => { "respond_ip_bytes" => "rcvdipbytes" } - rename => { "original_ip_bytes" => "sentipbytes" } - rename => { "local_respond" => "local_resp" } - rename => { "local_orig" => "localorig" } - rename => { "missed_bytes" => "missingbytes" } + if "bro_dns" in [class] { + mutate{ + #add_field = { "metaclass" => "dns"} + rename => { "answers" => "answer" } + rename => { "query" => "domain" } + rename => { "query_class" => "queryclass" } + rename => { "query_class_name" => "queryclassname" } + rename => { "query_type" => "querytype" } + rename => { "query_type_name" => "querytypename" } + rename => { "ra" => "recursionavailable" } + rename => { "rd" => "recursiondesired" } + } } - } - if "bro_dns" in [program] { - mutate{ - #add_field = { "metaclass" => "dns"} - rename => { "query" => "domain" } - rename => { "query_class" => "queryclass" } - rename => { "query_class_name" => "queryclassname" } - rename => { "query_type" => "querytype" } - rename => { "query_type_name" => "querytypename" } - rename => { "ra" => "recursionavailable" } - rename => { "rd" => "recursiondesired" } - + if "bro_dhcp" in [class] { + mutate{ + #add_field = { "metaclass" => "dhcp"} + rename => { "ips" => "ip" } + } } - } - if "bro_dhcp" in [program] { - mutate{ - #add_field = { "metaclass" => "dhcp"} - rename => { "ips" => "ip" } + if "bro_files" in [class] { + mutate{ + #add_field = { "metaclass" => "dns"} + rename => { "missing_bytes" => "missingbytes" } + rename => { "fuid" => "fileid" } + rename => { "uid" => "connectionid" } + } } - } - if "bro_files" in [program] { - mutate{ - #add_field = { "metaclass" => "dns"} - rename => { "missing_bytes" => "missingbytes" } - rename => { "fuid" => "fileid" } - rename => { "uid" => "connectionid" } + if "bro_http" in [class] { + mutate{ + #add_field = { "metaclass" => "dns"} + rename => { "virtual_host" => "hostname" } + rename => { "status_code" => "statuscode" } + rename => { "status_message" => "statusmsg" } + rename => { "resp_mime_types" => "rcvdmimetype" } + rename => { "resp_fuids" => "rcvdfileid" } + rename => { "response_body_len" => "rcvdbodybytes" } + rename => { "request_body_len" => "sentbodybytes" } + rename => { "uid" => "connectionid" } + rename => { "ts"=> "eventtime" } + rename => { "@timestamp"=> "eventtime" } + } } - } - if "bro_http" in [program] { - mutate{ - #add_field = { "metaclass" => "dns"} - rename => { "status_code" => "statuscode" } - rename => { "status_message" => "statusmsg" } - rename => { "resp_mime_types" => "rcvdmimetype" } - rename => { "resp_fuids" => "rcvdfileid" } - rename => { "response_body_len" => "rcvdbodybytes" } - rename => { "request_body_len" => "sentbodybytes" } - + if "bro_ssl" in [class] { + mutate{ + #add_field = { "metaclass" => "dns"} + rename => { "status_code" => "statuscode" } + rename => { "status_message" => "statusmsg" } + rename => { "resp_mime_types" => "rcvdmimetype" } + rename => { "resp_fuids" => "rcvdfileid" } + rename => { "response_body_len" => "rcvdbodybytes" } + rename => { "request_body_len" => "sentbodybytes" } + } } } } + +#output { +# if [event_type] =~ /^bro_conn|bro_dns|bro_http|bro_files|bro_ssl$/ { +# http { +# url => "https://helix-integrations.cloud.aws.apps.fireeye.com/api/upload" +# http_method => post +# http_compression => true +# socket_timeout => 60 +# headers => ["Authorization","{{ HELIX_API_KEY }}"] +# format => json_batch +# } +# } +#} output { - if "fe_clone" in [type] { - http { - url => "https://helix-integrations.cloud.aws.apps.fireeye.com/api/upload?source=test&format=json" - http_method => post - http_compression => true - headers => ["Authorization", "{{ HELIX_API_KEY }}"] - format => json_batch + if [event_type] =~ /^bro_conn|bro_dns|bro_http|bro_files|bro_ssl$/ { + file { + path => "/var/log/logstash/output.json" } } }