CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 17:22:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

Helix - Change Parsers for Helix

Browse Source
This commit is contained in:
Mike Reeves
2019-12-10 13:50:27 -05:00
parent c46c539277
commit e134071295
2 changed files with 122 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
salt/logstash/conf/conf.enabled.txt.so-helix
View File

@@ -8,4 +8,39 @@
## ##
# All of the defaults are loaded. # All of the defaults are loaded.
/usr/share/logstash/pipeline.dynamic/0010_input_hhbeats.conf /usr/share/logstash/pipeline.dynamic/0010_input_hhbeats.conf
/usr/share/logstash/pipeline.so/1033_preprocess_snort.conf
/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf
/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf
/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf
/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf
/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf
/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf
/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf
/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf
/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf
/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf
/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf
/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf
/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf
/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf
/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf
/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf
/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf
/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf
/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf
/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf
/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf
/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf
/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf
/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf
/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf
/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf
/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf
/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf
/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf
/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf
/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf
/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf
/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf
/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf
/usr/share/logstash/pipeline.dynamic/9997_output_helix.conf /usr/share/logstash/pipeline.dynamic/9997_output_helix.conf

148
salt/logstash/files/dynamic/9997_output_helix.conf
View File

@@ -1,7 +1,7 @@
{% set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %} {% set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %}
filter { filter {
if "fe_clone" in [type] { if [type] =~ /^bro_conn|bro_dns|bro_http|bro_files|bro_ssl$/ {
grok { grok {
match => [ match => [
"source_ip", "^%{IPV4:srcipv4}$", "source_ip", "^%{IPV4:srcipv4}$",
@@ -14,6 +14,11 @@ filter {
"destination_ip", "^%{IPV4:dstipv4}$" "destination_ip", "^%{IPV4:dstipv4}$"
] ]
} }
grok {
match => [ "syslog-tags", "^.source.s_%{DATA:class}$" ]
}
geoip { geoip {
source => "[source_ip]" source => "[source_ip]"
target => "source_geo" target => "source_geo"
@@ -21,87 +26,108 @@ filter {
geoip { geoip {
source => "[destination_ip]" source => "[destination_ip]"
target => "destination_geo" target => "destination_geo"
} }
mutate { mutate {
#rename => { "%{[source_geo][country_code]}" => "srccountrycode" } #rename => { "%{[source_geo][country_code]}" => "srccountrycode" }
#rename => { "%{[destination_geo][country_code]}" => "dstcountrycode" } #rename => { "%{[destination_geo][country_code]}" => "dstcountrycode" }
rename => { "syslog-host_from" => "sensor" } rename => { "syslog-host_from" => "sensor" }
rename => { "message" => "rawmsg" } rename => { "message" => "rawmsg" }
rename => { "event_type" => "program" } #rename => { "event_type" => "program" }
copy => { "program" => "class" } #copy => { "program" => "class" }
rename => { "source_port" => "srcport" } rename => { "source_port" => "srcport" }
rename => { "destination_port" => "dstport" } rename => { "destination_port" => "dstport" }
remove_field => ["source_ip", "destination_ip"] remove_field => ["source_ip", "destination_ip"]
remove_field => ["sensorname", "sensor_name", "service", "source", "tags", "syslog-host"] remove_field => ["sensorname", "sensor_name", "service", "source", "tags", "syslog-host"]
remove_field => ["sensor_name", "source_ips", "ips", "destination_ips", "syslog-priority", "syslog-file_name", "syslog-facility"] remove_field => ["sensor_name", "source_ips", "ips", "destination_ips", "syslog-priority", "syslog-file_name", "syslog-facility"]
}
if "bro_conn" in [class] {
mutate {
#add_field => { "metaclass" => "connection" }
rename => { "original_bytes" => "sentbytes" }
rename => { "respond_bytes" => "rcvdbytes" }
rename => { "connection_state" => "connstate" }
rename => { "uid" => "connectionid" }
rename => { "respond_packets" => "rcvdpackets" }
rename => { "original_packets" => "sentpackets" }
rename => { "respond_ip_bytes" => "rcvdipbytes" }
rename => { "original_ip_bytes" => "sentipbytes" }
rename => { "local_respond" => "local_resp" }
rename => { "local_orig" => "localorig" }
rename => { "missed_bytes" => "missingbytes" }
} }
} }
if "bro_conn" in [program] { if "bro_dns" in [class] {
mutate { mutate{
#add_field => { "metaclass" => "connection" } #add_field = { "metaclass" => "dns"}
rename => { "original_bytes" => "sentbytes" } rename => { "answers" => "answer" }
rename => { "respond_bytes" => "rcvdbytes" } rename => { "query" => "domain" }
rename => { "connection_state" => "connstate" } rename => { "query_class" => "queryclass" }
rename => { "uid" => "connectionid" } rename => { "query_class_name" => "queryclassname" }
rename => { "respond_packets" => "rcvdpackets" } rename => { "query_type" => "querytype" }
rename => { "original_packets" => "sentpackets" } rename => { "query_type_name" => "querytypename" }
rename => { "respond_ip_bytes" => "rcvdipbytes" } rename => { "ra" => "recursionavailable" }
rename => { "original_ip_bytes" => "sentipbytes" } rename => { "rd" => "recursiondesired" }
rename => { "local_respond" => "local_resp" } }
rename => { "local_orig" => "localorig" }
rename => { "missed_bytes" => "missingbytes" }
} }
} if "bro_dhcp" in [class] {
if "bro_dns" in [program] { mutate{
mutate{ #add_field = { "metaclass" => "dhcp"}
#add_field = { "metaclass" => "dns"} rename => { "ips" => "ip" }
rename => { "query" => "domain" } }
rename => { "query_class" => "queryclass" }
rename => { "query_class_name" => "queryclassname" }
rename => { "query_type" => "querytype" }
rename => { "query_type_name" => "querytypename" }
rename => { "ra" => "recursionavailable" }
rename => { "rd" => "recursiondesired" }
} }
} if "bro_files" in [class] {
if "bro_dhcp" in [program] { mutate{
mutate{ #add_field = { "metaclass" => "dns"}
#add_field = { "metaclass" => "dhcp"} rename => { "missing_bytes" => "missingbytes" }
rename => { "ips" => "ip" } rename => { "fuid" => "fileid" }
rename => { "uid" => "connectionid" }
}
} }
} if "bro_http" in [class] {
if "bro_files" in [program] { mutate{
mutate{ #add_field = { "metaclass" => "dns"}
#add_field = { "metaclass" => "dns"} rename => { "virtual_host" => "hostname" }
rename => { "missing_bytes" => "missingbytes" } rename => { "status_code" => "statuscode" }
rename => { "fuid" => "fileid" } rename => { "status_message" => "statusmsg" }
rename => { "uid" => "connectionid" } rename => { "resp_mime_types" => "rcvdmimetype" }
rename => { "resp_fuids" => "rcvdfileid" }
rename => { "response_body_len" => "rcvdbodybytes" }
rename => { "request_body_len" => "sentbodybytes" }
rename => { "uid" => "connectionid" }
rename => { "ts"=> "eventtime" }
rename => { "@timestamp"=> "eventtime" }
}
} }
} if "bro_ssl" in [class] {
if "bro_http" in [program] { mutate{
mutate{ #add_field = { "metaclass" => "dns"}
#add_field = { "metaclass" => "dns"} rename => { "status_code" => "statuscode" }
rename => { "status_code" => "statuscode" } rename => { "status_message" => "statusmsg" }
rename => { "status_message" => "statusmsg" } rename => { "resp_mime_types" => "rcvdmimetype" }
rename => { "resp_mime_types" => "rcvdmimetype" } rename => { "resp_fuids" => "rcvdfileid" }
rename => { "resp_fuids" => "rcvdfileid" } rename => { "response_body_len" => "rcvdbodybytes" }
rename => { "response_body_len" => "rcvdbodybytes" } rename => { "request_body_len" => "sentbodybytes" }
rename => { "request_body_len" => "sentbodybytes" } }
} }
} }
} }
#output {
# if [event_type] =~ /^bro_conn|bro_dns|bro_http|bro_files|bro_ssl$/ {
# http {
# url => "https://helix-integrations.cloud.aws.apps.fireeye.com/api/upload"
# http_method => post
# http_compression => true
# socket_timeout => 60
# headers => ["Authorization","{{ HELIX_API_KEY }}"]
# format => json_batch
# }
# }
#}
output { output {
if "fe_clone" in [type] { if [event_type] =~ /^bro_conn|bro_dns|bro_http|bro_files|bro_ssl$/ {
http { file {
url => "https://helix-integrations.cloud.aws.apps.fireeye.com/api/upload?source=test&format=json" path => "/var/log/logstash/output.json"
http_method => post
http_compression => true
headers => ["Authorization", "{{ HELIX_API_KEY }}"]
format => json_batch
} }
} }
} }