Helix - Change Parsers for Helix

salt/logstash/conf/conf.enabled.txt.so-helix

@@ -8,4 +8,39 @@
 ##
 # All of the defaults are loaded.
 /usr/share/logstash/pipeline.dynamic/0010_input_hhbeats.conf
+/usr/share/logstash/pipeline.so/1033_preprocess_snort.conf
+/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf
+/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf
+/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf
+/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf
+/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf
+/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf
+/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf
+/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf
+/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf
+/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf
+/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf
+/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf
+/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf
+/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf
+/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf
+/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf
+/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf
+/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf
+/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf
+/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf
+/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf
+/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf
+/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf
+/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf
+/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf
+/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf
+/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf
+/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf
+/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf
+/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf
+/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf
+/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf
+/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf
+/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf
 /usr/share/logstash/pipeline.dynamic/9997_output_helix.conf

salt/logstash/files/dynamic/9997_output_helix.conf

@@ -1,7 +1,7 @@
 {% set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %}
 
 filter {
-  if "fe_clone" in [type] {
+  if [type] =~ /^bro_conn|bro_dns|bro_http|bro_files|bro_ssl$/ {
     grok {
       match => [
         "source_ip", "^%{IPV4:srcipv4}$",
@@ -14,6 +14,11 @@ filter {
         "destination_ip", "^%{IPV4:dstipv4}$"
        ]
     }
+
+    grok {
+      match => [ "syslog-tags",  "^.source.s_%{DATA:class}$" ]
+    }
+
     geoip {
       source => "[source_ip]"
       target => "source_geo"
@@ -21,24 +26,21 @@ filter {
     geoip {
       source => "[destination_ip]"
       target => "destination_geo"
-
     }
     mutate {
       #rename => { "%{[source_geo][country_code]}" => "srccountrycode" }
       #rename => { "%{[destination_geo][country_code]}" => "dstcountrycode" }
       rename => { "syslog-host_from" => "sensor" }
       rename => { "message" => "rawmsg" }
-      rename => { "event_type" => "program" }
+      #rename => { "event_type" => "program" }
-      copy => { "program" => "class" }
+      #copy => { "program" => "class" }
       rename => { "source_port" => "srcport" }
       rename => { "destination_port" => "dstport" }
-
       remove_field => ["source_ip", "destination_ip"]
       remove_field => ["sensorname", "sensor_name", "service", "source", "tags", "syslog-host"]
       remove_field => ["sensor_name", "source_ips", "ips", "destination_ips", "syslog-priority", "syslog-file_name", "syslog-facility"]
     }
-    }
+    if "bro_conn" in [class] {
-  if "bro_conn" in [program] {
       mutate {
         #add_field => { "metaclass" => "connection" }
         rename => { "original_bytes" => "sentbytes" }
@@ -54,9 +56,10 @@ filter {
         rename => { "missed_bytes" => "missingbytes" }
       }
     }
-  if "bro_dns" in [program] {
+    if "bro_dns" in [class] {
       mutate{
         #add_field = { "metaclass" => "dns"}
+        rename => { "answers" => "answer" }
         rename => { "query" => "domain" }
         rename => { "query_class" => "queryclass" }
         rename => { "query_class_name" => "queryclassname" }
@@ -64,16 +67,15 @@ filter {
         rename => { "query_type_name" => "querytypename" }
         rename => { "ra" => "recursionavailable" }
         rename => { "rd" => "recursiondesired" }
-
       }
     }
-  if "bro_dhcp" in [program] {
+    if "bro_dhcp" in [class] {
       mutate{
         #add_field = { "metaclass" => "dhcp"}
         rename => { "ips" => "ip" }
       }
     }
-  if "bro_files" in [program] {
+    if "bro_files" in [class] {
       mutate{
         #add_field = { "metaclass" => "dns"}
         rename => { "missing_bytes" => "missingbytes" }
@@ -81,7 +83,22 @@ filter {
         rename => { "uid" => "connectionid" }
       }
     }
-  if "bro_http" in [program] {
+    if "bro_http" in [class] {
+      mutate{
+        #add_field = { "metaclass" => "dns"}
+        rename => { "virtual_host" => "hostname" }
+        rename => { "status_code" => "statuscode" }
+        rename => { "status_message" => "statusmsg" }
+        rename => { "resp_mime_types" => "rcvdmimetype" }
+        rename => { "resp_fuids" => "rcvdfileid" }
+        rename => { "response_body_len" => "rcvdbodybytes" }
+        rename => { "request_body_len" => "sentbodybytes" }
+        rename => { "uid" => "connectionid" }
+        rename => { "ts"=> "eventtime" }
+	      rename => { "@timestamp"=> "eventtime" }
+      }
+    }
+    if "bro_ssl" in [class] {
       mutate{
         #add_field = { "metaclass" => "dns"}
         rename => { "status_code" => "statuscode" }
@@ -90,18 +107,27 @@ filter {
         rename => { "resp_fuids" => "rcvdfileid" }
         rename => { "response_body_len" => "rcvdbodybytes" }
         rename => { "request_body_len" => "sentbodybytes" }
+      }
+    }
+  }
+}
 
-    }
+#output {
-  }
+#  if [event_type] =~ /^bro_conn|bro_dns|bro_http|bro_files|bro_ssl$/ {
-}
+#    http {
+#      url => "https://helix-integrations.cloud.aws.apps.fireeye.com/api/upload"
+#      http_method => post
+#      http_compression => true
+#      socket_timeout => 60
+#      headers => ["Authorization","{{ HELIX_API_KEY }}"]
+#      format => json_batch
+#    }
+#  }
+#}
 output {
-  if "fe_clone" in [type] {
+  if [event_type] =~ /^bro_conn|bro_dns|bro_http|bro_files|bro_ssl$/ {
-    http {
+    file {
-      url => "https://helix-integrations.cloud.aws.apps.fireeye.com/api/upload?source=test&format=json"
+      path => "/var/log/logstash/output.json"      
-      http_method => post
-      http_compression => true
-      headers => ["Authorization", "{{ HELIX_API_KEY }}"]
-      format => json_batch
     }
   }
 }