securityonion/salt/logstash/pipelines/config/so/9997_output_helix.conf.jinja

{% set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %}
{% set UNIQUEID = salt['pillar.get']('sensor:uniqueid', '') %}
{% set CBNAME = grains.host %}

filter {
  if [type] =~ /^bro_conn|bro_dns|bro_http|bro_files|bro_ssl|bro_dhcp|bro_x509|suricata$/ {
    grok {
      match => [
        "source_ip", "^%{IPV4:srcipv4}$",
        "source_ip",  "(?<srcipv6>^([0-9A-Fa-f]{0,4}:){2,7}([0-9A-Fa-f]{1,4}$|((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.|$)){4})$)"
        ]
    }
    grok {
      match => [
        "destination_ip",  "(?<dstipv6>^([0-9A-Fa-f]{0,4}:){2,7}([0-9A-Fa-f]{1,4}$|((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.|$)){4})$)",
        "destination_ip", "^%{IPV4:dstipv4}$"
       ]
    }

    #geoip {
    #  source => "[source_ip]"
    #  target => "source_geo"
    #}
    #geoip {
    #  source => "[destination_ip]"
    #  target => "destination_geo"
    #}
    mutate {
      rename => { "[beat_host][name]" => "sensor" }
      copy => { "sensor" => "rawmsghostname" }
      rename => { "message" => "rawmsg" }
      copy => { "type" => "class" }
      copy => { "class" => "program"}
      rename => { "source_port" => "srcport" }
      rename => { "destination_port" => "dstport" }
      rename => { "[log][file][path]" => "filepath" }
      add_field => { "meta_cbid" => "{{ UNIQUEID }}" }
      add_field => { "meta_cbname" => "{{ CBNAME }}" }
      remove_field => ["source_ip", "destination_ip", "syslog-host_from"]
      remove_field => ["beat_host", "timestamp", "type", "log", "@version", "@timestamp"]
      remove_field => ["sensorname", "sensor_name", "service", "source", "tags", "syslog-host"]
      remove_field => ["sensor_name", "source_ips", "ips", "destination_ips", "syslog-priority", "syslog-file_name", "syslog-facility"]
    }
    if "bro_conn" in [class] {
      mutate {
        #add_field => { "metaclass" => "connection" }
        rename => { "original_bytes" => "sentbytes" }
        rename => { "respond_bytes" => "rcvdbytes" }
        rename => { "connection_state" => "connstate" }
        rename => { "uid" => "connectionid" }
        rename => { "respond_packets" => "rcvdpackets" }
        rename => { "original_packets" => "sentpackets" }
        rename => { "respond_ip_bytes" => "rcvdipbytes" }
        rename => { "original_ip_bytes" => "sentipbytes" }
        rename => { "local_respond" => "local_resp" }
        rename => { "local_orig" => "localorig" }
        rename => { "missed_bytes" => "missingbytes" }
        rename => { "connection_state_description" => "description" }
      }
    }
    if "bro_dns" in [class] {
      mutate{
        #add_field = { "metaclass" => "dns"}
        rename => { "answers" => "answer" }
        rename => { "query" => "domain" }
        rename => { "query_class" => "queryclass" }
        rename => { "query_class_name" => "queryclassname" }
        rename => { "query_type" => "querytype" }
        rename => { "query_type_name" => "querytypename" }
        rename => { "ra" => "recursionavailable" }
        rename => { "rd" => "recursiondesired" }
        rename => { "uid" => "connectionid" }
        rename => { "ttls" => "ttl" }
        rename => { "transaction_id" => "transactionid" }
      }
    }
    if "bro_dhcp" in [class] {
      mutate{
        #add_field = { "metaclass" => "dhcp"}
        rename => { "message_types" => "direction" }
        rename => { "uid" => "connectionid" }
	rename => { "lease_time" => "duration" }
      }
    }
    if "bro_files" in [class] {
      mutate{
        #add_field = { "metaclass" => "dns"}
        rename => { "missing_bytes" => "missingbytes" }
        rename => { "seen_bytes" => "seenbytes" }
        rename => { "overflow_bytes" => "overflowbytes" }
        rename => { "fuid" => "fileid" }
        rename => { "conn_uids" => "connectionid" }
        rename => { "is_orig" => "isorig" }
        rename => { "timed_out" => "timedout" }
        rename => { "local_orig" => "localorig" }
        rename => { "file_ip" => "tx_host" }
      }
    }
    if "bro_http" in [class] {
      mutate{
        #add_field = { "metaclass" => "dns"}
        rename => { "virtual_host" => "hostname" }
        rename => { "status_code" => "statuscode" }
        rename => { "status_message" => "statusmsg" }
        rename => { "resp_mime_types" => "rcvdmimetype" }
        rename => { "resp_fuids" => "rcvdfileid" }
        rename => { "response_body_len" => "rcvdbodybytes" }
        rename => { "request_body_len" => "sentbodybytes" }
        rename => { "uid" => "connectionid" }
        rename => { "ts"=> "eventtime" }
        rename => { "@timestamp"=> "eventtime" }
        rename => { "trans_depth" => "depth" }
        rename => { "request_body_length" => "sentbodybytes" }
        rename => { "response_body_length" => "rcvdbodybytes" }
      }
    }
    if "bro_ssl" in [class] {
      mutate{
        #add_field = { "metaclass" => "dns"}
        rename => { "status_code" => "statuscode" }
        rename => { "status_message" => "statusmsg" }
        rename => { "resp_mime_types" => "rcvdmimetype" }
        rename => { "resp_fuids" => "rcvdfileid" }
        rename => { "response_body_len" => "rcvdbodybytes" }
        rename => { "request_body_len" => "sentbodybytes" }
        rename => { "uid" => "connectionid" }
      }
    }
    if "bro_weird" in [class] {
      mutate{
        #add_field = { "metaclass" => "dns"}
        rename => { "name" => "eventname" }
      }
    }
    if "bro_x509" in [class] {
      mutate{
        #add_field = { "metaclass" => "dns"}
	rename => { "certificate_common_name" => "certname" }
        rename => { "certificate_subject" => "certsubject" }
	rename => { "issuer_common_name" => "issuer" }
	rename => { "certificate_issuer" => "issuersubject" }
	rename => { "certificate_not_valid_before" => "issuetime" }
	rename => { "certificate_key_type" => "cert_type" }
      }
    }
  }
}

output {
  if [class] =~ /^bro_conn|bro_dns|bro_http|bro_files|bro_ssl|bro_dhcp|bro_x509|suricata$/ {
    http {
      url => "https://helix-integrations.cloud.aws.apps.fireeye.com/api/upload"
      http_method => post
      http_compression => true
      socket_timeout => 60
      headers => ["Authorization","{{ HELIX_API_KEY }}"]
      format => json_batch
    }
  }
}