From 8472b24a67938b68cf804524d08824fe15ab3b7f Mon Sep 17 00:00:00 2001 From: doug Date: Mon, 23 Sep 2019 16:04:23 -0400 Subject: [PATCH] parse Bro logs using Elasticsearch ingest node --- salt/elasticsearch/files/ingest/bro_common | 9 ++ .../elasticsearch/files/ingest/bro_common_ssl | 58 +++++++++++++ salt/elasticsearch/files/ingest/bro_conn | 48 +++++++++++ salt/elasticsearch/files/ingest/bro_dce_rpc | 20 +++++ salt/elasticsearch/files/ingest/bro_dhcp | 20 +++++ salt/elasticsearch/files/ingest/bro_dnp3 | 19 +++++ salt/elasticsearch/files/ingest/bro_dns | 35 ++++++++ salt/elasticsearch/files/ingest/bro_dpd | 19 +++++ salt/elasticsearch/files/ingest/bro_files | 32 +++++++ salt/elasticsearch/files/ingest/bro_ftp | 33 +++++++ salt/elasticsearch/files/ingest/bro_http | 42 +++++++++ salt/elasticsearch/files/ingest/bro_intel | 29 +++++++ salt/elasticsearch/files/ingest/bro_irc | 25 ++++++ salt/elasticsearch/files/ingest/bro_kerberos | 30 +++++++ salt/elasticsearch/files/ingest/bro_modbus | 18 ++++ salt/elasticsearch/files/ingest/bro_mysql | 21 +++++ salt/elasticsearch/files/ingest/bro_notice | 36 ++++++++ salt/elasticsearch/files/ingest/bro_ntlm | 24 ++++++ salt/elasticsearch/files/ingest/bro_pe | 23 +++++ salt/elasticsearch/files/ingest/bro_radius | 25 ++++++ salt/elasticsearch/files/ingest/bro_rdp | 31 +++++++ salt/elasticsearch/files/ingest/bro_rfb | 26 ++++++ .../elasticsearch/files/ingest/bro_signatures | 22 +++++ salt/elasticsearch/files/ingest/bro_sip | 37 ++++++++ salt/elasticsearch/files/ingest/bro_smb_files | 31 +++++++ .../files/ingest/bro_smb_mapping | 21 +++++ salt/elasticsearch/files/ingest/bro_smtp | 38 +++++++++ salt/elasticsearch/files/ingest/bro_snmp | 25 ++++++ salt/elasticsearch/files/ingest/bro_socks | 28 ++++++ salt/elasticsearch/files/ingest/bro_software | 23 +++++ salt/elasticsearch/files/ingest/bro_ssh | 40 +++++++++ salt/elasticsearch/files/ingest/bro_ssl | 33 +++++++ salt/elasticsearch/files/ingest/bro_syslog | 21 +++++ salt/elasticsearch/files/ingest/bro_tunnels | 18 ++++ salt/elasticsearch/files/ingest/bro_weird | 20 +++++ salt/elasticsearch/files/ingest/bro_x509 | 44 ++++++++++ salt/elasticsearch/files/ingest/common | 52 ++++++++++++ salt/elasticsearch/files/ingest/common_nids | 17 ++++ salt/elasticsearch/files/ingest/ossec | 53 ++++++++++++ salt/elasticsearch/files/ingest/sguild_nids | 25 ++++++ salt/elasticsearch/files/ingest/snort | 21 +++++ .../files/so-elasticsearch-pipelines | 52 ++++++++++++ salt/elasticsearch/init.sls | 4 + salt/logstash/conf/conf.enabled.txt.parser | 85 ++++++++++--------- salt/logstash/conf/conf.enabled.txt.so-eval | 77 ++++++++--------- salt/logstash/conf/conf.enabled.txt.storage | 77 ++++++++--------- .../files/dynamic/9000_output_bro.conf | 1 + so-setup-network.sh | 10 +-- 48 files changed, 1375 insertions(+), 123 deletions(-) create mode 100644 salt/elasticsearch/files/ingest/bro_common create mode 100644 salt/elasticsearch/files/ingest/bro_common_ssl create mode 100644 salt/elasticsearch/files/ingest/bro_conn create mode 100644 salt/elasticsearch/files/ingest/bro_dce_rpc create mode 100644 salt/elasticsearch/files/ingest/bro_dhcp create mode 100644 salt/elasticsearch/files/ingest/bro_dnp3 create mode 100644 salt/elasticsearch/files/ingest/bro_dns create mode 100644 salt/elasticsearch/files/ingest/bro_dpd create mode 100644 salt/elasticsearch/files/ingest/bro_files create mode 100644 salt/elasticsearch/files/ingest/bro_ftp create mode 100644 salt/elasticsearch/files/ingest/bro_http create mode 100644 salt/elasticsearch/files/ingest/bro_intel create mode 100644 salt/elasticsearch/files/ingest/bro_irc create mode 100644 salt/elasticsearch/files/ingest/bro_kerberos create mode 100644 salt/elasticsearch/files/ingest/bro_modbus create mode 100644 salt/elasticsearch/files/ingest/bro_mysql create mode 100644 salt/elasticsearch/files/ingest/bro_notice create mode 100644 salt/elasticsearch/files/ingest/bro_ntlm create mode 100644 salt/elasticsearch/files/ingest/bro_pe create mode 100644 salt/elasticsearch/files/ingest/bro_radius create mode 100644 salt/elasticsearch/files/ingest/bro_rdp create mode 100644 salt/elasticsearch/files/ingest/bro_rfb create mode 100644 salt/elasticsearch/files/ingest/bro_signatures create mode 100644 salt/elasticsearch/files/ingest/bro_sip create mode 100644 salt/elasticsearch/files/ingest/bro_smb_files create mode 100644 salt/elasticsearch/files/ingest/bro_smb_mapping create mode 100644 salt/elasticsearch/files/ingest/bro_smtp create mode 100644 salt/elasticsearch/files/ingest/bro_snmp create mode 100644 salt/elasticsearch/files/ingest/bro_socks create mode 100644 salt/elasticsearch/files/ingest/bro_software create mode 100644 salt/elasticsearch/files/ingest/bro_ssh create mode 100644 salt/elasticsearch/files/ingest/bro_ssl create mode 100644 salt/elasticsearch/files/ingest/bro_syslog create mode 100644 salt/elasticsearch/files/ingest/bro_tunnels create mode 100644 salt/elasticsearch/files/ingest/bro_weird create mode 100644 salt/elasticsearch/files/ingest/bro_x509 create mode 100644 salt/elasticsearch/files/ingest/common create mode 100644 salt/elasticsearch/files/ingest/common_nids create mode 100644 salt/elasticsearch/files/ingest/ossec create mode 100644 salt/elasticsearch/files/ingest/sguild_nids create mode 100644 salt/elasticsearch/files/ingest/snort create mode 100755 salt/elasticsearch/files/so-elasticsearch-pipelines diff --git a/salt/elasticsearch/files/ingest/bro_common b/salt/elasticsearch/files/ingest/bro_common new file mode 100644 index 000000000..98618ce56 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_common @@ -0,0 +1,9 @@ +{ + "description" : "bro_common", + "processors" : [ + { "rename": { "field": "@timestamp", "target_field": "timestamp", "ignore_missing": true } }, + { "date": { "field": "message2.ts", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "ignore_failure": true } }, + { "remove": { "field": "message2.ts", "ignore_failure": true } }, + { "pipeline": { "name": "common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_common_ssl b/salt/elasticsearch/files/ingest/bro_common_ssl new file mode 100644 index 000000000..faf1666ac --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_common_ssl @@ -0,0 +1,58 @@ +{ + "description" : "bro_common_ssl", + "processors" : [ + { + "kv": { + "field": "certificate_issuer", + "field_split": ",", + "value_split": "=", + "ignore_missing": true, + "ignore_failure": true, + "include_keys": [ "CN", "C", "O", "OU", "ST", "SN", "L", "DC", "GN", "pseudonym", "serialNumber", "title", "initials" ] + } + }, + { "rename":{ "field": "CN", "target_field": "issuer_common_name", "ignore_failure": true } }, + { "rename":{ "field": "C", "target_field": "issuer_country_code", "ignore_failure": true } }, + { "rename":{ "field": "O", "target_field": "issuer_organization", "ignore_failure": true } }, + { "rename":{ "field": "OU", "target_field": "issuer_organization_unit", "ignore_failure": true } }, + { "rename":{ "field": "ST", "target_field": "issuer_state", "ignore_failure": true } }, + { "rename":{ "field": "SN", "target_field": "issuer_surname", "ignore_failure": true } }, + { "rename":{ "field": "L", "target_field": "issuer_locality", "ignore_failure": true } }, + { "rename":{ "field": "DC", "target_field": "issuer_distinguised_name", "ignore_failure": true } }, + { "rename":{ "field": "GN", "target_field": "issuer_given_name", "ignore_failure": true } }, + { "rename":{ "field": "pseudonym", "target_field": "issuer_pseudonym", "ignore_failure": true } }, + { "rename":{ "field": "serialNumber", "target_field": "issuer_serial_number", "ignore_failure": true } }, + { "rename":{ "field": "title", "target_field": "issuer_title", "ignore_failure": true } }, + { "rename":{ "field": "initials", "target_field": "issuer_initials", "ignore_failure": true } }, + { + "kv": { + "field": "certificate_subject", + "field_split": ",", + "value_split": "=", + "ignore_missing": true, + "ignore_failure": true, + "include_keys": [ "CN", "C", "O", "OU", "ST", "SN", "L", "GN", "pseudonym", "serialNumber", "title", "initials" ] + } + }, + { "rename":{ "field": "CN", "target_field": "certificate_common_name", "ignore_failure": true } }, + { "rename":{ "field": "C", "target_field": "certificate_country_code", "ignore_failure": true } }, + { "rename":{ "field": "O", "target_field": "certificate_organization", "ignore_failure": true } }, + { "rename":{ "field": "OU", "target_field": "certificate_organization_unit","ignore_failure": true } }, + { "rename":{ "field": "ST", "target_field": "certificate_state", "ignore_failure": true } }, + { "rename":{ "field": "SN", "target_field": "certificate_surname", "ignore_failure": true } }, + { "rename":{ "field": "L", "target_field": "certificate_locality", "ignore_failure": true } }, + { "rename":{ "field": "GN", "target_field": "certificate_given_name", "ignore_failure": true } }, + { "rename":{ "field": "pseudonym", "target_field": "certificate_pseudonym", "ignore_failure": true } }, + { "rename":{ "field": "serialNumber", "target_field": "certificate_serial_number", "ignore_failure": true } }, + { "rename":{ "field": "title", "target_field": "certificate_title", "ignore_failure": true } }, + { "rename":{ "field": "initials", "target_field": "certificate_initials", "ignore_failure": true } }, + { "script":{ "lang": "painless", "source": "ctx.certificate_common_name_length = ctx.certificate_common_name.length()", "ignore_failure": true } }, + { "script":{ "lang": "painless", "source": "ctx.issuer_common_name_length = ctx.issuer_common_name.length()", "ignore_failure": true } }, + { "script":{ "lang": "painless", "source": "ctx.server_name_length = ctx.server_name.length()", "ignore_failure": true } }, + { + "pipeline": { + "name": "bro_common" + } + } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_conn b/salt/elasticsearch/files/ingest/bro_conn new file mode 100644 index 000000000..b12be156e --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_conn @@ -0,0 +1,48 @@ +{ + "description" : "bro_conn", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.proto", "target_field": "protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.service", "target_field": "service", "ignore_missing": true } }, + { "rename": { "field": "message2.duration", "target_field": "duration", "ignore_missing": true } }, + { "rename": { "field": "message2.orig_bytes", "target_field": "original_bytes", "ignore_missing": true } }, + { "rename": { "field": "message2.resp_bytes", "target_field": "respond_bytes", "ignore_missing": true } }, + { "rename": { "field": "message2.conn_state", "target_field": "connection_state", "ignore_missing": true } }, + { "rename": { "field": "message2.local_orig", "target_field": "local_orig", "ignore_missing": true } }, + { "rename": { "field": "message2.local_resp", "target_field": "local_respond", "ignore_missing": true } }, + { "rename": { "field": "message2.missed_bytes", "target_field": "missed_bytes", "ignore_missing": true } }, + { "rename": { "field": "message2.history", "target_field": "history", "ignore_missing": true } }, + { "rename": { "field": "message2.orig_pkts", "target_field": "original_packets", "ignore_missing": true } }, + { "rename": { "field": "message2.orig_ip_bytes", "target_field": "original_ip_bytes", "ignore_missing": true } }, + { "rename": { "field": "message2.resp_pkts", "target_field": "respond_packets", "ignore_missing": true } }, + { "rename": { "field": "message2.resp_ip_bytes", "target_field": "respond_ip_bytes", "ignore_missing": true } }, + { "rename": { "field": "message2.tunnel_parents", "target_field": "tunnel_parents", "ignore_missing": true } }, + { "rename": { "field": "message2.orig_cc", "target_field": "original_country_code","ignore_missing": true } }, + { "rename": { "field": "message2.resp_cc", "target_field": "respond_country_code", "ignore_missing": true } }, + { "rename": { "field": "message2.sensorname", "target_field": "sensor_name", "ignore_missing": true } }, + { "script": { "lang": "painless", "source": "ctx.total_bytes = (ctx.original_bytes + ctx.respond_bytes)", "ignore_failure": true } }, + { "set": { "if": "ctx.connection_state == 'S0'", "field": "connection_state_description", "value": "Connection attempt seen, no reply" } }, + { "set": { "if": "ctx.connection_state == 'S1'", "field": "connection_state_description", "value": "Connection established, not terminated" } }, + { "set": { "if": "ctx.connection_state == 'S2'", "field": "connection_state_description", "value": "Connection established and close attempt by originator seen (but no reply from responder)" } }, + { "set": { "if": "ctx.connection_state == 'S3'", "field": "connection_state_description", "value": "Connection established and close attempt by responder seen (but no reply from originator)" } }, + { "set": { "if": "ctx.connection_state == 'SF'", "field": "connection_state_description", "value": "Normal SYN/FIN completion" } }, + { "set": { "if": "ctx.connection_state == 'REJ'", "field": "connection_state_description", "value": "Connection attempt rejected" } }, + { "set": { "if": "ctx.connection_state == 'RSTO'", "field": "connection_state_description", "value": "Connection established, originator aborted (sent a RST)" } }, + { "set": { "if": "ctx.connection_state == 'RSTR'", "field": "connection_state_description", "value": "Established, responder aborted" } }, + { "set": { "if": "ctx.connection_state == 'RSTOS0'","field": "connection_state_description", "value": "Originator sent a SYN followed by a RST, we never saw a SYN-ACK from the responder" } }, + { "set": { "if": "ctx.connection_state == 'RSTRH'", "field": "connection_state_description", "value": "Responder sent a SYN ACK followed by a RST, we never saw a SYN from the (purported) originator" } }, + { "set": { "if": "ctx.connection_state == 'SH'", "field": "connection_state_description", "value": "Originator sent a SYN followed by a FIN, we never saw a SYN ACK from the responder (hence the connection was 'half' open)" } }, + { "set": { "if": "ctx.connection_state == 'SHR'", "field": "connection_state_description", "value": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator" } }, + { "set": { "if": "ctx.connection_state == 'OTH'", "field": "connection_state_description", "value": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)" } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_dce_rpc b/salt/elasticsearch/files/ingest/bro_dce_rpc new file mode 100644 index 000000000..105905245 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_dce_rpc @@ -0,0 +1,20 @@ +{ + "description" : "bro_dce_rpc", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.rtt", "target_field": "rtt", "ignore_missing": true } }, + { "rename": { "field": "message2.named_pipe", "target_field": "named_pipe", "ignore_missing": true } }, + { "rename": { "field": "message2.endpoint", "target_field": "endpoint", "ignore_missing": true } }, + { "rename": { "field": "message2.operation", "target_field": "operation", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_dhcp b/salt/elasticsearch/files/ingest/bro_dhcp new file mode 100644 index 000000000..010d0f85b --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_dhcp @@ -0,0 +1,20 @@ +{ + "description" : "bro_dhcp", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uids", "target_field": "uid", "ignore_missing": true } }, + { "rename": { "field": "message2.mac", "target_field": "mac", "ignore_missing": true } }, + { "rename": { "field": "message2.assigned_ip", "target_field": "assigned_ip", "ignore_missing": true } }, + { "rename": { "field": "message2.lease_time", "target_field": "lease_time", "ignore_missing": true } }, + { "rename": { "field": "message2.trans_id", "target_field": "transaction_id", "ignore_missing": true } }, + { "rename": { "field": "message2.assigned_addr", "target_field": "assigned_ip", "ignore_missing": true } }, + { "rename": { "field": "message2.client_addr", "target_field": "source_ip", "ignore_missing": true } }, + { "rename": { "field": "message2.server_addr", "target_field": "destination_ip", "ignore_missing": true } }, + { "rename": { "field": "message2.requested_addr", "target_field": "requested_ip", "ignore_missing": true } }, + { "rename": { "field": "message2.domain", "target_field": "domain_name", "ignore_missing": true } }, + { "rename": { "field": "message2.host_name", "target_field": "hostname", "ignore_missing": true } }, + { "rename": { "field": "message2.duration", "target_field": "duration", "ignore_missing": true } }, + { "rename": { "field": "message2.msg_types", "target_field": "message_types", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_dnp3 b/salt/elasticsearch/files/ingest/bro_dnp3 new file mode 100644 index 000000000..bebb85ecb --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_dnp3 @@ -0,0 +1,19 @@ +{ + "description" : "bro_dnp3", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.fc_request", "target_field": "fc_request", "ignore_missing": true } }, + { "rename": { "field": "message2.fc_reply", "target_field": "fc_reply", "ignore_missing": true } }, + { "rename": { "field": "message2.iin", "target_field": "iin", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_dns b/salt/elasticsearch/files/ingest/bro_dns new file mode 100644 index 000000000..be8d59294 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_dns @@ -0,0 +1,35 @@ +{ + "description" : "bro_dns", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.proto", "target_field": "protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.trans_id", "target_field": "transaction_id", "ignore_missing": true } }, + { "rename": { "field": "message2.rtt", "target_field": "rtt", "ignore_missing": true } }, + { "rename": { "field": "message2.query", "target_field": "query", "ignore_missing": true } }, + { "rename": { "field": "message2.qclass", "target_field": "query_class", "ignore_missing": true } }, + { "rename": { "field": "message2.qclass_name", "target_field": "query_class_name", "ignore_missing": true } }, + { "rename": { "field": "message2.qtype", "target_field": "query_type", "ignore_missing": true } }, + { "rename": { "field": "message2.qtype_name", "target_field": "query_type_name", "ignore_missing": true } }, + { "rename": { "field": "message2.rcode", "target_field": "rcode", "ignore_missing": true } }, + { "rename": { "field": "message2.rcode_name", "target_field": "rcode_name", "ignore_missing": true } }, + { "rename": { "field": "message2.AA", "target_field": "aa", "ignore_missing": true } }, + { "rename": { "field": "message2.TC", "target_field": "tc", "ignore_missing": true } }, + { "rename": { "field": "message2.RD", "target_field": "rd", "ignore_missing": true } }, + { "rename": { "field": "message2.RA", "target_field": "ra", "ignore_missing": true } }, + { "rename": { "field": "message2.Z", "target_field": "z", "ignore_missing": true } }, + { "rename": { "field": "message2.answers", "target_field": "answers", "ignore_missing": true } }, + { "rename": { "field": "message2.TTLs", "target_field": "ttls", "ignore_missing": true } }, + { "rename": { "field": "message2.rejected", "target_field": "rejected", "ignore_missing": true } }, + { "script": { "lang": "painless", "source": "ctx.query_length = ctx.query.length()", "ignore_failure": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_dpd b/salt/elasticsearch/files/ingest/bro_dpd new file mode 100644 index 000000000..caf66d39e --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_dpd @@ -0,0 +1,19 @@ +{ + "description" : "bro_dpd", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.proto", "target_field": "protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.analyzer", "target_field": "analyzer", "ignore_missing": true } }, + { "rename": { "field": "message2.failure_reason", "target_field": "failure_reason", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_files b/salt/elasticsearch/files/ingest/bro_files new file mode 100644 index 000000000..4337b75f2 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_files @@ -0,0 +1,32 @@ +{ + "description" : "bro_files", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.fuid", "target_field": "fuid", "ignore_missing": true } }, + { "rename": { "field": "message2.tx_hosts", "target_field": "file_ip", "ignore_missing": true } }, + { "rename": { "field": "message2.rx_hosts.0", "target_field": "destination_ip", "ignore_missing": true } }, + { "remove": { "field": "message2.rx_hosts", "ignore_missing": true } }, + { "rename": { "field": "message2.conn_uids", "target_field": "uid", "ignore_missing": true } }, + { "remove": { "field": "source", "ignore_missing": true } }, + { "rename": { "field": "message2.source", "target_field": "source", "ignore_missing": true } }, + { "rename": { "field": "message2.depth", "target_field": "depth", "ignore_missing": true } }, + { "rename": { "field": "message2.analyzers", "target_field": "analyzer", "ignore_missing": true } }, + { "rename": { "field": "message2.mime_type", "target_field": "mimetype", "ignore_missing": true } }, + { "rename": { "field": "message2.filename", "target_field": "file_name", "ignore_missing": true } }, + { "rename": { "field": "message2.duration", "target_field": "duration", "ignore_missing": true } }, + { "rename": { "field": "message2.local_orig", "target_field": "local_orig", "ignore_missing": true } }, + { "rename": { "field": "message2.is_orig", "target_field": "is_orig", "ignore_missing": true } }, + { "rename": { "field": "message2.seen_bytes", "target_field": "seen_bytes", "ignore_missing": true } }, + { "rename": { "field": "message2.total_bytes", "target_field": "total_bytes", "ignore_missing": true } }, + { "rename": { "field": "message2.missing_bytes", "target_field": "missing_bytes", "ignore_missing": true } }, + { "rename": { "field": "message2.overflow_bytes", "target_field": "overflow_bytes", "ignore_missing": true } }, + { "rename": { "field": "message2.timedout", "target_field": "timed_out", "ignore_missing": true } }, + { "rename": { "field": "message2.parent_fuid", "target_field": "parent_fuid", "ignore_missing": true } }, + { "rename": { "field": "message2.md5", "target_field": "md5", "ignore_missing": true } }, + { "rename": { "field": "message2.sha1", "target_field": "sha1", "ignore_missing": true } }, + { "rename": { "field": "message2.extracted", "target_field": "extracted", "ignore_missing": true } }, + { "rename": { "field": "message2.extracted_cutoff", "target_field": "extracted_cutoff", "ignore_missing": true } }, + { "rename": { "field": "message2.extracted_size", "target_field": "extracted_size", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_ftp b/salt/elasticsearch/files/ingest/bro_ftp new file mode 100644 index 000000000..34775072d --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_ftp @@ -0,0 +1,33 @@ +{ + "description" : "bro_http", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.user", "target_field": "username", "ignore_missing": true } }, + { "rename": { "field": "message2.password", "target_field": "password", "ignore_missing": true } }, + { "rename": { "field": "message2.command", "target_field": "ftp_command", "ignore_missing": true } }, + { "rename": { "field": "message2.arg", "target_field": "ftp_argument", "ignore_missing": true } }, + { "rename": { "field": "message2.mime_type", "target_field": "mimetype", "ignore_missing": true } }, + { "rename": { "field": "message2.file_size", "target_field": "file_size", "ignore_missing": true } }, + { "rename": { "field": "message2.reply_code", "target_field": "reply_code", "ignore_missing": true } }, + { "rename": { "field": "message2.reply_msg", "target_field": "reply_message", "ignore_missing": true } }, + { "dot_expander": { "field": "data_channel.passive", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.data_channel.passive","target_field": "data_channel_passive", "ignore_missing": true } }, + { "dot_expander": { "field": "data_channel.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.data_channel.orig_h","target_field": "data_channel_source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "data_channel.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.data_channel.resp_h","target_field": "data_channel_destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "data_channel.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.data_channel.resp_p","target_field": "data_channel_destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.fuid", "target_field": "fuid", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_http b/salt/elasticsearch/files/ingest/bro_http new file mode 100644 index 000000000..842a12bc9 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_http @@ -0,0 +1,42 @@ +{ + "description" : "bro_http", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.trans_depth", "target_field": "trans_depth", "ignore_missing": true } }, + { "rename": { "field": "message2.method", "target_field": "method", "ignore_missing": true } }, + { "rename": { "field": "message2.host", "target_field": "virtual_host", "ignore_missing": true } }, + { "rename": { "field": "message2.uri", "target_field": "uri", "ignore_missing": true } }, + { "rename": { "field": "message2.referrer", "target_field": "referrer", "ignore_missing": true } }, + { "rename": { "field": "message2.version", "target_field": "version", "ignore_missing": true } }, + { "rename": { "field": "message2.user_agent", "target_field": "useragent", "ignore_missing": true } }, + { "rename": { "field": "message2.request_body_len", "target_field": "request_body_length", "ignore_missing": true } }, + { "rename": { "field": "message2.response_body_len","target_field": "response_body_length", "ignore_missing": true } }, + { "rename": { "field": "message2.status_code", "target_field": "status_code", "ignore_missing": true } }, + { "rename": { "field": "message2.status_msg", "target_field": "status_message", "ignore_missing": true } }, + { "rename": { "field": "message2.info_code", "target_field": "info_code", "ignore_missing": true } }, + { "rename": { "field": "message2.info_msg", "target_field": "info_message", "ignore_missing": true } }, + { "remove": { "field": "message2.tags", "ignore_failure": true } }, + { "rename": { "field": "message2.username", "target_field": "user", "ignore_missing": true } }, + { "rename": { "field": "message2.password", "target_field": "password", "ignore_missing": true } }, + { "rename": { "field": "message2.proxied", "target_field": "proxied", "ignore_missing": true } }, + { "rename": { "field": "message2.orig_fuids", "target_field": "orig_fuids", "ignore_missing": true } }, + { "rename": { "field": "message2.orig_filenames", "target_field": "orig_filenames", "ignore_missing": true } }, + { "rename": { "field": "message2.orig_mime_types", "target_field": "orig_mime_types", "ignore_missing": true } }, + { "rename": { "field": "message2.resp_fuids", "target_field": "resp_fuids", "ignore_missing": true } }, + { "rename": { "field": "message2.resp_filenames", "target_field": "resp_filenames", "ignore_missing": true } }, + { "rename": { "field": "message2.resp_mime_types", "target_field": "resp_mime_types", "ignore_missing": true } }, + { "script": { "lang": "painless", "source": "ctx.uri_length = ctx.uri.length()", "ignore_failure": true } }, + { "script": { "lang": "painless", "source": "ctx.useragent_length = ctx.useragent.length()", "ignore_failure": true } }, + { "script": { "lang": "painless", "source": "ctx.virtual_host_length = ctx.virtual_host.length()", "ignore_failure": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_intel b/salt/elasticsearch/files/ingest/bro_intel new file mode 100644 index 000000000..20bf90c5a --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_intel @@ -0,0 +1,29 @@ +{ + "description" : "bro_intel", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "dot_expander": { "field": "seen.indicator", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.seen.indicator", "target_field": "indicator", "ignore_missing": true } }, + { "dot_expander": { "field": "seen.indicator_type", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.seen.indicator_type", "target_field": "indicator_type", "ignore_missing": true } }, + { "dot_expander": { "field": "seen.where", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.seen.where", "target_field": "seen_where", "ignore_missing": true } }, + { "dot_expander": { "field": "seen.node", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.seen.node", "target_field": "seen_node", "ignore_missing": true } }, + { "rename": { "field": "message2.matched", "target_field": "matched", "ignore_missing": true } }, + { "rename": { "field": "message2.sources", "target_field": "sources", "ignore_missing": true } }, + { "rename": { "field": "message2.fuid", "target_field": "fuid", "ignore_missing": true } }, + { "rename": { "field": "message2.file_mime_type", "target_field": "mimetype", "ignore_missing": true } }, + { "rename": { "field": "message2.file_desc", "target_field": "file_description", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_irc b/salt/elasticsearch/files/ingest/bro_irc new file mode 100644 index 000000000..c2a5ba22d --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_irc @@ -0,0 +1,25 @@ +{ + "description" : "bro_irc", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.nick", "target_field": "nick", "ignore_missing": true } }, + { "rename": { "field": "message2.user", "target_field": "irc_username", "ignore_missing": true } }, + { "rename": { "field": "message2.command", "target_field": "irc_command", "ignore_missing": true } }, + { "rename": { "field": "message2.value", "target_field": "value", "ignore_missing": true } }, + { "rename": { "field": "message2.addl", "target_field": "additional_info", "ignore_missing": true } }, + { "rename": { "field": "message2.dcc_file_name", "target_field": "dcc_file_name", "ignore_missing": true } }, + { "rename": { "field": "message2.dcc_file_size", "target_field": "dcc_file_size", "ignore_missing": true } }, + { "rename": { "field": "message2.dcc_mime_type", "target_field": "dcc_mime_type", "ignore_missing": true } }, + { "rename": { "field": "message2.fuid", "target_field": "fuid", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_kerberos b/salt/elasticsearch/files/ingest/bro_kerberos new file mode 100644 index 000000000..b338b5c96 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_kerberos @@ -0,0 +1,30 @@ +{ + "description" : "bro_kerberos", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.request_type", "target_field": "request_type", "ignore_missing": true } }, + { "rename": { "field": "message2.client", "target_field": "client", "ignore_missing": true } }, + { "rename": { "field": "message2.service", "target_field": "service", "ignore_missing": true } }, + { "rename": { "field": "message2.success", "target_field": "kerberos_success", "ignore_missing": true } }, + { "rename": { "field": "message2.error_msg", "target_field": "error_message", "ignore_missing": true } }, + { "rename": { "field": "message2.from", "target_field": "valid_from", "ignore_missing": true } }, + { "rename": { "field": "message2.till", "target_field": "valid_till", "ignore_missing": true } }, + { "rename": { "field": "message2.cipher", "target_field": "cipher", "ignore_missing": true } }, + { "rename": { "field": "message2.forwardable", "target_field": "forwardable", "ignore_missing": true } }, + { "rename": { "field": "message2.renewable", "target_field": "renewable", "ignore_missing": true } }, + { "rename": { "field": "message2.client_cert_subject", "target_field": "client_certificate_subject", "ignore_missing": true } }, + { "rename": { "field": "message2.client_cert_fuid", "target_field": "client_certificate_fuid", "ignore_missing": true } }, + { "rename": { "field": "message2.server_cert_subject", "target_field": "server_certificate_subject", "ignore_missing": true } }, + { "rename": { "field": "message2.server_cert_fuid", "target_field": "server_certificate_fuid", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_modbus b/salt/elasticsearch/files/ingest/bro_modbus new file mode 100644 index 000000000..10e7c271a --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_modbus @@ -0,0 +1,18 @@ +{ + "description" : "bro_modbus", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.func", "target_field": "function", "ignore_missing": true } }, + { "rename": { "field": "message2.exception", "target_field": "exception", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_mysql b/salt/elasticsearch/files/ingest/bro_mysql new file mode 100644 index 000000000..a01d57da2 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_mysql @@ -0,0 +1,21 @@ +{ + "description" : "bro_mysql", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.cmd", "target_field": "mysql_command", "ignore_missing": true } }, + { "rename": { "field": "message2.arg", "target_field": "mysql_argument", "ignore_missing": true } }, + { "rename": { "field": "message2.success", "target_field": "mysql_success", "ignore_missing": true } }, + { "rename": { "field": "message2.rows", "target_field": "rows", "ignore_missing": true } }, + { "rename": { "field": "message2.response", "target_field": "response", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_notice b/salt/elasticsearch/files/ingest/bro_notice new file mode 100644 index 000000000..6e43448d5 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_notice @@ -0,0 +1,36 @@ +{ + "description" : "bro_notice", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "remove": { "field": "message2.dst", "ignore_failure": true } }, + { "remove": { "field": "message2.src", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.fuid", "target_field": "fuid", "ignore_missing": true } }, + { "rename": { "field": "message2.mime", "target_field": "file_mime_type", "ignore_missing": true } }, + { "rename": { "field": "message2.desc", "target_field": "file_description", "ignore_missing": true } }, + { "rename": { "field": "message2.proto", "target_field": "protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.note", "target_field": "note", "ignore_missing": true } }, + { "rename": { "field": "message2.msg", "target_field": "msg", "ignore_missing": true } }, + { "rename": { "field": "message2.sub", "target_field": "sub_msg", "ignore_missing": true } }, + { "rename": { "field": "message2.p", "target_field": "p", "ignore_missing": true } }, + { "rename": { "field": "message2.n", "target_field": "n", "ignore_missing": true } }, + { "rename": { "field": "message2.peer_descr", "target_field": "peer_description", "ignore_missing": true } }, + { "rename": { "field": "message2.actions", "target_field": "action", "ignore_missing": true } }, + { "rename": { "field": "message2.suppress_for", "target_field": "suppress_for", "ignore_missing": true } }, + { "rename": { "field": "message2.dropped", "target_field": "dropped", "ignore_missing": true } }, + { "rename": { "field": "message2.destination_country_code", "target_field": "destination_country_code", "ignore_missing": true } }, + { "rename": { "field": "message2.destination_region", "target_field": "destination_region", "ignore_missing": true } }, + { "rename": { "field": "message2.destination_city", "target_field": "destination_city", "ignore_missing": true } }, + { "rename": { "field": "message2.destination_latitude", "target_field": "destination_latitude", "ignore_missing": true } }, + { "rename": { "field": "message2.destination_longitude", "target_field": "destination_longitude", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_ntlm b/salt/elasticsearch/files/ingest/bro_ntlm new file mode 100644 index 000000000..a3d130343 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_ntlm @@ -0,0 +1,24 @@ +{ + "description" : "bro_ntlm", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.hostname", "target_field": "hostname", "ignore_missing": true } }, + { "rename": { "field": "message2.domainname", "target_field": "domain_name", "ignore_missing": true } }, + { "rename": { "field": "message2.success", "target_field": "ntlm_success", "ignore_missing": true } }, + { "rename": { "field": "message2.status", "target_field": "status", "ignore_missing": true } }, + { "rename": { "field": "message2.username", "target_field": "username", "ignore_missing": true } }, + { "rename": { "field": "message2.server_dns_computer_name", "target_field": "server_dns_computer_name", "ignore_missing": true } }, + { "rename": { "field": "message2.server_nb_computer_name", "target_field": "server_nb_computer_name", "ignore_missing": true } }, + { "rename": { "field": "message2.server_tree_name", "target_field": "server_tree_name", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_pe b/salt/elasticsearch/files/ingest/bro_pe new file mode 100644 index 000000000..2597d3e26 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_pe @@ -0,0 +1,23 @@ +{ + "description" : "bro_pe", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id", "target_field": "fuid", "ignore_missing": true } }, + { "rename": { "field": "message2.machine", "target_field": "machine", "ignore_missing": true } }, + { "rename": { "field": "message2.compile_ts", "target_field": "compile_ts", "ignore_missing": true } }, + { "rename": { "field": "message2.os", "target_field": "os", "ignore_missing": true } }, + { "rename": { "field": "message2.subsystem", "target_field": "subsystem", "ignore_missing": true } }, + { "rename": { "field": "message2.is_exe", "target_field": "is_exe", "ignore_missing": true } }, + { "rename": { "field": "message2.is_64bit", "target_field": "is_64bit", "ignore_missing": true } }, + { "rename": { "field": "message2.uses_aslr", "target_field": "uses_aslr", "ignore_missing": true } }, + { "rename": { "field": "message2.uses_dep", "target_field": "uses_dep", "ignore_missing": true } }, + { "rename": { "field": "message2.uses_code_integrity","target_field": "uses_code_integrity","ignore_missing": true } }, + { "rename": { "field": "message2.uses_seh", "target_field": "uses_seh", "ignore_missing": true } }, + { "rename": { "field": "message2.has_import_table", "target_field": "has_import_table", "ignore_missing": true } }, + { "rename": { "field": "message2.has_export_table", "target_field": "has_export_table", "ignore_missing": true } }, + { "rename": { "field": "message2.has_cert_table", "target_field": "has_cert_table", "ignore_missing": true } }, + { "rename": { "field": "message2.has_debug_data", "target_field": "has_debug_data", "ignore_missing": true } }, + { "rename": { "field": "message2.section_names", "target_field": "section_names", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_radius b/salt/elasticsearch/files/ingest/bro_radius new file mode 100644 index 000000000..c333711d6 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_radius @@ -0,0 +1,25 @@ +{ + "description" : "bro_radius", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.username", "target_field": "username", "ignore_missing": true } }, + { "rename": { "field": "message2.mac", "target_field": "mac", "ignore_missing": true } }, + { "rename": { "field": "message2.framed_addr", "target_field": "framed_addr", "ignore_missing": true } }, + { "rename": { "field": "message2.remote_ip", "target_field": "remote_ip", "ignore_missing": true } }, + { "rename": { "field": "message2.connect_info", "target_field": "connect_info", "ignore_missing": true } }, + { "rename": { "field": "message2.reply_msg", "target_field": "reply_message", "ignore_missing": true } }, + { "rename": { "field": "message2.result", "target_field": "result", "ignore_missing": true } }, + { "remove": { "field": "message2.ttl", "ignore_failure": true } }, + { "rename": { "field": "message2.logged", "target_field": "logged", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_rdp b/salt/elasticsearch/files/ingest/bro_rdp new file mode 100644 index 000000000..b3cf206a5 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_rdp @@ -0,0 +1,31 @@ +{ + "description" : "bro_rdp", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.cookie", "target_field": "cookie", "ignore_missing": true } }, + { "rename": { "field": "message2.result", "target_field": "result", "ignore_missing": true } }, + { "rename": { "field": "message2.security_protocol","target_field": "security_protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.keyboard_layout", "target_field": "keyboard_layout", "ignore_missing": true } }, + { "rename": { "field": "message2.client_build", "target_field": "client_build", "ignore_missing": true } }, + { "rename": { "field": "message2.client_name", "target_field": "client_name", "ignore_missing": true } }, + { "rename": { "field": "message2.client_dig_product_id", "target_field": "client_digital_product_id", "ignore_missing": true } }, + { "rename": { "field": "message2.desktop_width", "target_field": "desktop_width", "ignore_missing": true } }, + { "rename": { "field": "message2.desktop_height", "target_field": "desktop_height", "ignore_missing": true } }, + { "rename": { "field": "message2.requested_color_depth", "target_field": "requested_color_depth", "ignore_missing": true } }, + { "rename": { "field": "message2.cert_type", "target_field": "certificate_type", "ignore_missing": true } }, + { "rename": { "field": "message2.cert_count", "target_field": "certificate_count", "ignore_missing": true } }, + { "rename": { "field": "message2.cert_permanent", "target_field": "certificate_permanent","ignore_missing": true } }, + { "rename": { "field": "message2.encryption_level", "target_field": "encryption_level", "ignore_missing": true } }, + { "rename": { "field": "message2.encryption_method","target_field": "encryption_method", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_rfb b/salt/elasticsearch/files/ingest/bro_rfb new file mode 100644 index 000000000..8f3cc86e7 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_rfb @@ -0,0 +1,26 @@ +{ + "description" : "bro_rfb", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.client_major_version", "target_field": "client_major_version", "ignore_missing": true } }, + { "rename": { "field": "message2.client_minor_version", "target_field": "client_minor_version", "ignore_missing": true } }, + { "rename": { "field": "message2.server_major_version", "target_field": "server_major_version", "ignore_missing": true } }, + { "rename": { "field": "message2.server_minor_version", "target_field": "server_minor_version", "ignore_missing": true } }, + { "rename": { "field": "message2.authentication_method", "target_field": "authentication_method","ignore_missing": true } }, + { "rename": { "field": "message2.auth", "target_field": "auth", "ignore_missing": true } }, + { "rename": { "field": "message2.share_flag", "target_field": "share_flag", "ignore_missing": true } }, + { "rename": { "field": "message2.desktop_name", "target_field": "desktop_name", "ignore_missing": true } }, + { "rename": { "field": "message2.width", "target_field": "width", "ignore_missing": true } }, + { "rename": { "field": "message2.height", "target_field": "height", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_signatures b/salt/elasticsearch/files/ingest/bro_signatures new file mode 100644 index 000000000..5dd3d9924 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_signatures @@ -0,0 +1,22 @@ +{ + "description" : "bro_signatures", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.note", "target_field": "note", "ignore_missing": true } }, + { "rename": { "field": "message2.sig_id", "target_field": "signature_id", "ignore_missing": true } }, + { "rename": { "field": "message2.event_msg", "target_field": "event_message", "ignore_missing": true } }, + { "rename": { "field": "message2.sub_msg", "target_field": "sub_message", "ignore_missing": true } }, + { "rename": { "field": "message2.sig_count", "target_field": "signature_count", "ignore_missing": true } }, + { "rename": { "field": "message2.host_count", "target_field": "host_count", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_sip b/salt/elasticsearch/files/ingest/bro_sip new file mode 100644 index 000000000..3a8b00d62 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_sip @@ -0,0 +1,37 @@ +{ + "description" : "bro_sip", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.trans_depth", "target_field": "trans_depth", "ignore_missing": true } }, + { "rename": { "field": "message2.method", "target_field": "method", "ignore_missing": true } }, + { "rename": { "field": "message2.uri", "target_field": "uri", "ignore_missing": true } }, + { "rename": { "field": "message2.date", "target_field": "date", "ignore_missing": true } }, + { "rename": { "field": "message2.request_from", "target_field": "request_from", "ignore_missing": true } }, + { "rename": { "field": "message2.request_to", "target_field": "request_to", "ignore_missing": true } }, + { "rename": { "field": "message2.response_from", "target_field": "response_from", "ignore_missing": true } }, + { "rename": { "field": "message2.response_to", "target_field": "response_to", "ignore_missing": true } }, + { "rename": { "field": "message2.reply_to", "target_field": "reply_to", "ignore_missing": true } }, + { "rename": { "field": "message2.call_id", "target_field": "call_id", "ignore_missing": true } }, + { "rename": { "field": "message2.seq", "target_field": "seq", "ignore_missing": true } }, + { "rename": { "field": "message2.subject", "target_field": "subject", "ignore_missing": true } }, + { "rename": { "field": "message2.request_path", "target_field": "request_path", "ignore_missing": true } }, + { "rename": { "field": "message2.response_path", "target_field": "response_path", "ignore_missing": true } }, + { "rename": { "field": "message2.user_agent", "target_field": "user_agent", "ignore_missing": true } }, + { "rename": { "field": "message2.status_code", "target_field": "status_code", "ignore_missing": true } }, + { "rename": { "field": "message2.status_msg", "target_field": "status_msg", "ignore_missing": true } }, + { "rename": { "field": "message2.warning", "target_field": "warning", "ignore_missing": true } }, + { "rename": { "field": "message2.request_body_len", "target_field": "request_body_length", "ignore_missing": true } }, + { "rename": { "field": "message2.response_body_len","target_field": "response_body_length", "ignore_missing": true } }, + { "rename": { "field": "message2.content_type", "target_field": "content_type", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_smb_files b/salt/elasticsearch/files/ingest/bro_smb_files new file mode 100644 index 000000000..83ba8bd67 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_smb_files @@ -0,0 +1,31 @@ +{ + "description" : "bro_smb_files", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.fuid", "target_field": "fuid", "ignore_missing": true } }, + { "rename": { "field": "message2.action", "target_field": "action", "ignore_missing": true } }, + { "remove": { "field": "path", "ignore_failure": true } }, + { "rename": { "field": "message2.path", "target_field": "path", "ignore_missing": true } }, + { "rename": { "field": "message2.name", "target_field": "name", "ignore_missing": true } }, + { "rename": { "field": "message2.size", "target_field": "size", "ignore_missing": true } }, + { "rename": { "field": "message2.prev_name", "target_field": "prev_name", "ignore_missing": true } }, + { "dot_expander": { "field": "times.modified", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.times.modified", "target_field": "times_modified", "ignore_missing": true } }, + { "dot_expander": { "field": "times.accessed", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.times.accessed", "target_field": "times_accessed", "ignore_missing": true } }, + { "dot_expander": { "field": "times.created", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.times.created", "target_field": "times_created", "ignore_missing": true } }, + { "dot_expander": { "field": "times.changed", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.times.changed", "target_field": "times_changed", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_smb_mapping b/salt/elasticsearch/files/ingest/bro_smb_mapping new file mode 100644 index 000000000..e1b6b5dfb --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_smb_mapping @@ -0,0 +1,21 @@ +{ + "description" : "bro_smb_files", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "remove": { "field": "path", "ignore_failure": true } }, + { "rename": { "field": "message2.path", "target_field": "path", "ignore_missing": true } }, + { "rename": { "field": "message2.service", "target_field": "service", "ignore_missing": true } }, + { "rename": { "field": "message2.native_file_system", "target_field": "native_file_system", "ignore_missing": true } }, + { "rename": { "field": "message2.share_type", "target_field": "share_type", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_smtp b/salt/elasticsearch/files/ingest/bro_smtp new file mode 100644 index 000000000..4bd85a293 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_smtp @@ -0,0 +1,38 @@ +{ + "description" : "bro_smtp", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "remove": { "field": "path", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.trans_depth", "target_field": "trans_depth", "ignore_missing": true } }, + { "rename": { "field": "message2.helo", "target_field": "helo", "ignore_missing": true } }, + { "rename": { "field": "message2.mailfrom", "target_field": "mail_from", "ignore_missing": true } }, + { "rename": { "field": "message2.rcptto", "target_field": "recipient_to", "ignore_missing": true } }, + { "rename": { "field": "message2.date", "target_field": "mail_date", "ignore_missing": true } }, + { "rename": { "field": "message2.from", "target_field": "from", "ignore_missing": true } }, + { "rename": { "field": "message2.to", "target_field": "to", "ignore_missing": true } }, + { "rename": { "field": "message2.cc", "target_field": "cc", "ignore_missing": true } }, + { "rename": { "field": "message2.reply_to", "target_field": "reply_to", "ignore_missing": true } }, + { "rename": { "field": "message2.msg_id", "target_field": "message_id", "ignore_missing": true } }, + { "rename": { "field": "message2.in_reply_to", "target_field": "in_reply_to", "ignore_missing": true } }, + { "rename": { "field": "message2.subject", "target_field": "subject", "ignore_missing": true } }, + { "rename": { "field": "message2.x_originating_ip", "target_field": "x_originating_ip", "ignore_missing": true } }, + { "rename": { "field": "message2.first_received", "target_field": "first_received", "ignore_missing": true } }, + { "rename": { "field": "message2.second_received", "target_field": "second_received", "ignore_missing": true } }, + { "rename": { "field": "message2.last_reply", "target_field": "last_reply", "ignore_missing": true } }, + { "rename": { "field": "message2.path", "target_field": "path", "ignore_missing": true } }, + { "rename": { "field": "message2.user_agent", "target_field": "useragent", "ignore_missing": true } }, + { "rename": { "field": "message2.tls", "target_field": "tls", "ignore_missing": true } }, + { "rename": { "field": "message2.fuids", "target_field": "fuids", "ignore_missing": true } }, + { "rename": { "field": "message2.is_webmail", "target_field": "is_webmail", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_snmp b/salt/elasticsearch/files/ingest/bro_snmp new file mode 100644 index 000000000..bec88c1af --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_snmp @@ -0,0 +1,25 @@ +{ + "description" : "bro_snmp", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.duration", "target_field": "duration", "ignore_missing": true } }, + { "rename": { "field": "message2.version", "target_field": "version", "ignore_missing": true } }, + { "rename": { "field": "message2.community", "target_field": "community", "ignore_missing": true } }, + { "rename": { "field": "message2.get_requests", "target_field": "get_requests", "ignore_missing": true } }, + { "rename": { "field": "message2.get_bulk_requests","target_field": "get_bulk_requests", "ignore_missing": true } }, + { "rename": { "field": "message2.get_responses", "target_field": "get_responses", "ignore_missing": true } }, + { "rename": { "field": "message2.set_requests", "target_field": "set_requests", "ignore_missing": true } }, + { "rename": { "field": "message2.display_string", "target_field": "display_string", "ignore_missing": true } }, + { "rename": { "field": "message2.up_since", "target_field": "up_since", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_socks b/salt/elasticsearch/files/ingest/bro_socks new file mode 100644 index 000000000..38c5dd528 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_socks @@ -0,0 +1,28 @@ +{ + "description" : "bro_socks", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.version", "target_field": "version", "ignore_missing": true } }, + { "rename": { "field": "message2.user", "target_field": "user", "ignore_missing": true } }, + { "rename": { "field": "message2.password", "target_field": "password", "ignore_missing": true } }, + { "rename": { "field": "message2.status", "target_field": "status", "ignore_missing": true } }, + { "rename": { "field": "message2.request_host", "target_field": "request_host", "ignore_missing": true } }, + { "dot_expander": { "field": "request.name", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.request.name", "target_field": "request_name", "ignore_missing": true } }, + { "rename": { "field": "message2.request_p", "target_field": "request_port", "ignore_missing": true } }, + { "dot_expander": { "field": "bound.host", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.bound.host", "target_field": "bound_host", "ignore_missing": true } }, + { "rename": { "field": "message2.bound_name", "target_field": "bound_name", "ignore_missing": true } }, + { "rename": { "field": "message2.bound_p", "target_field": "bound_port", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_software b/salt/elasticsearch/files/ingest/bro_software new file mode 100644 index 000000000..e742fda9e --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_software @@ -0,0 +1,23 @@ +{ + "description" : "bro_software", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "version.major", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.version.major", "target_field": "version_major", "ignore_missing": true } }, + { "dot_expander": { "field": "version.minor", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.version.minor", "target_field": "version_minor", "ignore_missing": true } }, + { "dot_expander": { "field": "version.minor2", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.version.minor2", "target_field": "version_minor2", "ignore_missing": true } }, + { "dot_expander": { "field": "version.minor3", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.version.minor3", "target_field": "version_minor3", "ignore_missing": true } }, + { "dot_expander": { "field": "version.addl", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.version.addl", "target_field": "version_additional_info", "ignore_missing": true } }, + { "rename": { "field": "message2.host", "target_field": "source_ip", "ignore_missing": true } }, + { "rename": { "field": "message2.host_p", "target_field": "source_port", "ignore_missing": true } }, + { "rename": { "field": "message2.software_type", "target_field": "software_type", "ignore_missing": true } }, + { "rename": { "field": "message2.name", "target_field": "name", "ignore_missing": true } }, + { "rename": { "field": "message2.unparsed_version", "target_field": "unparsed_version", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_ssh b/salt/elasticsearch/files/ingest/bro_ssh new file mode 100644 index 000000000..7df949503 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_ssh @@ -0,0 +1,40 @@ +{ + "description" : "bro_conn", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.version", "target_field": "version", "ignore_missing": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "rename": { "field": "message2.hassh", "target_field": "hassh", "ignore_missing": true } }, + { "rename": { "field": "message2.auth_success", "target_field": "authentication_success", "ignore_missing": true } }, + { "rename": { "field": "message2.auth_attempts", "target_field": "authentication_attempts", "ignore_missing": true } }, + { "rename": { "field": "message2.direction", "target_field": "direction", "ignore_missing": true } }, + { "rename": { "field": "message2.client", "target_field": "client", "ignore_missing": true } }, + { "rename": { "field": "message2.server", "target_field": "server", "ignore_missing": true } }, + { "rename": { "field": "message2.cipher_alg", "target_field": "cipher_algorithm", "ignore_missing": true } }, + { "rename": { "field": "message2.compression_alg", "target_field": "compression_algorithm", "ignore_missing": true } }, + { "rename": { "field": "message2.cshka", "target_field": "client_host_key_algorithms", "ignore_missing": true } }, + { "rename": { "field": "message2.host_key_alg", "target_field": "host_key_algorithm", "ignore_missing": true } }, + { "rename": { "field": "message2.hasshAlgorithms", "target_field": "hassh_algorithms", "ignore_missing": true } }, + { "rename": { "field": "message2.hasshServer", "target_field": "hassh_server", "ignore_missing": true } }, + { "rename": { "field": "message2.hasshVersion", "target_field": "hassh_version", "ignore_missing": true } }, + { "rename": { "field": "message2.kex_alg", "target_field": "kex_algorithm", "ignore_missing": true } }, + { "rename": { "field": "message2.mac_alg", "target_field": "mac_algorithm", "ignore_missing": true } }, + { "rename": { "field": "message2.sshka", "target_field": "server_host_key_algorithms", "ignore_missing": true } }, + { "rename": { "field": "message2.host_key", "target_field": "host_key", "ignore_missing": true } }, + { "rename": { "field": "message2.destination_region", "target_field": "destination_region", "ignore_missing": true } }, + { "rename": { "field": "message2.destination_city", "target_field": "destination_city", "ignore_missing": true } }, + { "rename": { "field": "message2.destination_latitude", "target_field": "destination_latitude", "ignore_missing": true } }, + { "rename": { "field": "message2.destination_longitude", "target_field": "destination_longitude", "ignore_missing": true } }, + { "rename": { "field": "message2.destination_country_code", "target_field": "destination_country_code", "ignore_missing": true } }, + { "rename": { "field": "message2.hasshServerAlgorithms", "target_field": "hassh_server_algorithms", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_ssl b/salt/elasticsearch/files/ingest/bro_ssl new file mode 100644 index 000000000..04d0fc8ec --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_ssl @@ -0,0 +1,33 @@ +{ + "description" : "bro_ssl", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.version", "target_field": "version", "ignore_missing": true } }, + { "rename": { "field": "message2.cipher", "target_field": "cipher", "ignore_missing": true } }, + { "rename": { "field": "message2.curve", "target_field": "curve", "ignore_missing": true } }, + { "rename": { "field": "message2.server_name", "target_field": "server_name", "ignore_missing": true } }, + { "rename": { "field": "message2.resumed", "target_field": "resumed", "ignore_missing": true } }, + { "rename": { "field": "message2.last_alert", "target_field": "last_alert", "ignore_missing": true } }, + { "rename": { "field": "message2.next_protocol", "target_field": "next_protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.established", "target_field": "established", "ignore_missing": true } }, + { "rename": { "field": "message2.cert_chain_fuids", "target_field": "certificate_chain_fuids", "ignore_missing": true } }, + { "rename": { "field": "message2.client_cert_chain_fuids", "target_field": "client_certificate_chain_fuids", "ignore_missing": true } }, + { "rename": { "field": "message2.subject", "target_field": "certificate_subject", "ignore_missing": true } }, + { "rename": { "field": "message2.issuer", "target_field": "certificate_issuer", "ignore_missing": true } }, + { "rename": { "field": "message2.client_subject", "target_field": "client_subject", "ignore_missing": true } }, + { "rename": { "field": "message2.client_issuer", "target_field": "client_issuer", "ignore_missing": true } }, + { "rename": { "field": "message2.validation_status","target_field": "validation_status", "ignore_missing": true } }, + { "rename": { "field": "message2.ja3", "target_field": "ja3", "ignore_missing": true } }, + { "rename": { "field": "message2.ja3s", "target_field": "ja3s", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common_ssl" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_syslog b/salt/elasticsearch/files/ingest/bro_syslog new file mode 100644 index 000000000..9599b435c --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_syslog @@ -0,0 +1,21 @@ +{ + "description" : "bro_syslog", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.proto", "target_field": "protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.facility", "target_field": "facility", "ignore_missing": true } }, + { "rename": { "field": "message2.severity", "target_field": "severity", "ignore_missing": true } }, + { "remove": { "field": "message", "ignore_failure": true } }, + { "rename": { "field": "message2.message", "target_field": "message", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_tunnels b/salt/elasticsearch/files/ingest/bro_tunnels new file mode 100644 index 000000000..50c12518f --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_tunnels @@ -0,0 +1,18 @@ +{ + "description" : "bro_tunnels", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.tunnel_type", "target_field": "tunnel_type", "ignore_missing": true } }, + { "rename": { "field": "message2.action", "target_field": "action", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_weird b/salt/elasticsearch/files/ingest/bro_weird new file mode 100644 index 000000000..b471f5e75 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_weird @@ -0,0 +1,20 @@ +{ + "description" : "bro_weird", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.uid", "target_field": "uid", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_h", "target_field": "source_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.orig_p", "target_field": "source_port", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_h", "target_field": "destination_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id.resp_p", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "message2.name", "target_field": "name", "ignore_missing": true } }, + { "rename": { "field": "message2.addl", "target_field": "additional_info", "ignore_missing": true } }, + { "rename": { "field": "message2.notice", "target_field": "notice", "ignore_missing": true } }, + { "rename": { "field": "message2.peer", "target_field": "peer", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/bro_x509 b/salt/elasticsearch/files/ingest/bro_x509 new file mode 100644 index 000000000..56e905347 --- /dev/null +++ b/salt/elasticsearch/files/ingest/bro_x509 @@ -0,0 +1,44 @@ +{ + "description" : "bro_x509", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.id", "target_field": "id", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.version", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.version", "target_field": "certificate_version", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.serial", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.serial", "target_field": "certificate_serial", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.subject", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.subject", "target_field": "certificate_subject", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.issuer", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.issuer", "target_field": "certificate_issuer", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.not_valid_before", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.not_valid_before", "target_field": "certificate_not_valid_before", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.not_valid_after", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.not_valid_after", "target_field": "certificate_not_valid_after", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.key_alg", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.key_alg", "target_field": "certificate_key_algorithm", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.sig_alg", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.sig_alg", "target_field": "certificate_signing_algorithm", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.key_type", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.key_type", "target_field": "certificate_key_type", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.key_length", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.key_length", "target_field": "certificate_key_length", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.exponent", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.exponent", "target_field": "certificate_exponent", "ignore_missing": true } }, + { "dot_expander": { "field": "certificate.curve", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.certificate.curve", "target_field": "certificate_curve", "ignore_missing": true } }, + { "dot_expander": { "field": "san.dns", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.san.dns", "target_field": "san_dns", "ignore_missing": true } }, + { "dot_expander": { "field": "san.uri", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.san.uri", "target_field": "san_uri", "ignore_missing": true } }, + { "dot_expander": { "field": "san.email", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.san.email", "target_field": "san_email", "ignore_missing": true } }, + { "dot_expander": { "field": "san.ip", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.san.ip", "target_field": "san_ip", "ignore_missing": true } }, + { "dot_expander": { "field": "basic_constraints.ca", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.basic_constraints.ca", "target_field": "basic_constraints_ca", "ignore_missing": true } }, + { "dot_expander": { "field": "basic_constraints.path_length", "path": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.basic_constraints.path_length", "target_field": "basic_constraints_path_length", "ignore_missing": true } }, + { "pipeline": { "name": "bro_common_ssl" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/common b/salt/elasticsearch/files/ingest/common new file mode 100644 index 000000000..ed227258e --- /dev/null +++ b/salt/elasticsearch/files/ingest/common @@ -0,0 +1,52 @@ +{ + "description" : "common", + "processors" : [ + { + "rename": { + "field": "type", + "target_field": "event_type", + "ignore_missing": true + } + }, + { + "geoip": { + "field": "destination_ip", + "target_field": "destination_geo", + "database_file": "GeoLite2-City.mmdb", + "ignore_missing": true, + "properties": ["ip", "country_iso_code", "country_name", "continent_name", "region_iso_code", "region_name", "city_name", "timezone", "location"] + } + }, + { + "geoip": { + "field": "source_ip", + "target_field": "source_geo", + "database_file": "GeoLite2-City.mmdb", + "ignore_missing": true, + "properties": ["ip", "country_iso_code", "country_name", "continent_name", "region_iso_code", "region_name", "city_name", "timezone", "location"] + } + }, + { + "split": { + "field": "_index", + "target_field": "index_name_prefix", + "separator": "-" + } + }, + { + "date_index_name": { + "field": "@timestamp", + "index_name_prefix": "{{index_name_prefix.0}}-{{index_name_prefix.1}}-", + "date_rounding": "d", + "ignore_failure": true, + "index_name_format": "yyyy.MM.dd" + } + }, + { + "remove": { + "field": "index_name_prefix", + "ignore_failure": true + } + } + ] +} diff --git a/salt/elasticsearch/files/ingest/common_nids b/salt/elasticsearch/files/ingest/common_nids new file mode 100644 index 000000000..4fffab7c1 --- /dev/null +++ b/salt/elasticsearch/files/ingest/common_nids @@ -0,0 +1,17 @@ +{ + "description" : "common_nids", + "processors" : [ + { "convert": { "field": "sid", "type": "integer" } }, + { "set": { "if": "ctx.sid < 1000000", "field": "signature_info", "value": "https://www.snort.org/search?query={{gid}}-{{sid}}" } }, + { "set": { "if": "ctx.sid > 1999999", "field": "signature_info", "value": "https://doc.emergingthreats.net/{{sid}}" } }, + { "remove": { "if": "ctx.sid > 2999999", "field": "signature_info" } }, + { "set": { "if": "ctx.priority == '1'", "field": "severity", "value": "High" } }, + { "set": { "if": "ctx.priority == '2'", "field": "severity", "value": "Medium" } }, + { "set": { "if": "ctx.priority == '3'", "field": "severity", "value": "Low" } }, + { "dissect": { "field": "alert", "pattern" : "%{rule_type} %{category} ", "ignore_failure": true } }, + { "set": { "if": "ctx.rule_type == 'GPL'", "field": "rule_type", "value": "Snort GPL" } }, + { "set": { "if": "ctx.rule_type == 'ET'", "field": "rule_type", "value": "Emerging Threats" } }, + { "lowercase": { "field": "category", "ignore_failure": true } }, + { "pipeline": { "name": "common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/ossec b/salt/elasticsearch/files/ingest/ossec new file mode 100644 index 000000000..2b6d19370 --- /dev/null +++ b/salt/elasticsearch/files/ingest/ossec @@ -0,0 +1,53 @@ +{ + "description" : "ossec", + "processors" : [ + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.agent", "target_field": "agent", "ignore_missing": true } }, + { "rename": { "field": "message2.data", "target_field": "data", "ignore_missing": true } }, + { "rename": { "field": "message2.decoder", "target_field": "decoder", "ignore_missing": true } }, + { "rename": { "field": "message2.full_log", "target_field": "full_log", "ignore_missing": true } }, + { "rename": { "field": "message2.id", "target_field": "id", "ignore_missing": true } }, + { "rename": { "field": "message2.location", "target_field": "location", "ignore_missing": true } }, + { "rename": { "field": "message2.manager", "target_field": "manager", "ignore_missing": true } }, + { "rename": { "field": "message2.predecoder", "target_field": "predecoder", "ignore_missing": true } }, + { "rename": { "field": "message2.timestamp", "target_field": "timestamp", "ignore_missing": true } }, + { "rename": { "field": "message2.rule", "target_field": "wazuh-rule", "ignore_missing": true } }, + { "rename": { "field": "data.command", "target_field": "command", "ignore_missing": true } }, + { "rename": { "field": "data.dstip", "target_field": "destination_ip", "ignore_missing": true } }, + { "rename": { "field": "data.dstport", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "data.dstuser", "target_field": "escalated_user", "ignore_missing": true } }, + { "rename": { "field": "data.srcip", "target_field": "source_ip", "ignore_missing": true } }, + { "rename": { "field": "data.srcuser", "target_field": "username", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.destinationHostname", "target_field": "destination_hostname", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.destinationIp", "target_field": "destination_ip", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.destinationPort", "target_field": "destination_port", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.image", "target_field": "image_path", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.parentImage", "target_field": "parent_image_path", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.sourceHostname", "target_field": "source_hostname", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.sourceIp", "target_field": "source_ip", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.sourcePort", "target_field": "source_port", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.targetFilename", "target_field": "target_filename", "ignore_missing": true } }, + { "rename": { "field": "data.win.eventdata.user", "target_field": "username", "ignore_missing": true } }, + { "rename": { "field": "data.win.system.eventID", "target_field": "event_id", "ignore_missing": true } }, + { "rename": { "field": "predecoder.program_name", "target_field": "process", "ignore_missing": true } }, + { "rename": { "field": "wazuh-rule.level", "target_field": "alert_level", "ignore_missing": true } }, + { "rename": { "field": "wazuh-rule.description", "target_field": "description", "ignore_missing": true } }, + { "set": { "if": "ctx.alert_level == 1", "field": "classification", "value": "None" } }, + { "set": { "if": "ctx.alert_level == 2", "field": "classification", "value": "System low priority notification" } }, + { "set": { "if": "ctx.alert_level == 3", "field": "classification", "value": "Successful/authorized event" } }, + { "set": { "if": "ctx.alert_level == 4", "field": "classification", "value": "System low priority error" } }, + { "set": { "if": "ctx.alert_level == 5", "field": "classification", "value": "User generated error" } }, + { "set": { "if": "ctx.alert_level == 6", "field": "classification", "value": "Low relevance attack" } }, + { "set": { "if": "ctx.alert_level == 7", "field": "classification", "value": "\"Bad word\" matching" } }, + { "set": { "if": "ctx.alert_level == 8", "field": "classification", "value": "First time seen" } }, + { "set": { "if": "ctx.alert_level == 9", "field": "classification", "value": "Error from invalid source" } }, + { "set": { "if": "ctx.alert_level == 10", "field": "classification", "value": "Multiple user generated errors" } }, + { "set": { "if": "ctx.alert_level == 11", "field": "classification", "value": "Integrity checking warning" } }, + { "set": { "if": "ctx.alert_level == 12", "field": "classification", "value": "High importance event" } }, + { "set": { "if": "ctx.alert_level == 13", "field": "classification", "value": "Unusal error (high importance)" } }, + { "set": { "if": "ctx.alert_level == 14", "field": "classification", "value": "High importance security event" } }, + { "set": { "if": "ctx.alert_level == 15", "field": "classification", "value": "Severe attack" } }, + { "append": { "if": "ctx.alert_level != null", "field": "tags", "value": ["alert"] } }, + { "pipeline": { "name": "common" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/sguild_nids b/salt/elasticsearch/files/ingest/sguild_nids new file mode 100644 index 000000000..c7bcdc418 --- /dev/null +++ b/salt/elasticsearch/files/ingest/sguild_nids @@ -0,0 +1,25 @@ +{ + "description" : "sguild_nids", + "processors" : [ + { + "dissect": { + "field": "message", + "pattern" : "%{} %{} %{} Alert Received: %{} %{priority} %{classification} %{interface} {%{alerttime}} %{} %{} {%{alert}} %{source_ip} %{destination_ip} %{protocol} %{source_port} %{destination_port} %{gid} %{sid} %{rev} ", + "on_failure": [ { "drop" : { } } ] + } + }, + { "set": { "if": "ctx.protocol == '1'", "field": "protocol", "value": "ICMP" } }, + { "set": { "if": "ctx.protocol == '6'", "field": "protocol", "value": "TCP" } }, + { "set": { "if": "ctx.protocol == '17'", "field": "protocol", "value": "UDP" } }, + { "remove": { "if": "ctx.source_ip == '{}'", "field": "source_ip" } }, + { "remove": { "if": "ctx.destination_ip == '{}'", "field": "destination_ip" } }, + { "remove": { "if": "ctx.protocol == '{}'", "field": "protocol" } }, + { "remove": { "if": "ctx.source_port == '{}'", "field": "source_port" } }, + { "remove": { "if": "ctx.destination_port == '{}'", "field": "destination_port" } }, + { "set": { "field": "type", "value": "snort" } }, + { "rename": { "field": "@timestamp", "target_field": "timestamp", "ignore_missing": true } }, + { "date": { "field": "alerttime", "target_field": "@timestamp", "formats": ["yyyy-MM-dd HH:mm:ss"], "ignore_failure": true } }, + { "remove": { "field": "alerttime", "ignore_missing": true } }, + { "pipeline": { "name": "common_nids" } } + ] +} diff --git a/salt/elasticsearch/files/ingest/snort b/salt/elasticsearch/files/ingest/snort new file mode 100644 index 000000000..b841ca917 --- /dev/null +++ b/salt/elasticsearch/files/ingest/snort @@ -0,0 +1,21 @@ +{ + "description" : "snort", + "processors" : [ + { + "dissect": { + "field": "message", + "pattern" : "[%{gid}:%{sid}:%{rev}] %{alert} [Classification: %{classification}] [Priority: %{priority}]: <%{interface}> {%{protocol}} %{source_ip_port} -> %{destination_ip_port}", + "on_failure": [ { "drop" : { } } ] + } + }, + { "split": { "field": "source_ip_port", "separator": ":", "ignore_failure": true } }, + { "split": { "field": "destination_ip_port", "separator": ":", "ignore_failure": true } }, + { "rename":{ "field": "source_ip_port.1", "target_field": "source_port", "ignore_failure": true } }, + { "rename":{ "field": "destination_ip_port.1", "target_field": "destination_port", "ignore_failure": true } }, + { "rename":{ "field": "source_ip_port.0", "target_field": "source_ip", "ignore_failure": true } }, + { "rename":{ "field": "destination_ip_port.0", "target_field": "destination_ip", "ignore_failure": true } }, + { "remove":{ "field": "source_ip_port", "ignore_failure": true } }, + { "remove":{ "field": "destination_ip_port", "ignore_failure": true } }, + { "pipeline": { "name": "common_nids" } } + ] +} diff --git a/salt/elasticsearch/files/so-elasticsearch-pipelines b/salt/elasticsearch/files/so-elasticsearch-pipelines new file mode 100755 index 000000000..c0dd44aa9 --- /dev/null +++ b/salt/elasticsearch/files/so-elasticsearch-pipelines @@ -0,0 +1,52 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +ELASTICSEARCH_HOST=$1 +ELASTICSEARCH_PORT=9200 + +# Define a default directory to load pipelines from +ELASTICSEARCH_INGEST_PIPELINES="/opt/so/saltstack/salt/elasticsearch/files/ingest/" + +# Wait for ElasticSearch to initialize +echo -n "Waiting for ElasticSearch..." +COUNT=0 +ELASTICSEARCH_CONNECTED="no" +while [[ "$COUNT" -le 240 ]]; do + curl ${ELASTICSEARCH_AUTH} --output /dev/null --silent --head --fail http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" + if [ $? -eq 0 ]; then + ELASTICSEARCH_CONNECTED="yes" + echo "connected!" + break + else + ((COUNT+=1)) + sleep 1 + echo -n "." + fi +done +if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then + echo + echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" + echo +fi + +cd ${ELASTICSEARCH_INGEST_PIPELINES} + +echo "Loading pipelines..." +for i in *; do echo $i; curl ${ELASTICSEARCH_AUTH} -XPUT http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done +echo + +cd - >/dev/null diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index c05cb83e4..1eb3a19e4 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -121,6 +121,10 @@ so-elasticsearch: - /nsm/elasticsearch:/usr/share/elasticsearch/data:rw - /opt/so/log/elasticsearch:/var/log/elasticsearch:rw +so-elasticsearch-pipelines: + cmd.run: + - name: /opt/so/saltstack/salt/elasticsearch/files/so-elasticsearch-pipelines {{ esclustername }} + # Tell the main cluster I am here #curl -XPUT http://\$ELASTICSEARCH_HOST:\$ELASTICSEARCH_PORT/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"$HOSTNAME": {"skip_unavailable": "true", "seeds": ["$DOCKER_INTERFACE:$REVERSE_PORT"]}}}}}' diff --git a/salt/logstash/conf/conf.enabled.txt.parser b/salt/logstash/conf/conf.enabled.txt.parser index a34b39c5f..6fbf3ba45 100644 --- a/salt/logstash/conf/conf.enabled.txt.parser +++ b/salt/logstash/conf/conf.enabled.txt.parser @@ -7,11 +7,12 @@ # /usr/share/logstash/pipeline.custom/1234_input_custom.conf ## # All of the defaults are loaded. +# Please note that Bro config is commented out because we're moving that parsing to Elasticsearch ingest. /usr/share/logstash/pipeline.dynamic/0900_input_redis.conf /usr/share/logstash/pipeline.so/1000_preprocess_log_elapsed.conf /usr/share/logstash/pipeline.so/1001_preprocess_syslogng.conf /usr/share/logstash/pipeline.so/1002_preprocess_json.conf -/usr/share/logstash/pipeline.so/1003_preprocess_bro.conf +#/usr/share/logstash/pipeline.so/1003_preprocess_bro.conf /usr/share/logstash/pipeline.so/1004_preprocess_syslog_types.conf /usr/share/logstash/pipeline.so/1026_preprocess_dhcp.conf /usr/share/logstash/pipeline.so/1029_preprocess_esxi.conf @@ -20,44 +21,44 @@ /usr/share/logstash/pipeline.so/1032_preprocess_mcafee.conf /usr/share/logstash/pipeline.so/1033_preprocess_snort.conf /usr/share/logstash/pipeline.so/1034_preprocess_syslog.conf -/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf -/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf -/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf -/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf -/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf -/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf -/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf -/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf -/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf -/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf -/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf -/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf -/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf -/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf -/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf -/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf -/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf -/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf -/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf -/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf -/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf -/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf -/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf -/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf -/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf -/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf -/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf -/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf -/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf -/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf -/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf -/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf -/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf -/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf +#/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf +#/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf +#/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf +#/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf +#/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf +#/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf +#/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf +#/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf +#/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf +#/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf +#/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf +#/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf +#/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf +#/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf +#/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf +#/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf +#/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf +#/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf +#/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf +#/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf +#/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf +#/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf +#/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf +#/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf +#/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf +#/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf +#/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf +#/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf +#/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf +#/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf +#/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf +#/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf +#/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf +#/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf /usr/share/logstash/pipeline.so/1998_test_data.conf /usr/share/logstash/pipeline.so/2000_network_flow.conf -/usr/share/logstash/pipeline.so/6000_bro.conf -/usr/share/logstash/pipeline.so/6001_bro_import.conf +#/usr/share/logstash/pipeline.so/6000_bro.conf +#/usr/share/logstash/pipeline.so/6001_bro_import.conf /usr/share/logstash/pipeline.so/6002_syslog.conf /usr/share/logstash/pipeline.so/6101_switch_brocade.conf /usr/share/logstash/pipeline.so/6200_firewall_fortinet.conf @@ -68,17 +69,17 @@ /usr/share/logstash/pipeline.so/6500_ossec.conf /usr/share/logstash/pipeline.so/6501_ossec_sysmon.conf /usr/share/logstash/pipeline.so/6502_ossec_autoruns.conf -/usr/share/logstash/pipeline.so/8000_postprocess_bro_cleanup.conf +#/usr/share/logstash/pipeline.so/8000_postprocess_bro_cleanup.conf /usr/share/logstash/pipeline.so/8001_postprocess_common_ip_augmentation.conf /usr/share/logstash/pipeline.so/8006_postprocess_dns.conf /usr/share/logstash/pipeline.so/8007_postprocess_dns_top1m_tagging.conf /usr/share/logstash/pipeline.so/8007_postprocess_http.conf /usr/share/logstash/pipeline.so/8008_postprocess_dns_whois_age.conf /usr/share/logstash/pipeline.so/8200_postprocess_tagging.conf -/usr/share/logstash/pipeline.so/8502_postprocess_freq_analysis_bro_dns.conf -/usr/share/logstash/pipeline.so/8503_postprocess_freq_analysis_bro_http.conf -/usr/share/logstash/pipeline.so/8504_postprocess_freq_analysis_bro_ssl.conf -/usr/share/logstash/pipeline.so/8505_postprocess_freq_analysis_bro_x509.conf +#/usr/share/logstash/pipeline.so/8502_postprocess_freq_analysis_bro_dns.conf +#/usr/share/logstash/pipeline.so/8503_postprocess_freq_analysis_bro_http.conf +#/usr/share/logstash/pipeline.so/8504_postprocess_freq_analysis_bro_ssl.conf +#/usr/share/logstash/pipeline.so/8505_postprocess_freq_analysis_bro_x509.conf /usr/share/logstash/pipeline.so/8998_postprocess_log_elapsed.conf /usr/share/logstash/pipeline.so/8999_postprocess_rename_type.conf /usr/share/logstash/pipeline.dynamic/9999_output_redis.conf diff --git a/salt/logstash/conf/conf.enabled.txt.so-eval b/salt/logstash/conf/conf.enabled.txt.so-eval index e7680b3be..d125fc829 100644 --- a/salt/logstash/conf/conf.enabled.txt.so-eval +++ b/salt/logstash/conf/conf.enabled.txt.so-eval @@ -7,6 +7,7 @@ # /usr/share/logstash/pipeline.custom/1234_input_custom.conf ## # All of the defaults are loaded. +# Please note that Bro config is commented out because we're moving that parsing to Elasticsearch ingest. #/usr/share/logstash/pipeline.so/0000_input_syslogng.conf #/usr/share/logstash/pipeline.so/0001_input_json.conf #/usr/share/logstash/pipeline.so/0002_input_windows_json.conf @@ -18,7 +19,7 @@ #/usr/share/logstash/pipeline.so/1000_preprocess_log_elapsed.conf #/usr/share/logstash/pipeline.so/1001_preprocess_syslogng.conf #/usr/share/logstash/pipeline.so/1002_preprocess_json.conf -/usr/share/logstash/pipeline.so/1003_preprocess_bro.conf +#/usr/share/logstash/pipeline.so/1003_preprocess_bro.conf #/usr/share/logstash/pipeline.so/1004_preprocess_syslog_types.conf /usr/share/logstash/pipeline.so/1026_preprocess_dhcp.conf #/usr/share/logstash/pipeline.so/1029_preprocess_esxi.conf @@ -27,44 +28,44 @@ #/usr/share/logstash/pipeline.so/1032_preprocess_mcafee.conf /usr/share/logstash/pipeline.so/1033_preprocess_snort.conf #/usr/share/logstash/pipeline.so/1034_preprocess_syslog.conf -/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf -/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf -/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf -/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf -/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf -/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf -/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf -/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf -/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf -/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf -/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf -/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf -/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf -/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf -/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf -/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf -/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf -/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf -/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf -/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf -/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf -/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf -/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf -/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf -/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf -/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf -/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf -/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf -/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf -/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf -/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf -/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf -/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf -/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf +#/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf +#/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf +#/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf +#/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf +#/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf +#/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf +#/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf +#/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf +#/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf +#/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf +#/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf +#/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf +#/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf +#/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf +#/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf +#/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf +#/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf +#/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf +#/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf +#/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf +#/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf +#/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf +#/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf +#/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf +#/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf +#/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf +#/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf +#/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf +#/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf +#/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf +#/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf +#/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf +#/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf +#/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf #/usr/share/logstash/pipeline.so/1998_test_data.conf #/usr/share/logstash/pipeline.so/2000_network_flow.conf -/usr/share/logstash/pipeline.so/6000_bro.conf -/usr/share/logstash/pipeline.so/6001_bro_import.conf +#/usr/share/logstash/pipeline.so/6000_bro.conf +#/usr/share/logstash/pipeline.so/6001_bro_import.conf #/usr/share/logstash/pipeline.so/6002_syslog.conf #/usr/share/logstash/pipeline.so/6101_switch_brocade.conf #/usr/share/logstash/pipeline.so/6200_firewall_fortinet.conf @@ -77,7 +78,7 @@ /usr/share/logstash/pipeline.so/6502_ossec_autoruns.conf /usr/share/logstash/pipeline.so/6600_winlogbeat_sysmon.conf /usr/share/logstash/pipeline.so/6700_winlogbeat.conf -/usr/share/logstash/pipeline.so/8000_postprocess_bro_cleanup.conf +#/usr/share/logstash/pipeline.so/8000_postprocess_bro_cleanup.conf /usr/share/logstash/pipeline.so/8001_postprocess_common_ip_augmentation.conf #/usr/share/logstash/pipeline.so/8006_postprocess_dns.conf #/usr/share/logstash/pipeline.so/8007_postprocess_dns_top1m_tagging.conf diff --git a/salt/logstash/conf/conf.enabled.txt.storage b/salt/logstash/conf/conf.enabled.txt.storage index 6d2d581db..470f19c55 100644 --- a/salt/logstash/conf/conf.enabled.txt.storage +++ b/salt/logstash/conf/conf.enabled.txt.storage @@ -7,11 +7,12 @@ # /usr/share/logstash/pipeline.custom/1234_input_custom.conf ## # All of the defaults are loaded. +# Please note that Bro config is commented out because we're moving that parsing to Elasticsearch ingest. /usr/share/logstash/pipeline.dynamic/0900_input_redis.conf /usr/share/logstash/pipeline.so/1000_preprocess_log_elapsed.conf /usr/share/logstash/pipeline.so/1001_preprocess_syslogng.conf /usr/share/logstash/pipeline.so/1002_preprocess_json.conf -/usr/share/logstash/pipeline.so/1003_preprocess_bro.conf +#/usr/share/logstash/pipeline.so/1003_preprocess_bro.conf /usr/share/logstash/pipeline.so/1004_preprocess_syslog_types.conf /usr/share/logstash/pipeline.so/1026_preprocess_dhcp.conf /usr/share/logstash/pipeline.so/1029_preprocess_esxi.conf @@ -20,44 +21,44 @@ /usr/share/logstash/pipeline.so/1032_preprocess_mcafee.conf /usr/share/logstash/pipeline.so/1033_preprocess_snort.conf /usr/share/logstash/pipeline.so/1034_preprocess_syslog.conf -/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf -/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf -/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf -/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf -/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf -/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf -/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf -/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf -/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf -/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf -/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf -/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf -/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf -/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf -/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf -/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf -/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf -/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf -/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf -/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf -/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf -/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf -/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf -/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf -/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf -/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf -/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf -/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf -/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf -/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf -/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf -/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf -/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf -/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf +#/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf +#/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf +#/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf +#/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf +#/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf +#/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf +#/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf +#/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf +#/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf +#/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf +#/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf +#/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf +#/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf +#/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf +#/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf +#/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf +#/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf +#/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf +#/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf +#/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf +#/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf +#/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf +#/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf +#/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf +#/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf +#/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf +#/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf +#/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf +#/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf +#/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf +#/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf +#/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf +#/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf +#/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf /usr/share/logstash/pipeline.so/1998_test_data.conf /usr/share/logstash/pipeline.so/2000_network_flow.conf -/usr/share/logstash/pipeline.so/6000_bro.conf -/usr/share/logstash/pipeline.so/6001_bro_import.conf +#/usr/share/logstash/pipeline.so/6000_bro.conf +#/usr/share/logstash/pipeline.so/6001_bro_import.conf /usr/share/logstash/pipeline.so/6002_syslog.conf /usr/share/logstash/pipeline.so/6101_switch_brocade.conf /usr/share/logstash/pipeline.so/6200_firewall_fortinet.conf @@ -70,7 +71,7 @@ /usr/share/logstash/pipeline.so/6502_ossec_autoruns.conf /usr/share/logstash/pipeline.so/6600_winlogbeat_sysmon.conf /usr/share/logstash/pipeline.so/6700_winlogbeat.conf -/usr/share/logstash/pipeline.so/8000_postprocess_bro_cleanup.conf +#/usr/share/logstash/pipeline.so/8000_postprocess_bro_cleanup.conf /usr/share/logstash/pipeline.so/8001_postprocess_common_ip_augmentation.conf #/usr/share/logstash/pipeline.so/8006_postprocess_dns.conf #/usr/share/logstash/pipeline.so/8007_postprocess_dns_top1m_tagging.conf diff --git a/salt/logstash/files/dynamic/9000_output_bro.conf b/salt/logstash/files/dynamic/9000_output_bro.conf index 54d2d20ca..553500281 100644 --- a/salt/logstash/files/dynamic/9000_output_bro.conf +++ b/salt/logstash/files/dynamic/9000_output_bro.conf @@ -20,6 +20,7 @@ output { if "bro" in [tags] and "test_data" not in [tags] and "import" not in [tags] { # stdout { codec => rubydebug } elasticsearch { + pipeline => "%{event_type}" hosts => "{{ ES }}" index => "logstash-bro-%{+YYYY.MM.dd}" template_name => "logstash" diff --git a/so-setup-network.sh b/so-setup-network.sh index 13906311d..6f1055efc 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -466,11 +466,11 @@ install_master() { ls_heapsize() { # Determine LS Heap Size - if [ $TOTAL_MEM -ge 16000 ] ; then - LS_HEAP_SIZE="4192m" + if [ $TOTAL_MEM -ge 32000 ] ; then + LS_HEAP_SIZE="1000m" else - # Set a max of 1GB heap if you have less than 16GB RAM - LS_HEAP_SIZE="2g" + # If minimal RAM, then set minimal heap + LS_HEAP_SIZE="500m" fi } @@ -1752,7 +1752,7 @@ if (whiptail_you_sure); then es_heapsize ls_heapsize NODE_ES_HEAP_SIZE="600m" - NODE_LS_HEAP_SIZE="2000m" + NODE_LS_HEAP_SIZE="500m" LSPIPELINEWORKERS=1 LSPIPELINEBATCH=125 LSINPUTTHREADS=1