From 53284a71738221a8d0bc9606aceecdc9e69f76d5 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 30 Nov 2018 13:13:55 +0000 Subject: [PATCH 1/4] Bro - Switch logs to JSON format --- salt/bro/files/local.bro | 3 ++- salt/bro/files/local.bro.community | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/bro/files/local.bro b/salt/bro/files/local.bro index dbad5cf61..fc5e18517 100644 --- a/salt/bro/files/local.bro +++ b/salt/bro/files/local.bro @@ -141,4 +141,5 @@ #@load custom/somebropolicy.bro # Write logs in JSON -#redef LogAscii::use_json = T; +redef LogAscii::use_json = T; +redef LogAscii::json_timestamps = JSON::TS_ISO8601; diff --git a/salt/bro/files/local.bro.community b/salt/bro/files/local.bro.community index 58d079552..2ae12d7f2 100644 --- a/salt/bro/files/local.bro.community +++ b/salt/bro/files/local.bro.community @@ -129,4 +129,5 @@ #@load custom/somebropolicy.bro # Use JSON -#redef LogAscii::use_json = T; +redef LogAscii::use_json = T; +redef LogAscii::json_timestamps = JSON::TS_ISO8601; From 2695a8e1a791390860a9dafc66319402f4fea514 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 30 Nov 2018 13:15:35 +0000 Subject: [PATCH 2/4] Filebeat - Modify config for IDS type --- salt/filebeat/etc/filebeat.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 8b4520a3a..f8fb5acf5 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -30,9 +30,9 @@ filebeat.prospectors: paths: - /suricata/eve.json fields: - type: snort + type: ids + engine: suricata fields_under_root: true - tags: ["ids"] clean_removed: false close_removed: false From 05accf301748907e0deda580fba7d512dab392ed Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 30 Nov 2018 13:17:53 +0000 Subject: [PATCH 3/4] Logstash - Output config update for IDS type --- salt/logstash/files/dynamic/9033_output_snort.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/logstash/files/dynamic/9033_output_snort.conf b/salt/logstash/files/dynamic/9033_output_snort.conf index a953a2db2..6c310b91e 100644 --- a/salt/logstash/files/dynamic/9033_output_snort.conf +++ b/salt/logstash/files/dynamic/9033_output_snort.conf @@ -9,14 +9,14 @@ # Last Update: 12/9/2016 filter { - if [event_type] == "snort" and "test_data" not in [tags] { + if [event_type] == "ids" and "test_data" not in [tags] { mutate { ##add_tag => [ "conf_file_9033"] } } } output { - if [event_type] == "snort" and "test_data" not in [tags] { + if [event_type] == "ids" and "test_data" not in [tags] { #stdout { codec => rubydebug } elasticsearch { hosts => "{{ ES }}" From 056e22dd1118fd0487116fcf815948c38a88fbce Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 30 Nov 2018 13:22:31 +0000 Subject: [PATCH 4/4] Logstash - Update template for Suricata metadata fields --- salt/logstash/etc/logstash-template.json | 87 +++++++++++++++++++++++- 1 file changed, 86 insertions(+), 1 deletion(-) diff --git a/salt/logstash/etc/logstash-template.json b/salt/logstash/etc/logstash-template.json index 6822a82a3..70d7624a1 100644 --- a/salt/logstash/etc/logstash-template.json +++ b/salt/logstash/etc/logstash-template.json @@ -5,7 +5,8 @@ "settings":{ "number_of_replicas":0, "number_of_shards":1, - "index.refresh_interval":"30s" + "index.refresh_interval":"30s", + "index.mapping.total_fields.limit": 10000 }, "mappings":{ "doc":{ @@ -206,6 +207,10 @@ "basic_constraints_path_length":{ "type":"long" }, + "beat_host":{ + "type":"object", + "dynamic": true + }, "bound_port":{ "type":"long" }, @@ -860,6 +865,10 @@ } } }, + "dhcp":{ + "type":"object", + "dynamic": true + }, "dir":{ "type":"text", "fields":{ @@ -884,6 +893,10 @@ } } }, + "dns":{ + "type":"object", + "dynamic": true + }, "domain_age":{ "type":"text", "fields":{ @@ -919,6 +932,10 @@ } } }, + "email":{ + "type":"object", + "dynamic": true + }, "enabled":{ "type":"text", "fields":{ @@ -951,6 +968,14 @@ } } }, + "engine":{ + "type":"text", + "fields":{ + "keyword":{ + "type":"keyword" + } + } + }, "entry":{ "type":"text", "fields":{ @@ -1066,6 +1091,10 @@ } } }, + "fileinfo":{ + "type":"object", + "dynamic": true + }, "file_ip":{ "type":"ip", "fields":{ @@ -1101,6 +1130,10 @@ } } }, + "flow":{ + "type":"object", + "dynamic": true + }, "flow_id":{ "type":"text", "fields":{ @@ -1338,6 +1371,10 @@ } } }, + "http":{ + "type":"object", + "dynamic": true + }, "id":{ "type":"text", "fields":{ @@ -1346,6 +1383,14 @@ } } }, + "ids_event_type":{ + "type":"text", + "fields":{ + "keyword":{ + "type":"keyword" + } + } + }, "iin":{ "type":"text", "fields":{ @@ -1647,6 +1692,10 @@ } } }, + "krb5":{ + "type":"object", + "dynamic": true + }, "last_alert":{ "type":"text", "fields":{ @@ -1826,6 +1875,10 @@ } } }, + "metadata":{ + "type":"object", + "dynamic": true + }, "method":{ "type":"text", "fields":{ @@ -1907,6 +1960,10 @@ } } }, + "netflow":{ + "type":"object", + "dynamic": true + }, "next_protocol":{ "type":"text", "fields":{ @@ -2802,6 +2859,14 @@ } } }, + "smb":{ + "type":"object", + "dynamic": true + }, + "smtp":{ + "type":"object", + "dynamic": true + }, "software_type":{ "type":"text", "fields":{ @@ -3094,6 +3159,10 @@ } } }, + "tcp":{ + "type":"object", + "dynamic": true + }, "tcp_flags":{ "type":"text", "fields":{ @@ -3243,6 +3312,14 @@ } } }, + "tx_id":{ + "type":"text", + "fields":{ + "keyword":{ + "type":"keyword" + } + } + }, "type":{ "type":"text", "fields":{ @@ -3294,6 +3371,14 @@ "uri_length":{ "type":"long" }, + "url":{ + "type":"text", + "fields": { + "keyword":{ + "type":"keyword" + } + } + }, "username":{ "type":"text", "fields": {