From d0ad7e7ef9fbf782f9b70168bdd61578cb469109 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 1 Apr 2020 14:57:32 -0400 Subject: [PATCH 1/7] Update config for search pipeline --- pillar/logstash/search.sls | 51 +++----------------------------------- 1 file changed, 4 insertions(+), 47 deletions(-) diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 4c295df14..aba6b8308 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -2,59 +2,16 @@ logstash: pipelines: search: config: - - so/1000_preprocess_log_elapsed.conf - - so/1001_preprocess_syslogng.conf - - so/1002_preprocess_json.conf - - so/1004_preprocess_syslog_types.conf - - so/1026_preprocess_dhcp.conf - - so/1029_preprocess_esxi.conf - - so/1030_preprocess_greensql.conf - - so/1031_preprocess_iis.conf - - so/1032_preprocess_mcafee.conf - - so/1033_preprocess_snort.conf - - so/1034_preprocess_syslog.conf - - so/2000_network_flow.conf - - so/6002_syslog.conf - - so/6101_switch_brocade.conf - - so/6200_firewall_fortinet.conf - - so/6201_firewall_pfsense.conf - - so/6300_windows.conf - - so/6301_dns_windows.conf - - so/6400_suricata.conf - - so/6500_ossec.conf - - so/6501_ossec_sysmon.conf - - so/6502_ossec_autoruns.conf - - so/6600_winlogbeat_sysmon.conf - - so/6700_winlogbeat.conf - - so/7100_osquery_wel.conf - - so/7200_strelka.conf - - so/8001_postprocess_common_ip_augmentation.conf - - so/8007_postprocess_http.conf - - so/8200_postprocess_tagging.conf - - so/8998_postprocess_log_elapsed.conf - - so/8999_postprocess_rename_type.conf - - so/0900_input_redis.conf.jinja + - so/0900_input_redis.conf.jinja - so/9000_output_bro.conf.jinja - - so/9001_output_switch.conf.jinja - so/9002_output_import.conf.jinja - - so/9004_output_flow.conf.jinja - - so/9026_output_dhcp.conf.jinja - - so/9029_output_esxi.conf.jinja - - so/9030_output_greensql.conf.jinja - - so/9031_output_iis.conf.jinja - - so/9032_output_mcafee.conf.jinja - so/9033_output_snort.conf.jinja - - so/9034_output_syslog.conf.jinja - so/9100_output_osquery.conf.jinja - - so/9200_output_firewall.conf.jinja - - so/9300_output_windows.conf.jinja - - so/9301_output_dns_windows.conf.jinja - so/9400_output_suricata.conf.jinja - so/9500_output_beats.conf.jinja - so/9600_output_ossec.conf.jinja - so/9700_output_strelka.conf.jinja templates: - - so/beats-template.json - - so/logstash-ossec-template.json - - so/logstash-strelka-template.json - - so/logstash-template.json + - so/so-beats-template.json + - so/so-common-template.json + - so/so-zeek-template.json From 9f58081ce86356dd00c923b326fa6b120d999005 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 1 Apr 2020 14:57:47 -0400 Subject: [PATCH 2/7] Typo --- pillar/logstash/search.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index aba6b8308..1d96737e5 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -2,7 +2,7 @@ logstash: pipelines: search: config: - - so/0900_input_redis.conf.jinja + - so/0900_input_redis.conf.jinja - so/9000_output_bro.conf.jinja - so/9002_output_import.conf.jinja - so/9033_output_snort.conf.jinja From e1008269ce6f98b77c8627b89335495a614f82b0 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 1 Apr 2020 15:00:52 -0400 Subject: [PATCH 3/7] Update OSSEC output --- .../pipelines/config/so/9600_output_ossec.conf.jinja | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja index 53dae8825..93bffca7d 100644 --- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja +++ b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja @@ -9,7 +9,7 @@ # Last Update: 9/19/2018 filter { - if [event_type] =~ "ossec" { + if [module] =~ "ossec" { mutate { ##add_tag => [ "conf_file_9600"] } @@ -17,9 +17,9 @@ filter { } output { - if [event_type] =~ "ossec" or "ossec" in [tags] { + if [module] =~ "ossec" { elasticsearch { - pipeline => "%{event_type}" + pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" index => "so-ossec-%{+YYYY.MM.dd}" template_name => "so-common" From 82c99edbfc42bb81db78d015aea1dbe6b98b85ae Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 1 Apr 2020 15:02:52 -0400 Subject: [PATCH 4/7] Update Suricata output --- .../pipelines/config/so/9400_output_suricata.conf.jinja | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index 7b587242a..640c50f39 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -9,16 +9,16 @@ # Last Update: 12/9/2016 filter { - if [event_type] == "suricata" and "test_data" not in [tags] { + if [module] == "suricata" { mutate { ##add_tag => [ "conf_file_9400"] } } } output { - if [event_type] == "suricata" and "test_data" not in [tags] { - #stdout { codec => rubydebug } + if [module] =~ "suricata" { elasticsearch { + pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" index => "so-ids-%{+YYYY.MM.dd}" template => "/so-common-template.json" From 1df2302287bfd41d21b387a5cd400a3660f30dc9 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 1 Apr 2020 19:09:38 +0000 Subject: [PATCH 5/7] Update Zeek and Strelka --- ...0_output_bro.conf.jinja => 9000_output_zeek.conf.jinja} | 7 +++---- .../pipelines/config/so/9700_output_strelka.conf.jinja | 3 ++- 2 files changed, 5 insertions(+), 5 deletions(-) rename salt/logstash/pipelines/config/so/{9000_output_bro.conf.jinja => 9000_output_zeek.conf.jinja} (73%) diff --git a/salt/logstash/pipelines/config/so/9000_output_bro.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja similarity index 73% rename from salt/logstash/pipelines/config/so/9000_output_bro.conf.jinja rename to salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index 9ce08edf8..6def12650 100644 --- a/salt/logstash/pipelines/config/so/9000_output_bro.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -10,17 +10,16 @@ filter { - if "zeek" in [tags] and "test_data" not in [tags] and "import" not in [tags] { + if [module] =~ "zeek" { mutate { ##add_tag => [ "conf_file_9000"] } } } output { - if "zeek" in [tags] and "test_data" not in [tags] and "import" not in [tags] { -# stdout { codec => rubydebug } + if [module] =~ "zeek" { elasticsearch { - pipeline => "%{event_type}" + pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" index => "so-zeek-%{+YYYY.MM.dd}" template_name => "so-zeek" diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja index 5da6d0b12..1e192a8ee 100644 --- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja +++ b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja @@ -10,7 +10,7 @@ filter { - if [event_type] =~ "strelka" { + if [module] =~ "strelka" { mutate { ##add_tag => [ "conf_file_9000"] } @@ -19,6 +19,7 @@ filter { output { if [event_type] =~ "strelka" { elasticsearch { + pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" index => "so-strelka-%{+YYYY.MM.dd}" template_name => "so-common" From 09022ad7523d88bdad548962a8b679134c27c1a6 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 1 Apr 2020 19:11:10 +0000 Subject: [PATCH 6/7] Update Zeek and Strelka --- .../pipelines/config/so/9100_output_osquery.conf.jinja | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja index ca9c90215..d09aae10b 100644 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja @@ -9,11 +9,12 @@ output { - if "osquery" in [tags] { + if [module] =~ "osquery" { elasticsearch { + pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" index => "so-osquery-%{+YYYY.MM.dd}" template => "/so-common-template.json" } } -} \ No newline at end of file +} From 22e7777d67c390eaf8ee8b226e5925155a9c3533 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 1 Apr 2020 19:12:07 +0000 Subject: [PATCH 7/7] update search sls --- pillar/logstash/search.sls | 1 - 1 file changed, 1 deletion(-) diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 1d96737e5..edfb4eba3 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -5,7 +5,6 @@ logstash: - so/0900_input_redis.conf.jinja - so/9000_output_bro.conf.jinja - so/9002_output_import.conf.jinja - - so/9033_output_snort.conf.jinja - so/9100_output_osquery.conf.jinja - so/9400_output_suricata.conf.jinja - so/9500_output_beats.conf.jinja