From 5eb33d5ac7aeae18bf7b5fd95a7c3a3d18250bb5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 10 Jul 2020 13:53:55 -0400 Subject: [PATCH 1/2] Logstash Import and Template Assignment --- pillar/logstash/eval.sls | 9 +++++++ pillar/logstash/search.sls | 9 +++++++ salt/filebeat/etc/filebeat.yml | 4 +++ .../config/so/9000_output_zeek.conf.jinja | 15 +---------- .../config/so/9001_output_switch.conf.jinja | 27 ------------------- .../config/so/9002_output_import.conf.jinja | 18 +++---------- .../config/so/9004_output_flow.conf.jinja | 19 +++---------- .../config/so/9026_output_dhcp.conf.jinja | 26 ------------------ .../config/so/9029_output_esxi.conf.jinja | 25 ----------------- .../config/so/9030_output_greensql.conf.jinja | 25 ----------------- .../config/so/9031_output_iis.conf.jinja | 26 ------------------ .../config/so/9032_output_mcafee.conf.jinja | 26 ------------------ .../config/so/9033_output_snort.conf.jinja | 19 +++---------- .../config/so/9034_output_syslog.conf.jinja | 12 ++------- .../config/so/9100_output_osquery.conf.jinja | 9 +++---- .../config/so/9200_output_firewall.conf.jinja | 19 +++---------- .../config/so/9300_output_windows.conf.jinja | 27 ------------------- .../so/9301_output_dns_windows.conf.jinja | 27 ------------------- .../config/so/9400_output_suricata.conf.jinja | 18 +++---------- .../config/so/9500_output_beats.conf.jinja | 7 +++-- .../config/so/9600_output_ossec.conf.jinja | 17 ++---------- .../config/so/9700_output_strelka.conf.jinja | 17 ++---------- .../config/so/9999_output_redis.conf.jinja | 1 - .../templates/so/so-flow-template.json.jinja | 13 +++++++++ setup/so-functions | 5 ++++ 25 files changed, 70 insertions(+), 350 deletions(-) delete mode 100644 salt/logstash/pipelines/config/so/9001_output_switch.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9026_output_dhcp.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9029_output_esxi.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9030_output_greensql.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9031_output_iis.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9032_output_mcafee.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9300_output_windows.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9301_output_dns_windows.conf.jinja create mode 100644 salt/logstash/pipelines/templates/so/so-flow-template.json.jinja diff --git a/pillar/logstash/eval.sls b/pillar/logstash/eval.sls index d9e731e12..8613200fe 100644 --- a/pillar/logstash/eval.sls +++ b/pillar/logstash/eval.sls @@ -18,11 +18,20 @@ logstash: templates: - so/so-beats-template.json.jinja - so/so-common-template.json + - so/so-dhcp-template.json.jinja + - so/so-dns_windows-template.json.jinja + - so/so-esxi-template.json.jinja - so/so-firewall-template.json.jinja + - so/so-flow-template.json.jinja + - so/so-greensql-template.json.jinja - so/so-ids-template.json.jinja - so/so-import-template.json.jinja + - so/so-iss-template.json.jinja + - so/so-mcafee-template.json.jinja - so/so-osquery-template.json.jinja - so/so-ossec-template.json.jinja - so/so-strelka-template.json.jinja + - so/so-switch-template.json.jinja - so/so-syslog-template.json.jinja + - so/so-windows-template.json.jinja - so/so-zeek-template.json.jinja diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 6602e0591..2203b8452 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -14,11 +14,20 @@ logstash: templates: - so/so-beats-template.json.jinja - so/so-common-template.json + - so/so-dhcp-template.json.jinja + - so/so-dns_windows-template.json.jinja + - so/so-esxi-template.json.jinja - so/so-firewall-template.json.jinja + - so/so-flow-template.json.jinja + - so/so-greensql-template.json.jinja - so/so-ids-template.json.jinja - so/so-import-template.json.jinja + - so/so-iss-template.json.jinja + - so/so-mcafee-template.json.jinja - so/so-osquery-template.json.jinja - so/so-ossec-template.json.jinja - so/so-strelka-template.json.jinja + - so/so-switch-template.json.jinja - so/so-syslog-template.json.jinja + - so/so-windows-template.json.jinja - so/so-zeek-template.json.jinja diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 6d33c1bdf..77dd29dd9 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -126,6 +126,8 @@ filebeat.inputs: category: network imported: true processors: + - add_tags: + tags: [import] - dissect: tokenizer: "/nsm/import/%{import.id}/zeek/logs/%{import.file}" field: "log.file.path" @@ -164,6 +166,8 @@ filebeat.inputs: category: network imported: true processors: + - add_tags: + tags: [import] - dissect: tokenizer: "/nsm/import/%{import.id}/suricata/%{import.file}" field: "log.file.path" diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index 987614a2c..075aa0f93 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -3,21 +3,8 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - - -filter { - if [module] =~ "zeek" { - mutate { - ##add_tag => [ "conf_file_9000"] - } - } -} output { - if [module] =~ "zeek" { + if [module] =~ "zeek" and "import" not in [tags] { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" diff --git a/salt/logstash/pipelines/config/so/9001_output_switch.conf.jinja b/salt/logstash/pipelines/config/so/9001_output_switch.conf.jinja deleted file mode 100644 index 8e5e5f200..000000000 --- a/salt/logstash/pipelines/config/so/9001_output_switch.conf.jinja +++ /dev/null @@ -1,27 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if "switch" in [tags] and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9001"] - } - } -} -output { - if "switch" in [tags] and "test_data" not in [tags] { - #stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - index => "so-switch-%{+YYYY.MM.dd}" - template => "/so-common-template.json" - } - } -} diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja index 9153d5c44..f570e6171 100644 --- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja +++ b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja @@ -3,24 +3,14 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -# Updated by: Doug Burks -# Last Update: 5/16/2017 - -filter { - if "import" in [tags] and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9002"] - } - } -} output { - if "import" in [tags] and "test_data" not in [tags] { -# stdout { codec => rubydebug } + if "import" in [tags] { elasticsearch { + pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" index => "so-import-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/so-common-template.json" + template_name => "so-import" + template => "/so-import-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja index 2e1e79f8b..9d7b89f14 100644 --- a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja +++ b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja @@ -3,25 +3,14 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "sflow" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9004"] - } - } -} output { - if [event_type] == "sflow" and "test_data" not in [tags] { - #stdout { codec => rubydebug } + if [event_type] == "sflow" { elasticsearch { hosts => "{{ ES }}" index => "so-flow-%{+YYYY.MM.dd}" - template => "/so-common-template.json" + template_name => "so-flow" + template => "/so-flow-template.json" + template_overwrite => true } } } diff --git a/salt/logstash/pipelines/config/so/9026_output_dhcp.conf.jinja b/salt/logstash/pipelines/config/so/9026_output_dhcp.conf.jinja deleted file mode 100644 index 3da9e83ef..000000000 --- a/salt/logstash/pipelines/config/so/9026_output_dhcp.conf.jinja +++ /dev/null @@ -1,26 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "dhcp" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9026"] - } - } -} -output { - if [event_type] == "dhcp" and "test_data" not in [tags] { - #stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - template => "/so-common-template.json" - } - } -} diff --git a/salt/logstash/pipelines/config/so/9029_output_esxi.conf.jinja b/salt/logstash/pipelines/config/so/9029_output_esxi.conf.jinja deleted file mode 100644 index b84ab4ec9..000000000 --- a/salt/logstash/pipelines/config/so/9029_output_esxi.conf.jinja +++ /dev/null @@ -1,25 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "esxi" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9029"] - } - } -} -output { - if [event_type] == "esxi" and "test_data" not in [tags] { - elasticsearch { - hosts => "{{ ES }}" - template => "/so-common-template.json" - } - } -} diff --git a/salt/logstash/pipelines/config/so/9030_output_greensql.conf.jinja b/salt/logstash/pipelines/config/so/9030_output_greensql.conf.jinja deleted file mode 100644 index d6801530b..000000000 --- a/salt/logstash/pipelines/config/so/9030_output_greensql.conf.jinja +++ /dev/null @@ -1,25 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "greensql" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9030"] - } - } -} -output { - if [event_type] == "greensql" and "test_data" not in [tags] { - elasticsearch { - hosts => "{{ ES }}" - template => "/so-common-template.json" - } - } -} diff --git a/salt/logstash/pipelines/config/so/9031_output_iis.conf.jinja b/salt/logstash/pipelines/config/so/9031_output_iis.conf.jinja deleted file mode 100644 index 67616110f..000000000 --- a/salt/logstash/pipelines/config/so/9031_output_iis.conf.jinja +++ /dev/null @@ -1,26 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "iis" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9031"] - } - } -} -output { - if [event_type] == "iis" and "test_data" not in [tags] { - #stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - template => "/so-common-template.json" - } - } -} diff --git a/salt/logstash/pipelines/config/so/9032_output_mcafee.conf.jinja b/salt/logstash/pipelines/config/so/9032_output_mcafee.conf.jinja deleted file mode 100644 index c6641f671..000000000 --- a/salt/logstash/pipelines/config/so/9032_output_mcafee.conf.jinja +++ /dev/null @@ -1,26 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "mcafee" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9032"] - } - } -} -output { - if [event_type] == "mcafee" and "test_data" not in [tags] { - #stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - template => "/so-common-template.json" - } - } -} diff --git a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja index 0cc7a3b66..924081862 100644 --- a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja +++ b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja @@ -3,26 +3,13 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "ids" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9033"] - } - } -} output { - if [event_type] == "ids" and "test_data" not in [tags] { - #stdout { codec => rubydebug } + if [event_type] == "ids" and "import" not in [tags] { elasticsearch { hosts => "{{ ES }}" index => "so-ids-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/so-common-template.json" + template_name => "so-ids" + template => "/so-ids-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja index 59cae7b65..dc520ba6d 100644 --- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja +++ b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja @@ -3,22 +3,14 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} - -filter { - if [module] =~ "syslog" { - mutate { - ##add_tag => [ "conf_file_9000"] - } - } -} output { if [module] =~ "syslog" { elasticsearch { pipeline => "%{module}" hosts => "{{ ES }}" index => "so-syslog-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/so-common-template.json" + template_name => "so-syslog" + template => "/so-syslog-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja index 21ae77095..3351356dd 100644 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja @@ -3,18 +3,15 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -# Author: Josh Brower -# Last Update: 12/29/2018 -# Output to ES for osquery tagged logs - - output { if [module] =~ "osquery" { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" index => "so-osquery-%{+YYYY.MM.dd}" - template => "/so-common-template.json" + template_name => "so-osquery" + template => "/so-osquery-template.json" + template_overwrite => true } } } diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja index 54c75873d..39f78b07d 100644 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja @@ -3,26 +3,13 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if "firewall" in [tags] and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9200"] - } - } -} output { - if "firewall" in [tags] and "test_data" not in [tags] { -# stdout { codec => rubydebug } + if "firewall" in [tags] { elasticsearch { hosts => "{{ ES }}" index => "so-firewall-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/so-common-template.json" + template_name => "so-firewall" + template => "/so-firewall-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9300_output_windows.conf.jinja b/salt/logstash/pipelines/config/so/9300_output_windows.conf.jinja deleted file mode 100644 index cddda5541..000000000 --- a/salt/logstash/pipelines/config/so/9300_output_windows.conf.jinja +++ /dev/null @@ -1,27 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "windows" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9300"] - } - } -} -output { - if [event_type] == "windows" and "test_data" not in [tags] { - #stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - index => "so-windows-%{+YYYY.MM.dd}" - template => "/so-common-template.json" - } - } -} diff --git a/salt/logstash/pipelines/config/so/9301_output_dns_windows.conf.jinja b/salt/logstash/pipelines/config/so/9301_output_dns_windows.conf.jinja deleted file mode 100644 index 84fd1f5f7..000000000 --- a/salt/logstash/pipelines/config/so/9301_output_dns_windows.conf.jinja +++ /dev/null @@ -1,27 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "dns" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9301"] - } - } -} -output { - if [event_type] == "dns" and "test_data" not in [tags] { - #stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - index => "so-%{+YYYY.MM.dd}" - template => "/so-common-template.json" - } - } -} diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index 1d36d774d..48247ca9c 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -3,26 +3,14 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [module] == "suricata" { - mutate { - ##add_tag => [ "conf_file_9400"] - } - } -} output { - if [module] =~ "suricata" { + if [module] =~ "suricata" and "import" not in [tags] { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" index => "so-ids-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/so-common-template.json" + template_name => "so-ids" + template => "/so-ids-template.json" } } } diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja index 932a194ab..61a331873 100644 --- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja +++ b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja @@ -3,15 +3,14 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} - output { - if "beat-ext" in [tags] { + if "beat-ext" in [tags] and "import" not in [tags] { elasticsearch { pipeline => "beats.common" hosts => "{{ ES }}" index => "so-beats-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/so-common-template.json" + template_name => "so-beats" + template => "/so-beats-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja index 5a8f9f5ba..7b9af4ee0 100644 --- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja +++ b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja @@ -3,27 +3,14 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Updated by: Doug Burks -# Last Update: 9/19/2018 - -filter { - if [module] =~ "ossec" { - mutate { - ##add_tag => [ "conf_file_9600"] - } - } -} - output { if [module] =~ "ossec" { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" index => "so-ossec-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/so-common-template.json" + template_name => "so-ossec" + template => "/so-ossec-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja index 5116b86ea..e4869b4a5 100644 --- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja +++ b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja @@ -3,27 +3,14 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - - -filter { - if [module] =~ "strelka" { - mutate { - ##add_tag => [ "conf_file_9000"] - } - } -} output { if [event_type] =~ "strelka" { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" index => "so-strelka-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/so-common-template.json" + template_name => "so-strelka" + template => "/so-strelka-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja index afa8d290a..58bfc5b07 100644 --- a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja @@ -1,6 +1,5 @@ {% set MASTER = salt['pillar.get']('static:masterip', '') %} {% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %} - output { redis { host => '{{ MASTER }}' diff --git a/salt/logstash/pipelines/templates/so/so-flow-template.json.jinja b/salt/logstash/pipelines/templates/so/so-flow-template.json.jinja new file mode 100644 index 000000000..6c8f2fa9f --- /dev/null +++ b/salt/logstash/pipelines/templates/so/so-flow-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-flow:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-flow:refresh', '30s') %} +{ + "index_patterns": ["so-flow-*"], + "version": 50001, + "order": 11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/setup/so-functions b/setup/so-functions index 2eca9874d..4337d720c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1026,6 +1026,11 @@ master_static() { " warm: 7"\ " close: 30"\ " delete: 365"\ + " so-flow:"\ + " shards: 1"\ + " warm: 7"\ + " close: 30"\ + " delete: 365"\ " so-ids:"\ " shards: 1"\ " warm: 7"\ From 46d572fa8cd8c319be3b59b92bb9c5ebbd72b39d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 10 Jul 2020 15:51:12 -0400 Subject: [PATCH 2/2] Fix Filebeat spacing --- pillar/logstash/eval.sls | 8 -------- pillar/logstash/search.sls | 8 -------- salt/filebeat/etc/filebeat.yml | 4 ++-- 3 files changed, 2 insertions(+), 18 deletions(-) diff --git a/pillar/logstash/eval.sls b/pillar/logstash/eval.sls index 8613200fe..fcdd13bb7 100644 --- a/pillar/logstash/eval.sls +++ b/pillar/logstash/eval.sls @@ -18,20 +18,12 @@ logstash: templates: - so/so-beats-template.json.jinja - so/so-common-template.json - - so/so-dhcp-template.json.jinja - - so/so-dns_windows-template.json.jinja - - so/so-esxi-template.json.jinja - so/so-firewall-template.json.jinja - so/so-flow-template.json.jinja - - so/so-greensql-template.json.jinja - so/so-ids-template.json.jinja - so/so-import-template.json.jinja - - so/so-iss-template.json.jinja - - so/so-mcafee-template.json.jinja - so/so-osquery-template.json.jinja - so/so-ossec-template.json.jinja - so/so-strelka-template.json.jinja - - so/so-switch-template.json.jinja - so/so-syslog-template.json.jinja - - so/so-windows-template.json.jinja - so/so-zeek-template.json.jinja diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 2203b8452..9c069fd20 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -14,20 +14,12 @@ logstash: templates: - so/so-beats-template.json.jinja - so/so-common-template.json - - so/so-dhcp-template.json.jinja - - so/so-dns_windows-template.json.jinja - - so/so-esxi-template.json.jinja - so/so-firewall-template.json.jinja - so/so-flow-template.json.jinja - - so/so-greensql-template.json.jinja - so/so-ids-template.json.jinja - so/so-import-template.json.jinja - - so/so-iss-template.json.jinja - - so/so-mcafee-template.json.jinja - so/so-osquery-template.json.jinja - so/so-ossec-template.json.jinja - so/so-strelka-template.json.jinja - - so/so-switch-template.json.jinja - so/so-syslog-template.json.jinja - - so/so-windows-template.json.jinja - so/so-zeek-template.json.jinja diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 77dd29dd9..7b2289095 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -127,7 +127,7 @@ filebeat.inputs: imported: true processors: - add_tags: - tags: [import] + tags: [import] - dissect: tokenizer: "/nsm/import/%{import.id}/zeek/logs/%{import.file}" field: "log.file.path" @@ -167,7 +167,7 @@ filebeat.inputs: imported: true processors: - add_tags: - tags: [import] + tags: [import] - dissect: tokenizer: "/nsm/import/%{import.id}/suricata/%{import.file}" field: "log.file.path"