From 5eb33d5ac7aeae18bf7b5fd95a7c3a3d18250bb5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 10 Jul 2020 13:53:55 -0400 Subject: [PATCH 1/7] Logstash Import and Template Assignment --- pillar/logstash/eval.sls | 9 +++++++ pillar/logstash/search.sls | 9 +++++++ salt/filebeat/etc/filebeat.yml | 4 +++ .../config/so/9000_output_zeek.conf.jinja | 15 +---------- .../config/so/9001_output_switch.conf.jinja | 27 ------------------- .../config/so/9002_output_import.conf.jinja | 18 +++---------- .../config/so/9004_output_flow.conf.jinja | 19 +++---------- .../config/so/9026_output_dhcp.conf.jinja | 26 ------------------ .../config/so/9029_output_esxi.conf.jinja | 25 ----------------- .../config/so/9030_output_greensql.conf.jinja | 25 ----------------- .../config/so/9031_output_iis.conf.jinja | 26 ------------------ .../config/so/9032_output_mcafee.conf.jinja | 26 ------------------ .../config/so/9033_output_snort.conf.jinja | 19 +++---------- .../config/so/9034_output_syslog.conf.jinja | 12 ++------- .../config/so/9100_output_osquery.conf.jinja | 9 +++---- .../config/so/9200_output_firewall.conf.jinja | 19 +++---------- .../config/so/9300_output_windows.conf.jinja | 27 ------------------- .../so/9301_output_dns_windows.conf.jinja | 27 ------------------- .../config/so/9400_output_suricata.conf.jinja | 18 +++---------- .../config/so/9500_output_beats.conf.jinja | 7 +++-- .../config/so/9600_output_ossec.conf.jinja | 17 ++---------- .../config/so/9700_output_strelka.conf.jinja | 17 ++---------- .../config/so/9999_output_redis.conf.jinja | 1 - .../templates/so/so-flow-template.json.jinja | 13 +++++++++ setup/so-functions | 5 ++++ 25 files changed, 70 insertions(+), 350 deletions(-) delete mode 100644 salt/logstash/pipelines/config/so/9001_output_switch.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9026_output_dhcp.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9029_output_esxi.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9030_output_greensql.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9031_output_iis.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9032_output_mcafee.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9300_output_windows.conf.jinja delete mode 100644 salt/logstash/pipelines/config/so/9301_output_dns_windows.conf.jinja create mode 100644 salt/logstash/pipelines/templates/so/so-flow-template.json.jinja diff --git a/pillar/logstash/eval.sls b/pillar/logstash/eval.sls index d9e731e12..8613200fe 100644 --- a/pillar/logstash/eval.sls +++ b/pillar/logstash/eval.sls @@ -18,11 +18,20 @@ logstash: templates: - so/so-beats-template.json.jinja - so/so-common-template.json + - so/so-dhcp-template.json.jinja + - so/so-dns_windows-template.json.jinja + - so/so-esxi-template.json.jinja - so/so-firewall-template.json.jinja + - so/so-flow-template.json.jinja + - so/so-greensql-template.json.jinja - so/so-ids-template.json.jinja - so/so-import-template.json.jinja + - so/so-iss-template.json.jinja + - so/so-mcafee-template.json.jinja - so/so-osquery-template.json.jinja - so/so-ossec-template.json.jinja - so/so-strelka-template.json.jinja + - so/so-switch-template.json.jinja - so/so-syslog-template.json.jinja + - so/so-windows-template.json.jinja - so/so-zeek-template.json.jinja diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 6602e0591..2203b8452 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -14,11 +14,20 @@ logstash: templates: - so/so-beats-template.json.jinja - so/so-common-template.json + - so/so-dhcp-template.json.jinja + - so/so-dns_windows-template.json.jinja + - so/so-esxi-template.json.jinja - so/so-firewall-template.json.jinja + - so/so-flow-template.json.jinja + - so/so-greensql-template.json.jinja - so/so-ids-template.json.jinja - so/so-import-template.json.jinja + - so/so-iss-template.json.jinja + - so/so-mcafee-template.json.jinja - so/so-osquery-template.json.jinja - so/so-ossec-template.json.jinja - so/so-strelka-template.json.jinja + - so/so-switch-template.json.jinja - so/so-syslog-template.json.jinja + - so/so-windows-template.json.jinja - so/so-zeek-template.json.jinja diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 6d33c1bdf..77dd29dd9 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -126,6 +126,8 @@ filebeat.inputs: category: network imported: true processors: + - add_tags: + tags: [import] - dissect: tokenizer: "/nsm/import/%{import.id}/zeek/logs/%{import.file}" field: "log.file.path" @@ -164,6 +166,8 @@ filebeat.inputs: category: network imported: true processors: + - add_tags: + tags: [import] - dissect: tokenizer: "/nsm/import/%{import.id}/suricata/%{import.file}" field: "log.file.path" diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index 987614a2c..075aa0f93 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -3,21 +3,8 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - - -filter { - if [module] =~ "zeek" { - mutate { - ##add_tag => [ "conf_file_9000"] - } - } -} output { - if [module] =~ "zeek" { + if [module] =~ "zeek" and "import" not in [tags] { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" diff --git a/salt/logstash/pipelines/config/so/9001_output_switch.conf.jinja b/salt/logstash/pipelines/config/so/9001_output_switch.conf.jinja deleted file mode 100644 index 8e5e5f200..000000000 --- a/salt/logstash/pipelines/config/so/9001_output_switch.conf.jinja +++ /dev/null @@ -1,27 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if "switch" in [tags] and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9001"] - } - } -} -output { - if "switch" in [tags] and "test_data" not in [tags] { - #stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - index => "so-switch-%{+YYYY.MM.dd}" - template => "/so-common-template.json" - } - } -} diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja index 9153d5c44..f570e6171 100644 --- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja +++ b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja @@ -3,24 +3,14 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -# Updated by: Doug Burks -# Last Update: 5/16/2017 - -filter { - if "import" in [tags] and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9002"] - } - } -} output { - if "import" in [tags] and "test_data" not in [tags] { -# stdout { codec => rubydebug } + if "import" in [tags] { elasticsearch { + pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" index => "so-import-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/so-common-template.json" + template_name => "so-import" + template => "/so-import-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja index 2e1e79f8b..9d7b89f14 100644 --- a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja +++ b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja @@ -3,25 +3,14 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "sflow" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9004"] - } - } -} output { - if [event_type] == "sflow" and "test_data" not in [tags] { - #stdout { codec => rubydebug } + if [event_type] == "sflow" { elasticsearch { hosts => "{{ ES }}" index => "so-flow-%{+YYYY.MM.dd}" - template => "/so-common-template.json" + template_name => "so-flow" + template => "/so-flow-template.json" + template_overwrite => true } } } diff --git a/salt/logstash/pipelines/config/so/9026_output_dhcp.conf.jinja b/salt/logstash/pipelines/config/so/9026_output_dhcp.conf.jinja deleted file mode 100644 index 3da9e83ef..000000000 --- a/salt/logstash/pipelines/config/so/9026_output_dhcp.conf.jinja +++ /dev/null @@ -1,26 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "dhcp" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9026"] - } - } -} -output { - if [event_type] == "dhcp" and "test_data" not in [tags] { - #stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - template => "/so-common-template.json" - } - } -} diff --git a/salt/logstash/pipelines/config/so/9029_output_esxi.conf.jinja b/salt/logstash/pipelines/config/so/9029_output_esxi.conf.jinja deleted file mode 100644 index b84ab4ec9..000000000 --- a/salt/logstash/pipelines/config/so/9029_output_esxi.conf.jinja +++ /dev/null @@ -1,25 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "esxi" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9029"] - } - } -} -output { - if [event_type] == "esxi" and "test_data" not in [tags] { - elasticsearch { - hosts => "{{ ES }}" - template => "/so-common-template.json" - } - } -} diff --git a/salt/logstash/pipelines/config/so/9030_output_greensql.conf.jinja b/salt/logstash/pipelines/config/so/9030_output_greensql.conf.jinja deleted file mode 100644 index d6801530b..000000000 --- a/salt/logstash/pipelines/config/so/9030_output_greensql.conf.jinja +++ /dev/null @@ -1,25 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "greensql" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9030"] - } - } -} -output { - if [event_type] == "greensql" and "test_data" not in [tags] { - elasticsearch { - hosts => "{{ ES }}" - template => "/so-common-template.json" - } - } -} diff --git a/salt/logstash/pipelines/config/so/9031_output_iis.conf.jinja b/salt/logstash/pipelines/config/so/9031_output_iis.conf.jinja deleted file mode 100644 index 67616110f..000000000 --- a/salt/logstash/pipelines/config/so/9031_output_iis.conf.jinja +++ /dev/null @@ -1,26 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "iis" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9031"] - } - } -} -output { - if [event_type] == "iis" and "test_data" not in [tags] { - #stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - template => "/so-common-template.json" - } - } -} diff --git a/salt/logstash/pipelines/config/so/9032_output_mcafee.conf.jinja b/salt/logstash/pipelines/config/so/9032_output_mcafee.conf.jinja deleted file mode 100644 index c6641f671..000000000 --- a/salt/logstash/pipelines/config/so/9032_output_mcafee.conf.jinja +++ /dev/null @@ -1,26 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "mcafee" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9032"] - } - } -} -output { - if [event_type] == "mcafee" and "test_data" not in [tags] { - #stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - template => "/so-common-template.json" - } - } -} diff --git a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja index 0cc7a3b66..924081862 100644 --- a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja +++ b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja @@ -3,26 +3,13 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "ids" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9033"] - } - } -} output { - if [event_type] == "ids" and "test_data" not in [tags] { - #stdout { codec => rubydebug } + if [event_type] == "ids" and "import" not in [tags] { elasticsearch { hosts => "{{ ES }}" index => "so-ids-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/so-common-template.json" + template_name => "so-ids" + template => "/so-ids-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja index 59cae7b65..dc520ba6d 100644 --- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja +++ b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja @@ -3,22 +3,14 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} - -filter { - if [module] =~ "syslog" { - mutate { - ##add_tag => [ "conf_file_9000"] - } - } -} output { if [module] =~ "syslog" { elasticsearch { pipeline => "%{module}" hosts => "{{ ES }}" index => "so-syslog-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/so-common-template.json" + template_name => "so-syslog" + template => "/so-syslog-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja index 21ae77095..3351356dd 100644 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja @@ -3,18 +3,15 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -# Author: Josh Brower -# Last Update: 12/29/2018 -# Output to ES for osquery tagged logs - - output { if [module] =~ "osquery" { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" index => "so-osquery-%{+YYYY.MM.dd}" - template => "/so-common-template.json" + template_name => "so-osquery" + template => "/so-osquery-template.json" + template_overwrite => true } } } diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja index 54c75873d..39f78b07d 100644 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja @@ -3,26 +3,13 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if "firewall" in [tags] and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9200"] - } - } -} output { - if "firewall" in [tags] and "test_data" not in [tags] { -# stdout { codec => rubydebug } + if "firewall" in [tags] { elasticsearch { hosts => "{{ ES }}" index => "so-firewall-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/so-common-template.json" + template_name => "so-firewall" + template => "/so-firewall-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9300_output_windows.conf.jinja b/salt/logstash/pipelines/config/so/9300_output_windows.conf.jinja deleted file mode 100644 index cddda5541..000000000 --- a/salt/logstash/pipelines/config/so/9300_output_windows.conf.jinja +++ /dev/null @@ -1,27 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "windows" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9300"] - } - } -} -output { - if [event_type] == "windows" and "test_data" not in [tags] { - #stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - index => "so-windows-%{+YYYY.MM.dd}" - template => "/so-common-template.json" - } - } -} diff --git a/salt/logstash/pipelines/config/so/9301_output_dns_windows.conf.jinja b/salt/logstash/pipelines/config/so/9301_output_dns_windows.conf.jinja deleted file mode 100644 index 84fd1f5f7..000000000 --- a/salt/logstash/pipelines/config/so/9301_output_dns_windows.conf.jinja +++ /dev/null @@ -1,27 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "dns" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9301"] - } - } -} -output { - if [event_type] == "dns" and "test_data" not in [tags] { - #stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - index => "so-%{+YYYY.MM.dd}" - template => "/so-common-template.json" - } - } -} diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index 1d36d774d..48247ca9c 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -3,26 +3,14 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [module] == "suricata" { - mutate { - ##add_tag => [ "conf_file_9400"] - } - } -} output { - if [module] =~ "suricata" { + if [module] =~ "suricata" and "import" not in [tags] { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" index => "so-ids-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/so-common-template.json" + template_name => "so-ids" + template => "/so-ids-template.json" } } } diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja index 932a194ab..61a331873 100644 --- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja +++ b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja @@ -3,15 +3,14 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} - output { - if "beat-ext" in [tags] { + if "beat-ext" in [tags] and "import" not in [tags] { elasticsearch { pipeline => "beats.common" hosts => "{{ ES }}" index => "so-beats-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/so-common-template.json" + template_name => "so-beats" + template => "/so-beats-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja index 5a8f9f5ba..7b9af4ee0 100644 --- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja +++ b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja @@ -3,27 +3,14 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Updated by: Doug Burks -# Last Update: 9/19/2018 - -filter { - if [module] =~ "ossec" { - mutate { - ##add_tag => [ "conf_file_9600"] - } - } -} - output { if [module] =~ "ossec" { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" index => "so-ossec-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/so-common-template.json" + template_name => "so-ossec" + template => "/so-ossec-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja index 5116b86ea..e4869b4a5 100644 --- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja +++ b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja @@ -3,27 +3,14 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - - -filter { - if [module] =~ "strelka" { - mutate { - ##add_tag => [ "conf_file_9000"] - } - } -} output { if [event_type] =~ "strelka" { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" index => "so-strelka-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/so-common-template.json" + template_name => "so-strelka" + template => "/so-strelka-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja index afa8d290a..58bfc5b07 100644 --- a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja @@ -1,6 +1,5 @@ {% set MASTER = salt['pillar.get']('static:masterip', '') %} {% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %} - output { redis { host => '{{ MASTER }}' diff --git a/salt/logstash/pipelines/templates/so/so-flow-template.json.jinja b/salt/logstash/pipelines/templates/so/so-flow-template.json.jinja new file mode 100644 index 000000000..6c8f2fa9f --- /dev/null +++ b/salt/logstash/pipelines/templates/so/so-flow-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-flow:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-flow:refresh', '30s') %} +{ + "index_patterns": ["so-flow-*"], + "version": 50001, + "order": 11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/setup/so-functions b/setup/so-functions index 2eca9874d..4337d720c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1026,6 +1026,11 @@ master_static() { " warm: 7"\ " close: 30"\ " delete: 365"\ + " so-flow:"\ + " shards: 1"\ + " warm: 7"\ + " close: 30"\ + " delete: 365"\ " so-ids:"\ " shards: 1"\ " warm: 7"\ From 6e99ca600ff0ed4f078db25d55722979604a4baa Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 10 Jul 2020 17:56:40 +0000 Subject: [PATCH 2/7] Move zeek_clean to so-sensor-clean --- salt/common/init.sls | 10 ++ salt/common/tools/sbin/so-sensor-clean | 121 +++++++++++++++++++++++++ salt/zeek/init.sls | 26 +++--- 3 files changed, 144 insertions(+), 13 deletions(-) create mode 100644 salt/common/tools/sbin/so-sensor-clean diff --git a/salt/common/init.sls b/salt/common/init.sls index 0ecba198d..af2cb27b2 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -131,3 +131,13 @@ utilsyncscripts: - file_mode: 755 - template: jinja - source: salt://common/tools/sbin + +# Add sensor cleanup +/usr/sbin/so-sensor-clean: + cron.present: + - user: root + - minute: '*' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' diff --git a/salt/common/tools/sbin/so-sensor-clean b/salt/common/tools/sbin/so-sensor-clean new file mode 100644 index 000000000..37ed69d9a --- /dev/null +++ b/salt/common/tools/sbin/so-sensor-clean @@ -0,0 +1,121 @@ +#!/bin/bash + +# Delete Zeek Logs based on defined CRIT_DISK_USAGE value + +# Copyright 2014,2015,2016,2017,2018, 2019 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +SENSOR_DIR='/nsm' +CRIT_DISK_USAGE=30 +CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %) +LOG="/opt/so/log/sensor_clean.log" +TODAY=$(date -u "+%Y-%m-%d") + +clean () { + ## find the oldest Zeek logs directory + OLDEST_DIR=$(ls /nsm/zeek/logs/ | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1) + if [ -z "$OLDEST_DIR" -o "$OLDEST_DIR" == ".." -o "$OLDEST_DIR" == "." ] + then + echo "$(date) - No old Zeek logs available to clean up in /nsm/zeek/logs/" >> $LOG + #exit 0 + else + echo "$(date) - Removing directory: /nsm/zeek/logs/$OLDEST_DIR" >> $LOG + rm -rf /nsm/zeek/logs/"$OLDEST_DIR" + fi + + + ## Remarking for now, as we are moving extracted files to /nsm/strelka/processed + ## find oldest files in extracted directory and exclude today + #OLDEST_EXTRACT=$(find /nsm/zeek/extracted/complete -type f -printf '%T+ %p\n' 2>/dev/null | sort | grep -v $TODAY | head -n 1) + #if [ -z "$OLDEST_EXTRACT" -o "$OLDEST_EXTRACT" == ".." -o "$OLDEST_EXTRACT" == "." ] + #then + # echo "$(date) - No old extracted files available to clean up in /nsm/zeek/extracted/complete" >> $LOG + #else + # OLDEST_EXTRACT_DATE=`echo $OLDEST_EXTRACT | awk '{print $1}' | cut -d+ -f1` + # OLDEST_EXTRACT_FILE=`echo $OLDEST_EXTRACT | awk '{print $2}'` + # echo "$(date) - Removing extracted files for $OLDEST_EXTRACT_DATE" >> $LOG + # find /nsm/zeek/extracted/complete -type f -printf '%T+ %p\n' | grep $OLDEST_EXTRACT_DATE | awk '{print $2}' |while read FILE + # do + # echo "$(date) - Removing extracted file: $FILE" >> $LOG + # rm -f "$FILE" + # done + #fi + + ## Clean up Zeek extracted files processed by Strelka + STRELKA_FILES='/nsm/strelka/processed' + OLDEST_STRELKA=$(find $STRELKA_FILES -type f -printf '%T+ %p\n' | sort -n | head -n 1 ) + if [ -z "$OLDEST_STRELKA" -o "$OLDEST_STRELKA" == ".." -o "$OLDEST_STRELKA" == "." ] + then + echo "$(date) - No old files available to clean up in $STRELKA_FILES" >> $LOG + else + OLDEST_STRELKA_DATE=`echo $OLDEST_STRELKA | awk '{print $1}' | cut -d+ -f1` + OLDEST_STRELKA_FILE=`echo $OLDEST_STRELKA | awk '{print $2}'` + echo "$(date) - Removing extracted files for $OLDEST_STRELKA_DATE" >> $LOG + find $STRELKA_FILES -type f -printf '%T+ %p\n' | grep $OLDEST_STRELKA_DATE | awk '{print $2}' |while read FILE + do + echo "$(date) - Removing file: $FILE" >> $LOG + rm -f "$FILE" + done + fi + + ## Clean up Suricata log files + SURICATA_LOGS='/nsm/suricata' + OLDEST_SURICATA=$(find $STRELKA_FILES -type f -printf '%T+ %p\n' | sort -n | head -n 1) + if [ -z "$OLDEST_SURICATA" -o "$OLDEST_SURICATA" == ".." -o "$OLDEST_SURICATA" == "." ] + then + echo "$(date) - No old files available to clean up in $SURICATA_LOGS" >> $LOG + else + OLDEST_SURICATA_DATE=`echo $OLDEST_SURICATA | awk '{print $1}' | cut -d+ -f1` + OLDEST_SURICATA_FILE=`echo $OLDEST_SURICATA | awk '{print $2}'` + echo "$(date) - Removing logs for $OLDEST_SURICATA_DATE" >> $LOG + find $SURICATA_LOGS -type f -printf '%T+ %p\n' | grep $OLDEST_SURICATA_DATE | awk '{print $2}' |while read FILE + do + echo "$(date) - Removing file: $FILE" >> $LOG + rm -f "$FILE" + done + fi + + ## Clean up extracted pcaps from Steno + PCAPS='/nsm/pcapout' + OLDEST_PCAP=$(find $PCAPS -type f -printf '%T+ %p\n' | sort -n | head -n 1 ) + if [ -z "$OLDEST_PCAP" -o "$OLDEST_PCAP" == ".." -o "$OLDEST_PCAP" == "." ] + then + echo "$(date) - No old files available to clean up in $PCAPS" >> $LOG + else + OLDEST_PCAP_DATE=`echo $OLDEST_PCAP | awk '{print $1}' | cut -d+ -f1` + OLDEST_PCAP_FILE=`echo $OLDEST_PCAP | awk '{print $2}'` + echo "$(date) - Removing extracted files for $OLDEST_PCAP_DATE" >> $LOG + find $PCAPS -type f -printf '%T+ %p\n' | grep $OLDEST_PCAP_DATE | awk '{print $2}' |while read FILE + do + echo "$(date) - Removing file: $FILE" >> $LOG + rm -f "$FILE" + done + fi +} + +# Check to see if we are already running +IS_RUNNING=$(ps aux | grep "sensor_clean" | grep -v grep | wc -l) +[ "$IS_RUNNING" -gt 2 ] && echo "$(date) - $IS_RUNNING sensor clean script processes running...exiting." >> $LOG && exit 0 + +if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then + while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; + do + clean + CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %) + done +else + echo "$(date) - Current usage value of $CUR_USAGE not greater than CRIT_DISK_USAGE value of $CRIT_DISK_USAGE..." >> $LOG +fi + diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index 246b43c90..1ce991907 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -86,20 +86,20 @@ nodecfgsync: - group: 939 - template: jinja -zeekcleanscript: - file.managed: - - name: /usr/local/bin/zeek_clean - - source: salt://zeek/cron/zeek_clean - - mode: 755 +#zeekcleanscript: +# file.managed: +# - name: /usr/local/bin/zeek_clean +# - source: salt://zeek/cron/zeek_clean +# - mode: 755 -/usr/local/bin/zeek_clean: - cron.present: - - user: root - - minute: '*' - - hour: '*' - - daymonth: '*' - - month: '*' - - dayweek: '*' +#/usr/local/bin/zeek_clean: +# cron.present: +# - user: root +# - minute: '*' +# - hour: '*' +# - daymonth: '*' +# - month: '*' +# - dayweek: '*' plcronscript: file.managed: From 811bbb4cb0cf6439232bfdc674634c02bf4d3f8c Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 10 Jul 2020 12:49:12 -0400 Subject: [PATCH 3/7] Require sudo to run an import --- salt/common/tools/sbin/so-import-pcap | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-import-pcap b/salt/common/tools/sbin/so-import-pcap index 0c99ede64..d6815c450 100755 --- a/salt/common/tools/sbin/so-import-pcap +++ b/salt/common/tools/sbin/so-import-pcap @@ -17,7 +17,9 @@ {% set MASTER = salt['grains.get']('master') %} {% set VERSION = salt['pillar.get']('static:soversion') %} -{%- set MASTERIP = salt['pillar.get']('static:masterip') -%} +{% set MASTERIP = salt['pillar.get']('static:masterip') %} + +. /usr/sbin/so-common function usage { cat << EOF @@ -220,6 +222,6 @@ https://{{ MASTERIP }}/#/hunt?q=%2a%20%7C%20groupby%20event.module%20event.datas or you can manually set your Time Range to be: From: $START_OLDEST To: $END_NEWEST -Please note that it may take 30 seconds or more for events to appear in Kibana. +Please note that it may take 30 seconds or more for events to appear in Onion Hunt. EOF fi From 358ef78cd940afe8f6c0f483cf8f0c659b6c973f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 10 Jul 2020 13:58:46 -0400 Subject: [PATCH 4/7] Do not stop curator since imported data will now be placed into a longer term so-import index --- salt/common/tools/sbin/so-import-pcap | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/salt/common/tools/sbin/so-import-pcap b/salt/common/tools/sbin/so-import-pcap index d6815c450..289d27024 100755 --- a/salt/common/tools/sbin/so-import-pcap +++ b/salt/common/tools/sbin/so-import-pcap @@ -25,7 +25,8 @@ function usage { cat << EOF Usage: $0 [pcap-file-2] [pcap-file-N] -Imports one or more PCAP files for analysis. If available, curator will be automatically stopped. +Imports one or more PCAP files onto a sensor node. The PCAP traffic will be analyzed and +made available for review in the Security Onion toolset. EOF } @@ -112,14 +113,6 @@ for i in "$@"; do fi done -if ! [ -d /opt/so/conf/curator ]; then - echo "Curator is not installed on this node and cannot be stopped automatically." -else - echo -n "Stopping curator..." - so-curator-stop > /dev/null 2>&1 - echo "Done" -fi - # track if we have any valid or invalid pcaps INVALID_PCAPS="no" VALID_PCAPS="no" From 37ab252e01f512de3fe1170b3ba065154b8cb97a Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 10 Jul 2020 19:30:08 +0000 Subject: [PATCH 5/7] Add sensor proc eval --- salt/common/init.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/init.sls b/salt/common/init.sls index af2cb27b2..3515205ef 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -132,6 +132,7 @@ utilsyncscripts: - template: jinja - source: salt://common/tools/sbin +{% if role in ['eval', 'standalone', 'sensor', 'heavynode'] %} # Add sensor cleanup /usr/sbin/so-sensor-clean: cron.present: @@ -141,3 +142,4 @@ utilsyncscripts: - daymonth: '*' - month: '*' - dayweek: '*' +{% endif %} From d7f7fb801c91ec44a01e0203f0362f1f2541658f Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 10 Jul 2020 19:35:27 +0000 Subject: [PATCH 6/7] Set role --- salt/common/init.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/init.sls b/salt/common/init.sls index 3515205ef..ef558d3e5 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -1,3 +1,5 @@ +{% set role = grains.id.split('_') | last %} + # Add socore Group socoregroup: group.present: From 46d572fa8cd8c319be3b59b92bb9c5ebbd72b39d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 10 Jul 2020 15:51:12 -0400 Subject: [PATCH 7/7] Fix Filebeat spacing --- pillar/logstash/eval.sls | 8 -------- pillar/logstash/search.sls | 8 -------- salt/filebeat/etc/filebeat.yml | 4 ++-- 3 files changed, 2 insertions(+), 18 deletions(-) diff --git a/pillar/logstash/eval.sls b/pillar/logstash/eval.sls index 8613200fe..fcdd13bb7 100644 --- a/pillar/logstash/eval.sls +++ b/pillar/logstash/eval.sls @@ -18,20 +18,12 @@ logstash: templates: - so/so-beats-template.json.jinja - so/so-common-template.json - - so/so-dhcp-template.json.jinja - - so/so-dns_windows-template.json.jinja - - so/so-esxi-template.json.jinja - so/so-firewall-template.json.jinja - so/so-flow-template.json.jinja - - so/so-greensql-template.json.jinja - so/so-ids-template.json.jinja - so/so-import-template.json.jinja - - so/so-iss-template.json.jinja - - so/so-mcafee-template.json.jinja - so/so-osquery-template.json.jinja - so/so-ossec-template.json.jinja - so/so-strelka-template.json.jinja - - so/so-switch-template.json.jinja - so/so-syslog-template.json.jinja - - so/so-windows-template.json.jinja - so/so-zeek-template.json.jinja diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 2203b8452..9c069fd20 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -14,20 +14,12 @@ logstash: templates: - so/so-beats-template.json.jinja - so/so-common-template.json - - so/so-dhcp-template.json.jinja - - so/so-dns_windows-template.json.jinja - - so/so-esxi-template.json.jinja - so/so-firewall-template.json.jinja - so/so-flow-template.json.jinja - - so/so-greensql-template.json.jinja - so/so-ids-template.json.jinja - so/so-import-template.json.jinja - - so/so-iss-template.json.jinja - - so/so-mcafee-template.json.jinja - so/so-osquery-template.json.jinja - so/so-ossec-template.json.jinja - so/so-strelka-template.json.jinja - - so/so-switch-template.json.jinja - so/so-syslog-template.json.jinja - - so/so-windows-template.json.jinja - so/so-zeek-template.json.jinja diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 77dd29dd9..7b2289095 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -127,7 +127,7 @@ filebeat.inputs: imported: true processors: - add_tags: - tags: [import] + tags: [import] - dissect: tokenizer: "/nsm/import/%{import.id}/zeek/logs/%{import.file}" field: "log.file.path" @@ -167,7 +167,7 @@ filebeat.inputs: imported: true processors: - add_tags: - tags: [import] + tags: [import] - dissect: tokenizer: "/nsm/import/%{import.id}/suricata/%{import.file}" field: "log.file.path"